diff --git a/bin/ext/llapdump.sh b/bin/ext/llapdump.sh deleted file mode 100644 index 5a905fc6cb87..000000000000 --- a/bin/ext/llapdump.sh +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -THISSERVICE=llapdump -export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} " - -llapdump () { - CLASS=org.apache.hadoop.hive.llap.LlapDump - HIVE_OPTS='' - execHiveCmd $CLASS "$@" -} - -llapdump_help () { - echo "usage ./hive --service llapdump [-l ] [-u ] [-p ] " - echo "" - echo " --location (-l) hs2 url" - echo " --user (-u) user name" - echo " --pwd (-p) password" -} diff --git a/bin/hive b/bin/hive index 41ba504d8a52..d6aac1cf1490 100755 --- a/bin/hive +++ b/bin/hive @@ -50,10 +50,6 @@ while [ $# -gt 0 ]; do SERVICE=orcfiledump shift ;; - --llapdump) - SERVICE=llapdump - shift - ;; --replMigration) SERVICE=replMigration shift diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 301adc2c895d..bb51119e3a2b 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -130,12 +130,6 @@ public String toString() { public String toString() { return "RCfile"; } - }, - LLAP { - @Override - public String toString() { - return "Llap"; - } }; public static ResultFileFormat getInvalid() { @@ -5357,48 +5351,6 @@ public static enum ConfVars { LLAP_VALIDATE_ACLS("hive.llap.validate.acls", true, "Whether LLAP should reject permissive ACLs in some cases (e.g. its own management\n" + "protocol or ZK paths), similar to how ssh refuses a key with bad access permissions."), - LLAP_DAEMON_OUTPUT_SERVICE_PORT("hive.llap.daemon.output.service.port", 15003, - "LLAP daemon output service port"), - LLAP_DAEMON_OUTPUT_STREAM_TIMEOUT("hive.llap.daemon.output.stream.timeout", "120s", - new TimeValidator(TimeUnit.SECONDS), - "The timeout for the client to connect to LLAP output service and start the fragment\n" + - "output after sending the fragment. The fragment will fail if its output is not claimed."), - LLAP_DAEMON_OUTPUT_SERVICE_SEND_BUFFER_SIZE("hive.llap.daemon.output.service.send.buffer.size", - 128 * 1024, "Send buffer size to be used by LLAP daemon output service"), - LLAP_DAEMON_OUTPUT_SERVICE_MAX_PENDING_WRITES("hive.llap.daemon.output.service.max.pending.writes", - 8, "Maximum number of queued writes allowed per connection when sending data\n" + - " via the LLAP output service to external clients."), - LLAP_EXTERNAL_SPLITS_TEMP_TABLE_STORAGE_FORMAT("hive.llap.external.splits.temp.table.storage.format", - "orc", new StringSet("default", "text", "orc"), - "Storage format for temp tables created using LLAP external client"), - LLAP_EXTERNAL_CLIENT_USE_HYBRID_CALENDAR("hive.llap.external.client.use.hybrid.calendar", - false, - "Whether to use hybrid calendar for parsing of data/timestamps."), - - // ====== confs for llap-external-client cloud deployment ====== - LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED( - "hive.llap.external.client.cloud.deployment.setup.enabled", false, - "Tells whether to enable additional RPC port, auth mechanism for llap external clients. This is meant" - + "for cloud based deployments. When true, it has following effects - \n" - + "1. Enables an extra RPC port on LLAP daemon to accept fragments from external clients. See" - + "hive.llap.external.client.cloud.rpc.port\n" - + "2. Uses external hostnames of LLAP in splits, so that clients can submit from outside of cloud. " - + "Env variable PUBLIC_HOSTNAME should be available on LLAP machines.\n" - + "3. Uses JWT based authentication for splits to be validated at LLAP. See " - + "hive.llap.external.client.cloud.jwt.shared.secret.provider"), - LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT("hive.llap.external.client.cloud.rpc.port", 30004, - "The LLAP daemon RPC port for external clients when llap is running in cloud environment."), - LLAP_EXTERNAL_CLIENT_CLOUD_OUTPUT_SERVICE_PORT("hive.llap.external.client.cloud.output.service.port", 30005, - "LLAP output service port when llap is running in cloud environment"), - LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_PROVIDER( - "hive.llap.external.client.cloud.jwt.shared.secret.provider", - "org.apache.hadoop.hive.llap.security.DefaultJwtSharedSecretProvider", - "Shared secret provider to be used to sign JWT"), - LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET("hive.llap.external.client.cloud.jwt.shared.secret", - "", - "The LLAP daemon RPC port for external clients when llap is running in cloud environment. " - + "Length of the secret should be >= 32 bytes"), - // ====== confs for llap-external-client cloud deployment ====== LLAP_ENABLE_GRACE_JOIN_IN_LLAP("hive.llap.enable.grace.join.in.llap", false, "Override if grace join should be allowed to run in llap."), diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml index 33f5a9ed43d8..f4ef89aa3d44 100644 --- a/itests/hive-unit/pom.xml +++ b/itests/hive-unit/pom.xml @@ -62,10 +62,6 @@ org.apache.hive hive-llap-server - - org.apache.hive - hive-llap-ext-client - org.apache.hive hive-llap-server diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java deleted file mode 100644 index 9ee65a1c314c..000000000000 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap.ext; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.util.ArrayList; - -import org.apache.hadoop.hive.llap.LlapInputSplit; -import org.apache.hadoop.hive.llap.Schema; -import org.apache.hadoop.hive.llap.FieldDesc; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; - -import org.apache.hadoop.mapred.SplitLocationInfo; -import org.junit.Test; -import static org.junit.Assert.*; - -public class TestLlapInputSplit { - - @Test - public void testWritable() throws Exception { - int splitNum = 88; - byte[] planBytes = "0123456789987654321".getBytes(); - byte[] fragmentBytes = "abcdefghijklmnopqrstuvwxyz".getBytes(); - SplitLocationInfo[] locations = { - new SplitLocationInfo("location1", false), - new SplitLocationInfo("location2", false), - }; - - LlapDaemonInfo daemonInfo1 = new LlapDaemonInfo("host1", 30004, 15003); - LlapDaemonInfo daemonInfo2 = new LlapDaemonInfo("host2", 30004, 15003); - - LlapDaemonInfo[] llapDaemonInfos = {daemonInfo1, daemonInfo2}; - - ArrayList colDescs = new ArrayList(); - colDescs.add(new FieldDesc("col1", TypeInfoFactory.stringTypeInfo)); - colDescs.add(new FieldDesc("col2", TypeInfoFactory.intTypeInfo)); - Schema schema = new Schema(colDescs); - - byte[] tokenBytes = new byte[] { 1 }; - LlapInputSplit split1 = new LlapInputSplit(splitNum, planBytes, fragmentBytes, null, - locations, llapDaemonInfos, schema, "hive", tokenBytes, "some-dummy-jwt"); - ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(); - DataOutputStream dataOut = new DataOutputStream(byteOutStream); - split1.write(dataOut); - ByteArrayInputStream byteInStream = new ByteArrayInputStream(byteOutStream.toByteArray()); - DataInputStream dataIn = new DataInputStream(byteInStream); - LlapInputSplit split2 = new LlapInputSplit(); - split2.readFields(dataIn); - - // Did we read all the data? - assertEquals(0, byteInStream.available()); - - checkLlapSplits(split1, split2); - } - - static void checkLlapSplits(LlapInputSplit split1, LlapInputSplit split2) throws Exception { - - assertEquals(split1.getSplitNum(), split2.getSplitNum()); - assertArrayEquals(split1.getPlanBytes(), split2.getPlanBytes()); - assertArrayEquals(split1.getFragmentBytes(), split2.getFragmentBytes()); - assertArrayEquals(split1.getTokenBytes(), split2.getTokenBytes()); - SplitLocationInfo[] locationInfo1 = split1.getLocationInfo(); - SplitLocationInfo[] locationInfo2 = split2.getLocationInfo(); - for (int idx = 0; idx < locationInfo1.length; ++idx) { - assertEquals(locationInfo1[idx].getLocation(), locationInfo2[idx].getLocation()); - assertEquals(locationInfo1[idx].isInMemory(), locationInfo2[idx].isInMemory()); - assertEquals(locationInfo1[idx].isOnDisk(), locationInfo2[idx].isOnDisk()); - } - assertArrayEquals(split1.getLocations(), split2.getLocations()); - assertEquals(split1.getSchema().toString(), split2.getSchema().toString()); - assertEquals(split1.getLlapUser(), split2.getLlapUser()); - assertEquals(split1.getJwt(), split2.getJwt()); - assertArrayEquals(split1.getLlapDaemonInfos(), split2.getLlapDaemonInfos()); - } - -} diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java index 812e4a3706e1..cab79288e310 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java @@ -27,9 +27,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; @@ -43,13 +41,8 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.conf.HiveConfForTest; -import org.apache.hadoop.hive.metastore.api.LockState; -import org.apache.hadoop.hive.metastore.api.LockType; import org.apache.hadoop.hive.metastore.api.ShowCompactRequest; import org.apache.hadoop.hive.metastore.api.ShowCompactResponse; -import org.apache.hadoop.hive.metastore.api.ShowLocksRequest; -import org.apache.hadoop.hive.metastore.api.ShowLocksResponse; -import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil; import org.apache.hadoop.hive.metastore.txn.TxnStore; @@ -747,106 +740,6 @@ public void testBucketedAcidInsertWithRemoveUnion() throws Exception { } } - @Test - public void testGetSplitsLocks() throws Exception { - // Need to test this with LLAP settings, which requires some additional configurations set. - hiveConf.setVar(ConfVars.HIVE_FETCH_TASK_CONVERSION, "more"); - hiveConf.setVar(HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS, "localhost"); - - // SessionState/Driver needs to be restarted with the Tez conf settings. - restartSessionAndDriver(hiveConf); - TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); - - try { - // Request LLAP splits for a table. - String queryParam = "select * from " + Table.ACIDTBL; - runStatementOnDriver("select get_splits(\"" + queryParam + "\", 1)"); - - // The get_splits call should have resulted in a lock on ACIDTBL - ShowLocksResponse slr = txnHandler.showLocks(new ShowLocksRequest()); - TestTxnDbUtil.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, - "default", Table.ACIDTBL.name, null, slr.getLocks()); - assertEquals(1, slr.getLocksSize()); - - // Try another table. - queryParam = "select * from " + Table.ACIDTBLPART; - runStatementOnDriver("select get_splits(\"" + queryParam + "\", 1)"); - - // Should now have new lock on ACIDTBLPART - slr = txnHandler.showLocks(new ShowLocksRequest()); - TestTxnDbUtil.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, - "default", Table.ACIDTBLPART.name, null, slr.getLocks()); - assertEquals(2, slr.getLocksSize()); - - // There should be different txn IDs associated with each lock. - Set txnSet = new HashSet(); - for (ShowLocksResponseElement lockResponseElem : slr.getLocks()) { - txnSet.add(lockResponseElem.getTxnid()); - } - assertEquals(2, txnSet.size()); - - List rows = runStatementOnDriver("show transactions"); - // Header row + 2 transactions = 3 rows - assertEquals(3, rows.size()); - } finally { - // Close the session which should free up the TxnHandler/locks held by the session. - // Done in the finally block to make sure we free up the locks; otherwise - // the cleanup in tearDown() will get stuck waiting on the lock held here on ACIDTBL. - restartSessionAndDriver(hiveConf); - } - - // Lock should be freed up now. - ShowLocksResponse slr = txnHandler.showLocks(new ShowLocksRequest()); - assertEquals(0, slr.getLocksSize()); - - List rows = runStatementOnDriver("show transactions"); - // Transactions should be committed. - // No transactions - just the header row - assertEquals(1, rows.size()); - } - - @Test - public void testGetSplitsLocksWithMaterializedView() throws Exception { - // Need to test this with LLAP settings, which requires some additional configurations set. - hiveConf.setVar(ConfVars.HIVE_FETCH_TASK_CONVERSION, "more"); - hiveConf.setVar(HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS, "localhost"); - - // SessionState/Driver needs to be restarted with the Tez conf settings. - restartSessionAndDriver(hiveConf); - TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); - String mvName = "mv_acidTbl"; - try { - runStatementOnDriver("create materialized view " + mvName + " as select a from " + Table.ACIDTBL + " where a > 5"); - - // Request LLAP splits for a table. - String queryParam = "select a from " + Table.ACIDTBL + " where a > 5"; - runStatementOnDriver("select get_splits(\"" + queryParam + "\", 1)"); - - // The get_splits call should have resulted in a lock on ACIDTBL and materialized view mv_acidTbl - ShowLocksResponse slr = txnHandler.showLocks(new ShowLocksRequest()); - TestTxnDbUtil.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, - "default", Table.ACIDTBL.name, null, slr.getLocks()); - TestTxnDbUtil.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, - "default", mvName, null, slr.getLocks()); - assertEquals(2, slr.getLocksSize()); - } finally { - // Close the session which should free up the TxnHandler/locks held by the session. - // Done in the finally block to make sure we free up the locks; otherwise - // the cleanup in tearDown() will get stuck waiting on the lock held here on ACIDTBL. - restartSessionAndDriver(hiveConf); - runStatementOnDriver("drop materialized view if exists " + mvName); - } - - // Lock should be freed up now. - ShowLocksResponse slr = txnHandler.showLocks(new ShowLocksRequest()); - assertEquals(0, slr.getLocksSize()); - - List rows = runStatementOnDriver("show transactions"); - // Transactions should be committed. - // No transactions - just the header row - assertEquals(1, rows.size()); - } - /** * HIVE-20699 * diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/AbstractJdbcTriggersTest.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/AbstractJdbcTriggersTest.java index 3a993adaf57b..acd01f12f643 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/AbstractJdbcTriggersTest.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/AbstractJdbcTriggersTest.java @@ -41,7 +41,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.llap.LlapBaseInputFormat; import org.apache.hadoop.hive.metastore.api.WMTrigger; import org.apache.hadoop.hive.ql.wm.Trigger; import org.apache.hive.jdbc.miniHS2.MiniHS2; @@ -99,7 +98,6 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { - LlapBaseInputFormat.closeAll(); hs2Conn.close(); } diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/AbstractTestJdbcGenericUDTFGetSplits.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/AbstractTestJdbcGenericUDTFGetSplits.java deleted file mode 100644 index aceed9c02870..000000000000 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/AbstractTestJdbcGenericUDTFGetSplits.java +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.jdbc; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.llap.LlapBaseInputFormat; -import org.apache.hive.jdbc.miniHS2.MiniHS2; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.PrintStream; -import java.net.URL; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -/** - * AbstractTestJdbcGenericUDTFGetSplits. - */ -public abstract class AbstractTestJdbcGenericUDTFGetSplits { - protected static MiniHS2 miniHS2 = null; - protected static String dataFileDir; - protected static String tableName = "testtab1"; - protected static String partitionedTableName = "partitionedtesttab1"; - protected static HiveConf conf = null; - static Path kvDataFilePath; - protected Connection hs2Conn = null; - - @BeforeClass - public static void beforeTest() throws Exception { - String confDir = "../../data/conf/llap/"; - HiveConf.setHiveSiteLocation(new URL("file://" + new File(confDir).toURI().getPath() + "/hive-site.xml")); - System.out.println("Setting hive-site: " + HiveConf.getHiveSiteLocation()); - - conf = new HiveConf(); - conf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, false); - conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); - conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); - conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_DEFAULT_QUEUES, "default"); - conf.setTimeVar(HiveConf.ConfVars.HIVE_TRIGGER_VALIDATION_INTERVAL, 100, TimeUnit.MILLISECONDS); - conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS, true); - conf.setBoolVar(HiveConf.ConfVars.TEZ_EXEC_SUMMARY, true); - conf.setBoolVar(HiveConf.ConfVars.HIVE_STRICT_CHECKS_CARTESIAN, false); - conf.setVar(HiveConf.ConfVars.LLAP_IO_MEMORY_MODE, "none"); - conf.setVar(HiveConf.ConfVars.LLAP_EXTERNAL_SPLITS_TEMP_TABLE_STORAGE_FORMAT, "text"); - - conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + "/tez-site.xml")); - - miniHS2 = new MiniHS2(conf, MiniHS2.MiniClusterType.LLAP); - dataFileDir = conf.get("test.data.files").replace('\\', '/').replace("c:", ""); - kvDataFilePath = new Path(dataFileDir, "kv1.txt"); - - Map confOverlay = new HashMap<>(); - miniHS2.start(confOverlay); - } - - @AfterClass - public static void afterTest() throws Exception { - if (miniHS2.isStarted()) { - miniHS2.stop(); - } - } - - @Before - public void setUp() throws Exception { - hs2Conn = BaseJdbcWithMiniLlap.getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar"); - } - - @After - public void tearDown() throws Exception { - LlapBaseInputFormat.closeAll(); - hs2Conn.close(); - } - - protected void runQuery(final String query, final List setCmds, final int numRows) throws Exception { - - Connection con = hs2Conn; - BaseJdbcWithMiniLlap.createTestTable(con, null, tableName, kvDataFilePath.toString()); - - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setErr(new PrintStream(baos)); // capture stderr - final Statement selStmt = con.createStatement(); - Throwable throwable = null; - int rowCount = 0; - try { - try { - if (setCmds != null) { - for (String setCmd : setCmds) { - selStmt.execute(setCmd); - } - } - ResultSet resultSet = selStmt.executeQuery(query); - while (resultSet.next()) { - rowCount++; - } - } catch (SQLException e) { - throwable = e; - } - selStmt.close(); - assertNull(throwable); - System.out.println("Expected " + numRows + " rows for query '" + query + "'. Got: " + rowCount); - assertEquals("Expected rows: " + numRows + " got: " + rowCount, numRows, rowCount); - } finally { - baos.close(); - } - - } - - protected List getConfigs(String... more) { - List setCmds = new ArrayList<>(); - setCmds.add("set mapred.min.split.size=10"); - setCmds.add("set mapred.max.split.size=10"); - setCmds.add("set tez.grouping.min-size=10"); - setCmds.add("set tez.grouping.max-size=10"); - // to get at least 10 splits - setCmds.add("set tez.grouping.split-waves=10"); - if (more != null) { - setCmds.addAll(Arrays.asList(more)); - } - return setCmds; - } - - protected void testGenericUDTFOrderBySplitCount1(String udtfName, int[] expectedCounts) throws Exception { - String query = "select " + udtfName + "(" + "'select value from " + tableName + "', 10)"; - runQuery(query, getConfigs(), expectedCounts[0]); - - // Check number of splits is respected - query = "select get_splits(" + "'select value from " + tableName + "', 3)"; - runQuery(query, getConfigs(), 3); - - query = "select " + udtfName + "(" + "'select value from " + tableName + " order by under_col', 5)"; - runQuery(query, getConfigs(), expectedCounts[1]); - - query = "select " + udtfName + "(" + "'select value from " + tableName + " order by under_col limit 0', 5)"; - runQuery(query, getConfigs(), expectedCounts[2]); - - query = "select " + udtfName + "(" + "'select value from " + tableName + " limit 2', 5)"; - runQuery(query, getConfigs(), expectedCounts[3]); - - query = "select " + udtfName + "(" + "'select value from " + tableName + " group by value limit 2', 5)"; - runQuery(query, getConfigs(), expectedCounts[4]); - - query = "select " + udtfName + "(" + "'select value from " + tableName + " where value is not null limit 2', 5)"; - runQuery(query, getConfigs(), expectedCounts[5]); - - query = "select " + udtfName + "(" + "'select `value` from (select value from " + tableName + - " where value is not null order by value) as t', 5)"; - runQuery(query, getConfigs(), expectedCounts[6]); - } - - protected void testGenericUDTFOrderBySplitCount1OnPartitionedTable(String udtfName, int[] expectedCounts) - throws Exception { - createPartitionedTestTable(null, partitionedTableName); - - String query = "select " + udtfName + "(" + "'select id from " + partitionedTableName + "', 5)"; - runQuery(query, getConfigs(), expectedCounts[0]); - - query = "select " + udtfName + "(" + "'select id from " + partitionedTableName + " order by id', 5)"; - runQuery(query, getConfigs(), expectedCounts[1]); - - query = "select " + udtfName + "(" + "'select id from " + partitionedTableName + " limit 2', 5)"; - runQuery(query, getConfigs(), expectedCounts[2]); - - query = "select " + udtfName + "(" + "'select id from " + partitionedTableName + " where id != 0 limit 2', 5)"; - runQuery(query, getConfigs(), expectedCounts[3]); - - query = "select " + udtfName + "(" + "'select id from " + partitionedTableName + " group by id limit 2', 5)"; - runQuery(query, getConfigs(), expectedCounts[4]); - - } - - private void createPartitionedTestTable(String database, String tableName) throws Exception { - Statement stmt = hs2Conn.createStatement(); - - if (database != null) { - stmt.execute("CREATE DATABASE IF NOT EXISTS " + database); - stmt.execute("USE " + database); - } - - // create table - stmt.execute("DROP TABLE IF EXISTS " + tableName); - stmt.execute("CREATE TABLE " + tableName - + " (id INT) partitioned by (p1 int)"); - - // load data - for (int i=1; i<=5; i++) { - String values = ""; - for (int j=1; j<=10; j++) { - if (j != 10) { - values+= "(" + j +"),"; - } else { - values+= "(" + j +")"; - } - } - stmt.execute("insert into " + tableName + " partition (p1=" + i +") " + " values " + values); - } - - - ResultSet res = stmt.executeQuery("SELECT count(*) FROM " + tableName); - assertTrue(res.next()); - assertEquals(50, res.getInt(1)); - res.close(); - stmt.close(); - } -} diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java index a56785199955..70e79517f240 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java @@ -20,121 +20,21 @@ package org.apache.hive.jdbc; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import java.io.File; -import java.math.BigDecimal; -import java.net.URL; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.ql.ddl.process.kill.KillQueriesOperation; -import org.apache.hadoop.mapred.InputSplit; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.RecordReader; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.llap.FieldDesc; -import org.apache.hadoop.hive.llap.Row; -import org.apache.hadoop.hive.llap.Schema; -import org.apache.hadoop.io.NullWritable; - -import org.apache.hive.jdbc.miniHS2.MiniHS2; -import org.apache.hive.jdbc.miniHS2.MiniHS2.MiniClusterType; -import org.apache.hadoop.hive.common.type.Date; -import org.apache.hadoop.hive.common.type.Timestamp; -import org.apache.hadoop.hive.llap.LlapBaseInputFormat; - -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.Test; -import org.apache.hadoop.mapred.InputFormat; - -/** - * Specialize this base class for different serde's/formats - * {@link #beforeTest(boolean) beforeTest} should be called - * by sub-classes in a {@link org.junit.BeforeClass} initializer - */ public abstract class BaseJdbcWithMiniLlap { - - private static String dataFileDir; - private static Path kvDataFilePath; - private static Path dataTypesFilePath; - private static Path over10KFilePath; - - protected static MiniHS2 miniHS2 = null; - protected static HiveConf conf = null; - protected static Connection hs2Conn = null; - - // This method should be called by sub-classes in a @BeforeClass initializer - public static MiniHS2 beforeTest(HiveConf inputConf) throws Exception { - conf = inputConf; - miniHS2 = new MiniHS2(conf, MiniClusterType.LLAP); - dataFileDir = conf.get("test.data.files").replace('\\', '/').replace("c:", ""); - kvDataFilePath = new Path(dataFileDir, "kv1.txt"); - dataTypesFilePath = new Path(dataFileDir, "datatypes.txt"); - over10KFilePath = new Path(dataFileDir, "over10k"); - Map confOverlay = new HashMap(); - miniHS2.start(confOverlay); - return miniHS2; - } - - static HiveConf defaultConf() throws Exception { - String confDir = "../../data/conf/llap/"; - if (confDir != null && !confDir.isEmpty()) { - HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml")); - System.out.println("Setting hive-site: " + HiveConf.getHiveSiteLocation()); - } - HiveConf defaultConf = new HiveConf(); - defaultConf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); - defaultConf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); - defaultConf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + "/tez-site.xml")); - return defaultConf; - } - - @Before - public void setUp() throws Exception { - hs2Conn = getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar"); - } - public static Connection getConnection(String jdbcURL, String user, String pwd) throws SQLException { Connection conn = DriverManager.getConnection(jdbcURL, user, pwd); conn.createStatement().execute("set hive.support.concurrency = false"); return conn; } - @After - public void tearDown() throws Exception { - LlapBaseInputFormat.closeAll(); - hs2Conn.close(); - } - - @AfterClass - public static void afterTest() throws Exception { - if (miniHS2.isStarted()) { - miniHS2.stop(); - } - } - - protected void createTestTable(String tableName) throws Exception { - createTestTable(hs2Conn, null, tableName, kvDataFilePath.toString()); - } - public static void createTestTable(Connection connection, String database, String tableName, String srcFile) throws Exception { Statement stmt = connection.createStatement(); @@ -158,643 +58,5 @@ public static void createTestTable(Connection connection, String database, Strin res.close(); stmt.close(); } - - protected void createDataTypesTable(String tableName) throws Exception { - Statement stmt = hs2Conn.createStatement(); - - // create table - stmt.execute("DROP TABLE IF EXISTS " + tableName); - // tables with various types - stmt.execute("create table " + tableName - + " (c1 int, c2 boolean, c3 double, c4 string," - + " c5 array, c6 map, c7 map," - + " c8 struct," - + " c9 tinyint, c10 smallint, c11 float, c12 bigint," - + " c13 array>," - + " c14 map>," - + " c15 struct>," - + " c16 array,n:int>>," - + " c17 timestamp, " - + " c18 decimal(16,7), " - + " c19 binary, " - + " c20 date," - + " c21 varchar(20)," - + " c22 char(15)," - + " c23 binary" - + ")"); - stmt.execute("load data local inpath '" - + dataTypesFilePath.toString() + "' into table " + tableName); - stmt.close(); - } - - protected void createOver10KTable(String tableName) throws Exception { - try (Statement stmt = hs2Conn.createStatement()) { - - String createQuery = - "create table " + tableName + " (t tinyint, si smallint, i int, b bigint, f float, d double, bo boolean, " - + "s string, ts timestamp, `dec` decimal(4,2), bin binary) row format delimited fields terminated by '|'"; - - // create table - stmt.execute("DROP TABLE IF EXISTS " + tableName); - stmt.execute(createQuery); - // load data - stmt.execute("load data local inpath '" + over10KFilePath.toString() + "' into table " + tableName); - } - } - - @Test(timeout = 120000) - public void testLlapInputFormatEndToEnd() throws Exception { - createTestTable("testtab1"); - - int rowCount; - - RowCollector rowCollector = new RowCollector(); - String query = "select * from testtab1 where under_col = 0"; - rowCount = processQuery(query, 1, rowCollector); - assertEquals(3, rowCount); - assertArrayEquals(new String[] {"0", "val_0"}, rowCollector.rows.get(0)); - assertArrayEquals(new String[] {"0", "val_0"}, rowCollector.rows.get(1)); - assertArrayEquals(new String[] {"0", "val_0"}, rowCollector.rows.get(2)); - - // Try empty rows query - rowCollector.rows.clear(); - query = "select * from testtab1 where true = false"; - rowCount = processQuery(query, 1, rowCollector); - assertEquals(0, rowCount); - } - - @Test(timeout = 300000) - public void testMultipleBatchesOfComplexTypes() throws Exception { - final String tableName = "testMultipleBatchesOfComplexTypes"; - try (Statement stmt = hs2Conn.createStatement()) { - String createQuery = - "create table " + tableName + "(c1 array>, " - + "c2 int, " - + "c3 array>, " - + "c4 array>>) STORED AS ORC"; - - // create table - stmt.execute("DROP TABLE IF EXISTS " + tableName); - stmt.execute(createQuery); - // load data - stmt.execute("INSERT INTO " + tableName + " VALUES " - // value 1 - + "(ARRAY(NAMED_STRUCT('f1','a1', 'f2','a2'), NAMED_STRUCT('f1','a3', 'f2','a4')), " - + "1, ARRAY(ARRAY(1)), ARRAY(NAMED_STRUCT('f1',ARRAY('aa1')))), " - // value 2 - + "(ARRAY(NAMED_STRUCT('f1','b1', 'f2','b2'), NAMED_STRUCT('f1','b3', 'f2','b4')), 2, " - + "ARRAY(ARRAY(2,2), ARRAY(2,2)), " - + "ARRAY(NAMED_STRUCT('f1',ARRAY('aa2','aa2')), NAMED_STRUCT('f1',ARRAY('aa2','aa2')))), " - // value 3 - + "(ARRAY(NAMED_STRUCT('f1','c1', 'f2','c2'), NAMED_STRUCT('f1','c3', 'f2','c4'), " - + "NAMED_STRUCT('f1','c5', 'f2','c6')), 3, " + "ARRAY(ARRAY(3,3,3), ARRAY(3,3,3), ARRAY(3,3,3)), " - + "ARRAY(NAMED_STRUCT('f1',ARRAY('aa3','aa3','aa3')), " - + "NAMED_STRUCT('f1',ARRAY('aa3','aa3', 'aa3')), NAMED_STRUCT('f1',ARRAY('aa3','aa3', 'aa3')))), " - // value 4 - + "(ARRAY(NAMED_STRUCT('f1','d1', 'f2','d2'), NAMED_STRUCT('f1','d3', 'f2','d4')," - + " NAMED_STRUCT('f1','d5', 'f2','d6'), NAMED_STRUCT('f1','d7', 'f2','d8')), 4, " - + "ARRAY(ARRAY(4,4,4,4),ARRAY(4,4,4,4),ARRAY(4,4,4,4),ARRAY(4,4,4,4)), " - + "ARRAY(NAMED_STRUCT('f1',ARRAY('aa4','aa4','aa4', 'aa4')), " - + "NAMED_STRUCT('f1',ARRAY('aa4','aa4','aa4', 'aa4')), NAMED_STRUCT('f1',ARRAY('aa4','aa4','aa4', 'aa4'))," - + " NAMED_STRUCT('f1',ARRAY('aa4','aa4','aa4', 'aa4'))))"); - - // generate 4096 rows from above records - for (int i = 0; i < 10; i++) { - stmt.execute(String.format("insert into %s select * from %s", tableName, tableName)); - } - // validate test table - ResultSet res = stmt.executeQuery("SELECT count(*) FROM " + tableName); - assertTrue(res.next()); - assertEquals(4096, res.getInt(1)); - res.close(); - } - - RowCollector rowCollector = new RowCollector(); - String query = "select * from " + tableName; - int rowCount = processQuery(query, 1, rowCollector); - assertEquals(4096, rowCount); - - /* - * - * validate different rows - * [[[a1, a2], [a3, a4]], 1, [[1]], [[[aa1]]]] - * [[[b1, b2], [b3, b4]], 2, [[2, 2], [2, 2]], [[[aa2, aa2]], [[aa2, aa2]]]] - * [[[c1, c2], [c3, c4], [c5, c6]], 3, [[3, 3, 3], [3, 3, 3], [3, 3, 3]], [[[aa3, aa3, aa3]], [[aa3, aa3, aa3]], [[aa3, aa3, aa3]]]] - * [[[d1, d2], [d3, d4], [d5, d6], [d7, d8]], 4, [[4, 4, 4, 4], [4, 4, 4, 4], [4, 4, 4, 4], [4, 4, 4, 4]], [[[aa4, aa4, aa4, aa4]], [[aa4, aa4, aa4, aa4]], [[aa4, aa4, aa4, aa4]], [[aa4, aa4, aa4, aa4]]]] - * - */ - rowCollector.rows.clear(); - query = "select * from " + tableName + " where c2=1 limit 1"; - rowCount = processQuery(query, 1, rowCollector); - assertEquals(1, rowCount); - final String[] expected1 = - { "[[a1, a2], [a3, a4]]", - "1", - "[[1]]", - "[[[aa1]]]" - }; - assertArrayEquals(expected1, rowCollector.rows.get(0)); - - rowCollector.rows.clear(); - query = "select * from " + tableName + " where c2=2 limit 1"; - rowCount = processQuery(query, 1, rowCollector); - assertEquals(1, rowCount); - final String[] expected2 = - { "[[b1, b2], [b3, b4]]", - "2", - "[[2, 2], [2, 2]]", - "[[[aa2, aa2]], [[aa2, aa2]]]" - }; - assertArrayEquals(expected2, rowCollector.rows.get(0)); - - rowCollector.rows.clear(); - query = "select * from " + tableName + " where c2=3 limit 1"; - rowCount = processQuery(query, 1, rowCollector); - assertEquals(1, rowCount); - final String[] expected3 = - { "[[c1, c2], [c3, c4], [c5, c6]]", - "3", - "[[3, 3, 3], [3, 3, 3], [3, 3, 3]]", - "[[[aa3, aa3, aa3]], [[aa3, aa3, aa3]], [[aa3, aa3, aa3]]]" - }; - assertArrayEquals(expected3, rowCollector.rows.get(0)); - - rowCollector.rows.clear(); - query = "select * from " + tableName + " where c2=4 limit 1"; - rowCount = processQuery(query, 1, rowCollector); - assertEquals(1, rowCount); - final String[] expected4 = - { "[[d1, d2], [d3, d4], [d5, d6], [d7, d8]]", - "4", - "[[4, 4, 4, 4], [4, 4, 4, 4], [4, 4, 4, 4], [4, 4, 4, 4]]", - "[[[aa4, aa4, aa4, aa4]], [[aa4, aa4, aa4, aa4]], [[aa4, aa4, aa4, aa4]], [[aa4, aa4, aa4, aa4]]]" - }; - assertArrayEquals(expected4, rowCollector.rows.get(0)); - - } - - @Test(timeout = 300000) - public void testLlapInputFormatEndToEndWithMultipleBatches() throws Exception { - String tableName = "over10k_table"; - - createOver10KTable(tableName); - - int rowCount; - - // Try with more than one batch - RowCollector rowCollector = new RowCollector(); - String query = "select * from " + tableName; - rowCount = processQuery(query, 1, rowCollector); - assertEquals(9999, rowCount); - - // Try with less than one batch - rowCollector.rows.clear(); - query = "select * from " + tableName + " where s = 'rachel brown'"; - rowCount = processQuery(query, 1, rowCollector); - assertEquals(17, rowCount); - - // Try empty rows query - rowCollector.rows.clear(); - query = "select * from " + tableName + " where false"; - rowCount = processQuery(query, 1, rowCollector); - assertEquals(0, rowCount); - } - - @Test(timeout = 300000) - public void testInvalidReferenceCountScenario() throws Exception { - final String tableName = "testInvalidReferenceCountScenario"; - try (Statement stmt = hs2Conn.createStatement()) { - String createQuery = - "create table " + tableName + - "(arr1 array>>>, " - + "c2 int) STORED AS ORC"; - - // create table - stmt.execute("DROP TABLE IF EXISTS " + tableName); - stmt.execute(createQuery); - // load data - stmt.execute("INSERT INTO " + tableName + " VALUES " - // value 1 - + "(ARRAY(NAMED_STRUCT('f1','a1', " - + "'f2','a2', " - + "'arr2'," - + " ARRAY(" - + "NAMED_STRUCT('f3', cast(null as string), 'f4', cast(null as string), 'f5', cast(null as string)))), " - + "NAMED_STRUCT('f1','a1', 'f2','a2', 'arr2', " - + "ARRAY(NAMED_STRUCT('f3', 'fielddddddd3333333', 'f4', 'field4', 'f5', 'field5'))), " - + "NAMED_STRUCT('f1','a1', 'f2','a2', 'arr2', ARRAY(NAMED_STRUCT('f3', cast(null as string), " - + "'f4', cast(null as string), 'f5', cast(null as string)))), " - + "NAMED_STRUCT('f1','a1', 'f2','a2', 'arr2', ARRAY(NAMED_STRUCT('f3', 'fielddddddd3333333', " - + "'f4', 'field4', 'f5', 'field5'))), NAMED_STRUCT('f1','a1', 'f2','a2', 'arr2', " - + "ARRAY(NAMED_STRUCT('f3', cast(null as string), 'f4', cast(null as string), 'f5', cast(null as string))))," - + " NAMED_STRUCT('f1','a1', 'f2','a2', 'arr2', " - + "ARRAY(NAMED_STRUCT('f3', 'fielddddddd3333333', 'f4', 'field4', 'f5', 'field5')))), 1)"); - - // generate 16384 rows from above records - for (int i = 0; i < 14; i++) { - stmt.execute(String.format("insert into %s select * from %s", tableName, tableName)); - } - // validate test table - ResultSet res = stmt.executeQuery("SELECT count(*) FROM " + tableName); - assertTrue(res.next()); - assertEquals(16384, res.getInt(1)); - res.close(); - } - // should not throw - IllegalReferenceCountException: refCnt: 0 - RowCollector rowCollector = new RowCollector(); - String query = "select * from " + tableName; - int rowCount = processQuery(query, 1, rowCollector); - assertEquals(16384, rowCount); - - } - - @Test(timeout = 60000) - public void testNonAsciiStrings() throws Exception { - createTestTable("testtab_nonascii"); - - RowCollector rowCollector = new RowCollector(); - String nonAscii = "À côté du garçon"; - String query = "select value, '" + nonAscii + "' from testtab_nonascii where under_col=0"; - int rowCount = processQuery(query, 1, rowCollector); - assertEquals(3, rowCount); - - assertArrayEquals(new String[] {"val_0", nonAscii}, rowCollector.rows.get(0)); - assertArrayEquals(new String[] {"val_0", nonAscii}, rowCollector.rows.get(1)); - assertArrayEquals(new String[] {"val_0", nonAscii}, rowCollector.rows.get(2)); - } - - @Test(timeout = 60000) - public void testEscapedStrings() throws Exception { - createTestTable("testtab1"); - - RowCollector rowCollector = new RowCollector(); - String expectedVal1 = "'a',\"b\",\\c\\"; - String expectedVal2 = "multi\nline"; - String query = "select value, '\\'a\\',\"b\",\\\\c\\\\', 'multi\\nline' from testtab1 where under_col=0"; - int rowCount = processQuery(query, 1, rowCollector); - assertEquals(3, rowCount); - - assertArrayEquals(new String[] {"val_0", expectedVal1, expectedVal2}, rowCollector.rows.get(0)); - assertArrayEquals(new String[] {"val_0", expectedVal1, expectedVal2}, rowCollector.rows.get(1)); - assertArrayEquals(new String[] {"val_0", expectedVal1, expectedVal2}, rowCollector.rows.get(2)); - } - - @Test(timeout = 60000) - public void testDataTypes() throws Exception { - createDataTypesTable("datatypes"); - RowCollector2 rowCollector = new RowCollector2(); - String query = "select * from datatypes"; - int rowCount = processQuery(query, 1, rowCollector); - assertEquals(3, rowCount); - - // Verify schema - String[][] colNameTypes = new String[][] { - {"datatypes.c1", "int"}, - {"datatypes.c2", "boolean"}, - {"datatypes.c3", "double"}, - {"datatypes.c4", "string"}, - {"datatypes.c5", "array"}, - {"datatypes.c6", "map"}, - {"datatypes.c7", "map"}, - {"datatypes.c8", "struct"}, - {"datatypes.c9", "tinyint"}, - {"datatypes.c10", "smallint"}, - {"datatypes.c11", "float"}, - {"datatypes.c12", "bigint"}, - {"datatypes.c13", "array>"}, - {"datatypes.c14", "map>"}, - {"datatypes.c15", "struct>"}, - {"datatypes.c16", "array,n:int>>"}, - {"datatypes.c17", "timestamp"}, - {"datatypes.c18", "decimal(16,7)"}, - {"datatypes.c19", "binary"}, - {"datatypes.c20", "date"}, - {"datatypes.c21", "varchar(20)"}, - {"datatypes.c22", "char(15)"}, - {"datatypes.c23", "binary"}, - }; - FieldDesc fieldDesc; - assertEquals(23, rowCollector.numColumns); - for (int idx = 0; idx < rowCollector.numColumns; ++idx) { - fieldDesc = rowCollector.schema.getColumns().get(idx); - assertEquals("ColName idx=" + idx, colNameTypes[idx][0], fieldDesc.getName()); - assertEquals("ColType idx=" + idx, colNameTypes[idx][1], fieldDesc.getTypeInfo().getTypeName()); - } - - // First row is all nulls - Object[] rowValues = rowCollector.rows.get(0); - for (int idx = 0; idx < rowCollector.numColumns; ++idx) { - assertEquals("idx=" + idx, null, rowValues[idx]); - } - - // Second Row - rowValues = rowCollector.rows.get(1); - assertEquals(Integer.valueOf(-1), rowValues[0]); - assertEquals(Boolean.FALSE, rowValues[1]); - assertEquals(Double.valueOf(-1.1d), rowValues[2]); - assertEquals("", rowValues[3]); - - List c5Value = (List) rowValues[4]; - assertEquals(0, c5Value.size()); - - Map c6Value = (Map) rowValues[5]; - assertEquals(1, c6Value.size()); - assertEquals(null, c6Value.get(1)); - - Map c7Value = (Map) rowValues[6]; - assertEquals(1, c7Value.size()); - assertEquals("b", c7Value.get("a")); - - List c8Value = (List) rowValues[7]; - assertEquals(null, c8Value.get(0)); - assertEquals(null, c8Value.get(1)); - assertEquals(null, c8Value.get(2)); - - assertEquals(Byte.valueOf((byte) -1), rowValues[8]); - assertEquals(Short.valueOf((short) -1), rowValues[9]); - assertEquals(Float.valueOf(-1.0f), rowValues[10]); - assertEquals(Long.valueOf(-1l), rowValues[11]); - - List c13Value = (List) rowValues[12]; - assertEquals(0, c13Value.size()); - - Map c14Value = (Map) rowValues[13]; - assertEquals(1, c14Value.size()); - Map mapVal = (Map) c14Value.get(Integer.valueOf(1)); - assertEquals(1, mapVal.size()); - assertEquals(100, mapVal.get(Integer.valueOf(10))); - - List c15Value = (List) rowValues[14]; - assertEquals(null, c15Value.get(0)); - assertEquals(null, c15Value.get(1)); - - List c16Value = (List) rowValues[15]; - assertEquals(0, c16Value.size()); - - assertEquals(null, rowValues[16]); - assertEquals(null, rowValues[17]); - assertEquals(null, rowValues[18]); - assertEquals(null, rowValues[19]); - assertEquals(null, rowValues[20]); - assertEquals(null, rowValues[21]); - assertEquals(null, rowValues[22]); - - // Third row - rowValues = rowCollector.rows.get(2); - assertEquals(Integer.valueOf(1), rowValues[0]); - assertEquals(Boolean.TRUE, rowValues[1]); - assertEquals(Double.valueOf(1.1d), rowValues[2]); - assertEquals("1", rowValues[3]); - - c5Value = (List) rowValues[4]; - assertEquals(2, c5Value.size()); - assertEquals(Integer.valueOf(1), c5Value.get(0)); - assertEquals(Integer.valueOf(2), c5Value.get(1)); - - c6Value = (Map) rowValues[5]; - assertEquals(2, c6Value.size()); - assertEquals("x", c6Value.get(Integer.valueOf(1))); - assertEquals("y", c6Value.get(Integer.valueOf(2))); - - c7Value = (Map) rowValues[6]; - assertEquals(2, c7Value.size()); - assertEquals("v", c7Value.get("k")); - assertEquals("c", c7Value.get("b")); - - c8Value = (List) rowValues[7]; - assertEquals("a", c8Value.get(0)); - assertEquals(Integer.valueOf(9), c8Value.get(1)); - assertEquals(Double.valueOf(2.2d), c8Value.get(2)); - - assertEquals(Byte.valueOf((byte) 1), rowValues[8]); - assertEquals(Short.valueOf((short) 1), rowValues[9]); - assertEquals(Float.valueOf(1.0f), rowValues[10]); - assertEquals(Long.valueOf(1l), rowValues[11]); - - c13Value = (List) rowValues[12]; - assertEquals(2, c13Value.size()); - List listVal = (List) c13Value.get(0); - assertEquals("a", listVal.get(0)); - assertEquals("b", listVal.get(1)); - listVal = (List) c13Value.get(1); - assertEquals("c", listVal.get(0)); - assertEquals("d", listVal.get(1)); - - c14Value = (Map) rowValues[13]; - assertEquals(2, c14Value.size()); - mapVal = (Map) c14Value.get(Integer.valueOf(1)); - assertEquals(2, mapVal.size()); - assertEquals(Integer.valueOf(12), mapVal.get(Integer.valueOf(11))); - assertEquals(Integer.valueOf(14), mapVal.get(Integer.valueOf(13))); - mapVal = (Map) c14Value.get(Integer.valueOf(2)); - assertEquals(1, mapVal.size()); - assertEquals(Integer.valueOf(22), mapVal.get(Integer.valueOf(21))); - - c15Value = (List) rowValues[14]; - assertEquals(Integer.valueOf(1), c15Value.get(0)); - listVal = (List) c15Value.get(1); - assertEquals(2, listVal.size()); - assertEquals(Integer.valueOf(2), listVal.get(0)); - assertEquals("x", listVal.get(1)); - - c16Value = (List) rowValues[15]; - assertEquals(2, c16Value.size()); - listVal = (List) c16Value.get(0); - assertEquals(2, listVal.size()); - mapVal = (Map) listVal.get(0); - assertEquals(0, mapVal.size()); - assertEquals(Integer.valueOf(1), listVal.get(1)); - listVal = (List) c16Value.get(1); - mapVal = (Map) listVal.get(0); - assertEquals(2, mapVal.size()); - assertEquals("b", mapVal.get("a")); - assertEquals("d", mapVal.get("c")); - assertEquals(Integer.valueOf(2), listVal.get(1)); - - assertEquals(Timestamp.valueOf("2012-04-22 09:00:00.123456789"), rowValues[16]); - assertEquals(new BigDecimal("123456789.123456"), rowValues[17]); - assertArrayEquals("abcd".getBytes("UTF-8"), (byte[]) rowValues[18]); - assertEquals(Date.valueOf("2013-01-01"), rowValues[19]); - assertEquals("abc123", rowValues[20]); - assertEquals("abc123 ", rowValues[21]); - assertArrayEquals("X'01FF'".getBytes("UTF-8"), (byte[]) rowValues[22]); - } - - - @Test(timeout = 120000) - public void testComplexQuery() throws Exception { - createTestTable("testtab1"); - - RowCollector rowCollector = new RowCollector(); - String query = "select value, count(*) from testtab1 where under_col=0 group by value"; - int rowCount = processQuery(query, 1, rowCollector); - assertEquals(1, rowCount); - - assertArrayEquals(new String[] {"val_0", "3"}, rowCollector.rows.get(0)); - } - - protected interface RowProcessor { - void process(Row row); - } - - protected static class RowCollector implements RowProcessor { - ArrayList rows = new ArrayList(); - Schema schema = null; - int numColumns = 0; - - public void process(Row row) { - if (schema == null) { - schema = row.getSchema(); - numColumns = schema.getColumns().size(); - } - - String[] arr = new String[numColumns]; - for (int idx = 0; idx < numColumns; ++idx) { - Object val = row.getValue(idx); - arr[idx] = (val == null ? null : val.toString()); - } - rows.add(arr); - } - } - - // Save the actual values from each row as opposed to the String representation. - protected static class RowCollector2 implements RowProcessor { - ArrayList rows = new ArrayList(); - Schema schema = null; - int numColumns = 0; - - public void process(Row row) { - if (schema == null) { - schema = row.getSchema(); - numColumns = schema.getColumns().size(); - } - - Object[] arr = new Object[numColumns]; - for (int idx = 0; idx < numColumns; ++idx) { - arr[idx] = row.getValue(idx); - } - rows.add(arr); - } - } - - protected int processQuery(String query, int numSplits, RowProcessor rowProcessor) throws Exception { - return processQuery(null, query, numSplits, rowProcessor); - } - - protected abstract InputFormat getInputFormat(); - - protected int processQuery(String currentDatabase, String query, int numSplits, RowProcessor rowProcessor) - throws Exception { - String url = miniHS2.getJdbcURL(); - String user = System.getProperty("user.name"); - String pwd = user; - String handleId = UUID.randomUUID().toString(); - - InputFormat inputFormat = getInputFormat(); - - // Get splits - JobConf job = new JobConf(conf); - job.set(LlapBaseInputFormat.URL_KEY, url); - job.set(LlapBaseInputFormat.USER_KEY, user); - job.set(LlapBaseInputFormat.PWD_KEY, pwd); - job.set(LlapBaseInputFormat.QUERY_KEY, query); - job.set(LlapBaseInputFormat.HANDLE_ID, handleId); - if (currentDatabase != null) { - job.set(LlapBaseInputFormat.DB_KEY, currentDatabase); - } - - InputSplit[] splits = inputFormat.getSplits(job, numSplits); - - // Fetch rows from splits - int rowCount = 0; - for (InputSplit split : splits) { - System.out.println("Processing split " + split.getLocations()); - - RecordReader reader = inputFormat.getRecordReader(split, job, null); - Row row = reader.createValue(); - while (reader.next(NullWritable.get(), row)) { - rowProcessor.process(row); - ++rowCount; - } - reader.close(); - } - LlapBaseInputFormat.close(handleId); - - return rowCount; - } - - /** - * Test CLI kill command of a query that is running. - * We spawn 2 threads - one running the query and - * the other attempting to cancel. - * We're using a dummy udf to simulate a query, - * that runs for a sufficiently long time. - * @throws Exception - */ - @Test - public void testKillQuery() throws Exception { - String tableName = "testtab1"; - createTestTable(tableName); - Connection con = hs2Conn; - Connection con2 = getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar"); - - String udfName = TestJdbcWithMiniHS2.SleepMsUDF.class.getName(); - Statement stmt1 = con.createStatement(); - Statement stmt2 = con2.createStatement(); - stmt1.execute("create temporary function sleepMsUDF as '" + udfName + "'"); - stmt1.close(); - final Statement stmt = con.createStatement(); - - ExceptionHolder tExecuteHolder = new ExceptionHolder(); - ExceptionHolder tKillHolder = new ExceptionHolder(); - - // Thread executing the query - Thread tExecute = new Thread(new Runnable() { - @Override - public void run() { - try { - System.out.println("Executing query: "); - // The test table has 500 rows, so total query time should be ~ 500*500ms - stmt.executeQuery("select sleepMsUDF(t1.under_col, 100), t1.under_col, t2.under_col " + - "from " + tableName + " t1 join " + tableName + " t2 on t1.under_col = t2.under_col"); - fail("Expecting SQLException"); - } catch (SQLException e) { - tExecuteHolder.throwable = e; - } - } - }); - // Thread killing the query - Thread tKill = new Thread(new Runnable() { - @Override - public void run() { - try { - Thread.sleep(2000); - String queryId = ((HiveStatement) stmt).getQueryId(); - System.out.println("Killing query: " + queryId); - - stmt2.execute("kill query '" + queryId + "'"); - stmt2.close(); - } catch (Exception e) { - tKillHolder.throwable = e; - } - } - }); - - tExecute.start(); - tKill.start(); - tExecute.join(); - tKill.join(); - stmt.close(); - con2.close(); - - assertNotNull("tExecute", tExecuteHolder.throwable); - assertEquals(HiveStatement.QUERY_CANCELLED_MESSAGE + " "+ KillQueriesOperation.KILL_QUERY_MESSAGE, - tExecuteHolder.throwable.getMessage()); - assertNull("tCancel", tKillHolder.throwable); - } - - private static class ExceptionHolder { - Throwable throwable; - } } diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcGenericUDTFGetSplits.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcGenericUDTFGetSplits.java deleted file mode 100644 index 33a03642a48d..000000000000 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcGenericUDTFGetSplits.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.jdbc; - -import org.apache.hadoop.hive.llap.FieldDesc; -import org.apache.hadoop.hive.llap.LlapBaseInputFormat; -import org.apache.hadoop.hive.llap.LlapInputSplit; -import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; -import org.apache.hadoop.mapred.JobConf; -import org.junit.Ignore; -import org.junit.Test; - -import java.sql.ResultSet; -import java.sql.Statement; -import java.util.UUID; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -/** - * TestJdbcGenericUDTFGetSplits. - */ -public class TestJdbcGenericUDTFGetSplits extends AbstractTestJdbcGenericUDTFGetSplits { - - @Test(timeout = 200000) - public void testGetSplitsOrderBySplitCount1() throws Exception { - testGenericUDTFOrderBySplitCount1("get_splits", new int[] { 10, 5, 0, 2, 2, 2, 5 }); - } - - @Test(timeout = 200000) - public void testGetLlapSplitsOrderBySplitCount1() throws Exception { - testGenericUDTFOrderBySplitCount1("get_llap_splits", new int[] { 12, 7, 1, 4, 4, 4, 7 }); - } - - @Test(timeout = 200000) - public void testGetSplitsOrderBySplitCount1OnPartitionedTable() throws Exception { - testGenericUDTFOrderBySplitCount1OnPartitionedTable("get_splits", new int[]{5, 5, 1, 1, 1}); - } - - @Test(timeout = 200000) - public void testGetLlapSplitsOrderBySplitCount1OnPartitionedTable() throws Exception { - testGenericUDTFOrderBySplitCount1OnPartitionedTable("get_llap_splits", new int[]{7, 7, 3, 3, 3}); - } - - - - @Test - public void testDecimalPrecisionAndScale() throws Exception { - try (Statement stmt = hs2Conn.createStatement()) { - stmt.execute("CREATE TABLE decimal_test_table(decimal_col DECIMAL(6,2))"); - stmt.execute("INSERT INTO decimal_test_table VALUES(2507.92)"); - - ResultSet rs = stmt.executeQuery("SELECT * FROM decimal_test_table"); - assertTrue(rs.next()); - rs.close(); - - String url = miniHS2.getJdbcURL(); - String user = System.getProperty("user.name"); - String pwd = user; - String handleId = UUID.randomUUID().toString(); - String sql = "SELECT avg(decimal_col)/3 FROM decimal_test_table"; - - // make request through llap-ext-client - JobConf job = new JobConf(conf); - job.set(LlapBaseInputFormat.URL_KEY, url); - job.set(LlapBaseInputFormat.USER_KEY, user); - job.set(LlapBaseInputFormat.PWD_KEY, pwd); - job.set(LlapBaseInputFormat.QUERY_KEY, sql); - job.set(LlapBaseInputFormat.HANDLE_ID, handleId); - - LlapBaseInputFormat llapBaseInputFormat = new LlapBaseInputFormat(); - //schema split - LlapInputSplit schemaSplit = (LlapInputSplit) llapBaseInputFormat.getSplits(job, 0)[0]; - assertNotNull(schemaSplit); - FieldDesc fieldDesc = schemaSplit.getSchema().getColumns().get(0); - DecimalTypeInfo type = (DecimalTypeInfo) fieldDesc.getTypeInfo(); - assertEquals(12, type.getPrecision()); - assertEquals(8, type.scale()); - - LlapBaseInputFormat.close(handleId); - } - } - -} diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlapRow.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlapRow.java deleted file mode 100644 index 970b12ca4f0e..000000000000 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlapRow.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.jdbc; - -import org.apache.hadoop.hive.llap.Row; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.hive.llap.LlapRowInputFormat; -import org.junit.BeforeClass; -import org.junit.Before; -import org.junit.After; -import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.hive.conf.HiveConf; -import org.junit.Ignore; - -/** - * TestJdbcWithMiniLlap for llap Row format. - */ -@Ignore("HIVE-23549") -public class TestJdbcWithMiniLlapRow extends BaseJdbcWithMiniLlap { - - @BeforeClass - public static void beforeTest() throws Exception { - HiveConf conf = defaultConf(); - BaseJdbcWithMiniLlap.beforeTest(conf); - } - - @Override - protected InputFormat getInputFormat() { - return new LlapRowInputFormat(); - } - - @Override - @Ignore - public void testMultipleBatchesOfComplexTypes() { - // ToDo: FixMe - } - -} - diff --git a/itests/pom.xml b/itests/pom.xml index 6c64b0d730c3..4e011f4e0d50 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -266,11 +266,6 @@ ${project.version} tests - - org.apache.hive - hive-llap-ext-client - ${project.version} - org.apache.hive hive-kudu-handler diff --git a/llap-client/pom.xml b/llap-client/pom.xml index 89f8f51e34d4..d52509175347 100644 --- a/llap-client/pom.xml +++ b/llap-client/pom.xml @@ -219,21 +219,6 @@ - - org.apache.tez - tez-runtime-internals - true - - - org.slf4j - slf4j-log4j12 - - - commons-logging - commons-logging - - - ${basedir}/src/java diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapBaseRecordReader.java b/llap-client/src/java/org/apache/hadoop/hive/llap/LlapBaseRecordReader.java deleted file mode 100644 index 854b0971c4c8..000000000000 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapBaseRecordReader.java +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap; - -import com.google.common.base.Preconditions; - -import java.io.BufferedInputStream; -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.io.DataInputStream; -import java.util.concurrent.LinkedBlockingQueue; -import org.apache.hadoop.hive.llap.io.ChunkedInputStream; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapred.RecordReader; -import org.apache.hadoop.mapred.JobConf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Base LLAP RecordReader to handle receiving of the data from the LLAP daemon. - */ -public class LlapBaseRecordReader implements RecordReader { - private static final Logger LOG = LoggerFactory.getLogger(LlapBaseRecordReader.class); - - protected final ChunkedInputStream cin; - protected final DataInputStream din; - protected final Schema schema; - protected final Class clazz; - - protected Thread readerThread = null; - protected final LinkedBlockingQueue readerEvents = new LinkedBlockingQueue(); - protected final Closeable client; - private final Closeable socket; - private boolean closed = false; - - public LlapBaseRecordReader(InputStream in, Schema schema, - Class clazz, JobConf job, Closeable client, Closeable socket) { - String clientId = (client == null ? "" : client.toString()); - this.cin = new ChunkedInputStream(in, clientId); // Save so we can verify end of stream - // We need mark support - wrap with BufferedInputStream. - din = new DataInputStream(new BufferedInputStream(cin)); - this.schema = schema; - this.clazz = clazz; - this.readerThread = Thread.currentThread(); - this.client = client; - this.socket = socket; - } - - public Schema getSchema() { - return schema; - } - - @Override - public synchronized void close() throws IOException { - if (!closed) { - closed = true; - - Exception caughtException = null; - try { - din.close(); - } catch (Exception err) { - LOG.error("Error closing input stream:" + err.getMessage(), err); - caughtException = err; - } - // Don't close the socket - the stream already does that if needed. - - if (client != null) { - try { - client.close(); - } catch (Exception err) { - LOG.error("Error closing client:" + err.getMessage(), err); - caughtException = (caughtException == null ? err : caughtException); - } - } - - if (caughtException != null) { - throw new IOException("Exception during close: " + caughtException.getMessage(), caughtException); - } - } - } - - @Override - public long getPos() { - // dummy impl - return 0; - } - - @Override - public float getProgress() { - // dummy impl - return 0f; - } - - @Override - public NullWritable createKey() { - return NullWritable.get(); - } - - @Override - public V createValue() { - try { - return clazz.newInstance(); - } catch (Exception e) { - return null; - } - } - - @Override - public boolean next(NullWritable key, V value) throws IOException { - try { - // Need a way to know what thread to interrupt, since this is a blocking thread. - setReaderThread(Thread.currentThread()); - - if (hasInput()) { - value.readFields(din); - return true; - } else { - // End of input. Confirm we got end of stream indicator from server, - // as well as DONE status from fragment execution. - if (!cin.isEndOfData()) { - throw new IOException("Hit end of input, but did not find expected end of data indicator"); - } - - processReaderEvent(); - return false; - } - } catch (IOException io) { - failOnInterruption(io); - return false; - } - } - - protected void processReaderEvent() throws IOException { - // There should be a reader event available, or coming soon, so okay to be blocking call. - ReaderEvent event = getReaderEvent(); - switch (event.getEventType()) { - case DONE: - break; - default: - throw new IOException("Expected reader event with done status, but got " - + event.getEventType() + " with message " + event.getMessage()); - } - } - - protected void failOnInterruption(IOException io) throws IOException { - try { - if (Thread.interrupted()) { - // Either we were interrupted by one of: - // 1. handleEvent(), in which case there is a reader (error) event waiting for us in the queue - // 2. Some other unrelated cause which interrupted us, in which case there may not be a reader event coming. - // Either way we should not try to block trying to read the reader events queue. - if (readerEvents.isEmpty()) { - // Case 2. - throw io; - } else { - // Case 1. Fail the reader, sending back the error we received from the reader event. - ReaderEvent event = getReaderEvent(); - switch (event.getEventType()) { - case ERROR: - throw new IOException("Received reader event error: " + event.getMessage(), io); - default: - throw new IOException("Got reader event type " + event.getEventType() - + ", expected error event", io); - } - } - } else { - // If we weren't interrupted, just propagate the error - throw io; - } - } finally { - // The external client handling umbilical responses and the connection to read the incoming - // data are not coupled. Calling close() here to make sure an error in one will cause the - // other to be closed as well. - try { - close(); - } catch (Exception err) { - // Don't propagate errors from close() since this will lose the original error above. - LOG.error("Closing RecordReader due to error and hit another error during close()", err); - } - } - } - - /** - * Define success/error events which are passed to the reader from a different thread. - * The reader will check for these events on end of input and interruption of the reader thread. - */ - public static class ReaderEvent { - public enum EventType { - DONE, - ERROR - } - - protected final EventType eventType; - protected final String message; - - protected ReaderEvent(EventType type, String message) { - this.eventType = type; - this.message = message; - } - - public static ReaderEvent doneEvent() { - return new ReaderEvent(EventType.DONE, ""); - } - - public static ReaderEvent errorEvent(String message) { - return new ReaderEvent(EventType.ERROR, message); - } - - public EventType getEventType() { - return eventType; - } - - public String getMessage() { - return message; - } - } - - public void handleEvent(ReaderEvent event) { - switch (event.getEventType()) { - case DONE: - // Reader will check for the event queue upon the end of the input stream - no need to interrupt. - readerEvents.add(event); - break; - case ERROR: - readerEvents.add(event); - if (readerThread == null) { - throw new RuntimeException("Reader thread is unexpectedly null, during ReaderEvent error " + event.getMessage()); - } - // Reader is using a blocking socket .. interrupt it. - if (LOG.isDebugEnabled()) { - LOG.debug("Interrupting reader thread due to reader event with error " + event.getMessage()); - } - readerThread.interrupt(); - try { - socket.close(); - } catch (IOException e) { - // Leave the client to time out. - LOG.error("Cannot close the socket on error", e); - } - break; - default: - throw new RuntimeException("Unhandled ReaderEvent type " + event.getEventType() + " with message " + event.getMessage()); - } - } - - protected boolean hasInput() throws IOException { - din.mark(1); - if (din.read() >= 0) { - din.reset(); - return true; - } - return false; - } - - protected ReaderEvent getReaderEvent() throws IOException { - try { - ReaderEvent event = readerEvents.take(); - Preconditions.checkNotNull(event); - return event; - } catch (InterruptedException ie) { - throw new RuntimeException("Interrupted while getting readerEvents, not expected: " + ie.getMessage(), ie); - } - } - - protected synchronized void setReaderThread(Thread readerThread) { - this.readerThread = readerThread; - } - - protected synchronized Thread getReaderThread() { - return readerThread; - } -} diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java b/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java deleted file mode 100644 index 619346fd9684..000000000000 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hive.llap; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.hive.llap.ext.LlapDaemonInfo; -import org.apache.hadoop.mapred.InputSplitWithLocationInfo; -import org.apache.hadoop.mapred.SplitLocationInfo; - -public class LlapInputSplit implements InputSplitWithLocationInfo { - - private int splitNum; - private byte[] planBytes; - private byte[] fragmentBytes; - private SplitLocationInfo[] locations; - private LlapDaemonInfo[] llapDaemonInfos; - private Schema schema; - private String llapUser; - private byte[] fragmentBytesSignature; - private byte[] tokenBytes; - //only needed in cloud deployments for llap server to validate request from external llap clients. - //HS2 generates a JWT and populates this field while get_splits() call, this jwt gets validated at LLAP server - //when LlapInputSplit is submitted. - private String jwt; - - public LlapInputSplit() { - } - - public LlapInputSplit(int splitNum, byte[] planBytes, byte[] fragmentBytes, - byte[] fragmentBytesSignature, SplitLocationInfo[] locations, - LlapDaemonInfo[] llapDaemonInfos, Schema schema, - String llapUser, byte[] tokenBytes, String jwt) { - this.planBytes = planBytes; - this.fragmentBytes = fragmentBytes; - this.fragmentBytesSignature = fragmentBytesSignature; - this.locations = locations; - this.llapDaemonInfos = llapDaemonInfos; - this.schema = schema; - this.splitNum = splitNum; - this.llapUser = llapUser; - this.tokenBytes = tokenBytes; - this.jwt = jwt; - } - - public Schema getSchema() { - return schema; - } - - @Override - public long getLength() throws IOException { - return 0; - } - - @Override - public String[] getLocations() throws IOException { - String[] locs = new String[locations.length]; - for (int i = 0; i < locations.length; ++i) { - locs[i] = locations[i].getLocation(); - } - return locs; - } - - public int getSplitNum() { - return splitNum; - } - - public byte[] getPlanBytes() { - return planBytes; - } - - public byte[] getFragmentBytes() { - return fragmentBytes; - } - - public byte[] getFragmentBytesSignature() { - return fragmentBytesSignature; - } - - public byte[] getTokenBytes() { - return tokenBytes; - } - - public void setPlanBytes(byte[] planBytes) { - this.planBytes = planBytes; - } - - public void setSchema(Schema schema) { - this.schema = schema; - } - - public String getJwt() { - return jwt; - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeInt(splitNum); - out.writeInt(planBytes.length); - out.write(planBytes); - - out.writeInt(fragmentBytes.length); - out.write(fragmentBytes); - if (fragmentBytesSignature != null) { - out.writeInt(fragmentBytesSignature.length); - out.write(fragmentBytesSignature); - } else { - out.writeInt(0); - } - - out.writeInt(locations.length); - for (int i = 0; i < locations.length; ++i) { - out.writeUTF(locations[i].getLocation()); - } - - out.writeInt(llapDaemonInfos.length); - for (LlapDaemonInfo llapDaemonInfo : llapDaemonInfos) { - llapDaemonInfo.write(out); - } - - schema.write(out); - out.writeUTF(llapUser); - if (tokenBytes != null) { - out.writeInt(tokenBytes.length); - out.write(tokenBytes); - } else { - out.writeInt(0); - } - - if (jwt != null) { - out.writeUTF(jwt); - } - } - - @Override - public void readFields(DataInput in) throws IOException { - splitNum = in.readInt(); - int length = in.readInt(); - planBytes = new byte[length]; - in.readFully(planBytes); - - length = in.readInt(); - fragmentBytes = new byte[length]; - in.readFully(fragmentBytes); - length = in.readInt(); - if (length > 0) { - fragmentBytesSignature = new byte[length]; - in.readFully(fragmentBytesSignature); - } - - length = in.readInt(); - locations = new SplitLocationInfo[length]; - - for (int i = 0; i < length; ++i) { - locations[i] = new SplitLocationInfo(in.readUTF(), false); - } - - llapDaemonInfos = new LlapDaemonInfo[in.readInt()]; - for (int i = 0; i < llapDaemonInfos.length; i++) { - llapDaemonInfos[i] = new LlapDaemonInfo(); - llapDaemonInfos[i].readFields(in); - } - - schema = new Schema(); - schema.readFields(in); - llapUser = in.readUTF(); - length = in.readInt(); - if (length > 0) { - tokenBytes = new byte[length]; - in.readFully(tokenBytes); - } - jwt = in.readUTF(); - } - - @Override - public SplitLocationInfo[] getLocationInfo() throws IOException { - return locations; - } - - public String getLlapUser() { - return llapUser; - } - - public LlapDaemonInfo[] getLlapDaemonInfos() { - return llapDaemonInfos; - } -} diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapRowRecordReader.java b/llap-client/src/java/org/apache/hadoop/hive/llap/LlapRowRecordReader.java deleted file mode 100644 index 3dc450470ccc..000000000000 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapRowRecordReader.java +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap; - -import com.google.common.base.Preconditions; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.mapred.RecordReader; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.JobConf; - -import org.apache.hadoop.hive.common.type.HiveChar; -import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.llap.Row; -import org.apache.hadoop.hive.llap.FieldDesc; -import org.apache.hadoop.hive.llap.Schema; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.AbstractSerDe; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.io.HiveCharWritable; -import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; -import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; -import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe; -import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Row-based record reader for LLAP. - */ -public class LlapRowRecordReader implements RecordReader { - - private static final Logger LOG = LoggerFactory.getLogger(LlapRowRecordReader.class); - - protected final Configuration conf; - protected final RecordReader reader; - protected final Schema schema; - protected final AbstractSerDe serde; - protected final Writable data; - - public LlapRowRecordReader(Configuration conf, Schema schema, - RecordReader reader) throws IOException { - this.conf = conf; - this.schema = schema; - this.reader = reader; - this.data = reader.createValue(); - - try { - this.serde = initSerDe(conf); - } catch (SerDeException err) { - throw new IOException(err); - } - } - - @Override - public void close() throws IOException { - reader.close(); - } - - @Override - public NullWritable createKey() { - return NullWritable.get(); - } - - @Override - public Row createValue() { - return new Row(schema); - } - - @Override - public long getPos() throws IOException { - return 0; - } - - @Override - public float getProgress() throws IOException { - return 0; - } - - @Override - public boolean next(NullWritable key, Row value) throws IOException { - Preconditions.checkArgument(value != null); - - boolean hasNext = reader.next(key, data); - if (hasNext) { - // Deserialize data to column values, and populate the row record - Object rowObj; - try { - StructObjectInspector rowOI = (StructObjectInspector) serde.getObjectInspector(); - rowObj = serde.deserialize(data); - setRowFromStruct(value, rowObj, rowOI); - } catch (SerDeException err) { - LOG.debug("Error deserializing row from data: {}", data); - throw new IOException("Error deserializing row data", err); - } - } - - return hasNext; - } - - public Schema getSchema() { - return schema; - } - - static Object convertPrimitive(Object val, PrimitiveObjectInspector poi) { - switch (poi.getPrimitiveCategory()) { - // Save char/varchar as string - case CHAR: - return ((HiveChar) poi.getPrimitiveJavaObject(val)).getPaddedValue(); - case VARCHAR: - return ((HiveVarchar) poi.getPrimitiveJavaObject(val)).toString(); - case DECIMAL: - return ((HiveDecimal) poi.getPrimitiveJavaObject(val)).bigDecimalValue(); - default: - return poi.getPrimitiveJavaObject(val); - } - } - - static Object convertValue(Object val, ObjectInspector oi) { - if (val == null) { - return null; - } - - Object convertedVal = null; - ObjectInspector.Category oiCategory = oi.getCategory(); - switch (oiCategory) { - case PRIMITIVE: - convertedVal = convertPrimitive(val, (PrimitiveObjectInspector) oi); - break; - case LIST: - ListObjectInspector loi = (ListObjectInspector) oi; - int listSize = loi.getListLength(val); - // Per ListObjectInpsector.getListLength(), -1 length means null list. - if (listSize < 0) { - return null; - } - List convertedList = new ArrayList(listSize); - ObjectInspector listElementOI = loi.getListElementObjectInspector(); - for (int idx = 0; idx < listSize; ++idx) { - convertedList.add(convertValue(loi.getListElement(val, idx), listElementOI)); - } - convertedVal = convertedList; - break; - case MAP: - MapObjectInspector moi = (MapObjectInspector) oi; - int mapSize = moi.getMapSize(val); - // Per MapObjectInpsector.getMapSize(), -1 length means null map. - if (mapSize < 0) { - return null; - } - Map convertedMap = new LinkedHashMap(mapSize); - ObjectInspector mapKeyOI = moi.getMapKeyObjectInspector(); - ObjectInspector mapValOI = moi.getMapValueObjectInspector(); - Map mapCol = moi.getMap(val); - for (Object mapKey : mapCol.keySet()) { - Object convertedMapKey = convertValue(mapKey, mapKeyOI); - Object convertedMapVal = convertValue(mapCol.get(mapKey), mapValOI); - convertedMap.put(convertedMapKey, convertedMapVal); - } - convertedVal = convertedMap; - break; - case STRUCT: - StructObjectInspector soi = (StructObjectInspector) oi; - List convertedRow = new ArrayList(); - for (StructField structField : soi.getAllStructFieldRefs()) { - Object convertedFieldValue = convertValue( - soi.getStructFieldData(val, structField), - structField.getFieldObjectInspector()); - convertedRow.add(convertedFieldValue); - } - convertedVal = convertedRow; - break; - default: - throw new IllegalArgumentException("Cannot convert type " + oiCategory); - } - - return convertedVal; - } - - protected static void setRowFromStruct(Row row, Object structVal, StructObjectInspector soi) { - Schema structSchema = row.getSchema(); - // Add struct field data to the Row - List structFields = soi.getAllStructFieldRefs(); - for (int idx = 0; idx < structFields.size(); ++idx) { - StructField structField = structFields.get(idx); - - Object convertedFieldValue = convertValue( - soi.getStructFieldData(structVal, structField), - structField.getFieldObjectInspector()); - row.setValue(idx, convertedFieldValue); - } - } - - //Factory method for serDe - protected AbstractSerDe createSerDe() throws SerDeException { - return new LazyBinarySerDe(); - } - - protected AbstractSerDe initSerDe(Configuration conf) throws SerDeException { - Properties props = new Properties(); - StringBuilder columnsBuffer = new StringBuilder(); - StringBuilder typesBuffer = new StringBuilder(); - boolean isFirst = true; - for (FieldDesc colDesc : schema.getColumns()) { - if (!isFirst) { - columnsBuffer.append(','); - typesBuffer.append(','); - } - columnsBuffer.append(colDesc.getName()); - typesBuffer.append(colDesc.getTypeInfo().toString()); - isFirst = false; - } - String columns = columnsBuffer.toString(); - String types = typesBuffer.toString(); - props.put(serdeConstants.LIST_COLUMNS, columns); - props.put(serdeConstants.LIST_COLUMN_TYPES, types); - props.put(serdeConstants.ESCAPE_CHAR, "\\"); - AbstractSerDe createdSerDe = createSerDe(); - createdSerDe.initialize(conf, props, null); - - return createdSerDe; - } -} diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/SubmitWorkInfo.java b/llap-client/src/java/org/apache/hadoop/hive/llap/SubmitWorkInfo.java deleted file mode 100644 index 79395e8a31f0..000000000000 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/SubmitWorkInfo.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hive.llap; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.io.DataInputBuffer; -import org.apache.hadoop.io.DataOutputBuffer; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.tez.common.security.JobTokenIdentifier; -import org.apache.tez.common.security.JobTokenSecretManager; - -public class SubmitWorkInfo implements Writable { - - private ApplicationId fakeAppId; - private long creationTime; - private byte[] vertexSpec, vertexSpecSignature; - - // This is used to communicate over the LlapUmbilicalProtocol. Not related to tokens used to - // talk to LLAP daemons itself via the securit work. - private Token token; - private int vertexParallelism; - - public SubmitWorkInfo(ApplicationId fakeAppId, long creationTime, - int vertexParallelism, byte[] vertexSpec, byte[] vertexSpecSignature, - Token token) { - this.fakeAppId = fakeAppId; - this.token = token; - this.creationTime = creationTime; - this.vertexSpec = vertexSpec; - this.vertexSpecSignature = vertexSpecSignature; - this.vertexParallelism = vertexParallelism; - } - - // Empty constructor for writable etc. - public SubmitWorkInfo() { - } - - public ApplicationId getFakeAppId() { - return fakeAppId; - } - - public String getTokenIdentifier() { - return fakeAppId.toString(); - } - - public Token getToken() { - return token; - } - - public long getCreationTime() { - return creationTime; - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeLong(fakeAppId.getClusterTimestamp()); - out.writeInt(fakeAppId.getId()); - token.write(out); - out.writeLong(creationTime); - out.writeInt(vertexParallelism); - if (vertexSpec != null) { - out.writeInt(vertexSpec.length); - out.write(vertexSpec); - } else { - out.writeInt(0); - } - if (vertexSpecSignature != null) { - out.writeInt(vertexSpecSignature.length); - out.write(vertexSpecSignature); - } else { - out.writeInt(0); - } - } - - @Override - public void readFields(DataInput in) throws IOException { - long appIdTs = in.readLong(); - int appIdId = in.readInt(); - fakeAppId = ApplicationId.newInstance(appIdTs, appIdId); - token = new Token<>(); - token.readFields(in); - creationTime = in.readLong(); - vertexParallelism = in.readInt(); - int vertexSpecBytes = in.readInt(); - if (vertexSpecBytes > 0) { - vertexSpec = new byte[vertexSpecBytes]; - in.readFully(vertexSpec); - } - int vertexSpecSignBytes = in.readInt(); - if (vertexSpecSignBytes > 0) { - vertexSpecSignature = new byte[vertexSpecSignBytes]; - in.readFully(vertexSpecSignature); - } - } - - public static byte[] toBytes(SubmitWorkInfo submitWorkInfo) throws IOException { - DataOutputBuffer dob = new DataOutputBuffer(); - submitWorkInfo.write(dob); - return dob.getData(); - } - - public static SubmitWorkInfo fromBytes(byte[] submitWorkInfoBytes) throws IOException { - DataInputBuffer dib = new DataInputBuffer(); - dib.reset(submitWorkInfoBytes, 0, submitWorkInfoBytes.length); - SubmitWorkInfo submitWorkInfo = new SubmitWorkInfo(); - submitWorkInfo.readFields(dib); - return submitWorkInfo; - } - - public byte[] getVertexBinary() { - return vertexSpec; - } - - public byte[] getVertexSignature() { - return vertexSpecSignature; - } - - public int getVertexParallelism() { - return vertexParallelism; - } -} diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/coordinator/LlapCoordinator.java b/llap-client/src/java/org/apache/hadoop/hive/llap/coordinator/LlapCoordinator.java index bc93add89487..61f838757609 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/coordinator/LlapCoordinator.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/coordinator/LlapCoordinator.java @@ -20,7 +20,6 @@ package org.apache.hadoop.hive.llap.coordinator; import java.io.IOException; -import java.util.Random; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -30,13 +29,9 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.llap.DaemonId; import org.apache.hadoop.hive.llap.LlapUtil; -import org.apache.hadoop.hive.llap.coordinator.LlapCoordinator; -import org.apache.hadoop.hive.llap.security.LlapSigner; -import org.apache.hadoop.hive.llap.security.LlapSignerImpl; import org.apache.hadoop.hive.llap.security.LlapTokenLocalClient; import org.apache.hadoop.hive.llap.security.LlapTokenLocalClientImpl; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.ApplicationId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,17 +48,6 @@ public class LlapCoordinator { private static final Logger LOG = LoggerFactory.getLogger(LlapCoordinator.class); - /** We'll keep signers per cluster around for some time, for reuse. */ - private final Cache signers = CacheBuilder.newBuilder().removalListener( - new RemovalListener() { - @Override - public void onRemoval(RemovalNotification notification) { - if (notification.getValue() != null) { - notification.getValue().close(); - } - } - }).expireAfterAccess(10, TimeUnit.MINUTES).build(); - // TODO: probably temporary before HIVE-13698; after that we may create one per session. private static final Cache localClientCache = CacheBuilder .newBuilder().expireAfterAccess(10, TimeUnit.MINUTES) @@ -77,8 +61,6 @@ public void onRemoval(RemovalNotification notifica }).build(); private HiveConf hiveConf; - private String clusterUser; - private long startTime; private final AtomicInteger appIdCounter = new AtomicInteger(0); LlapCoordinator() { @@ -88,37 +70,6 @@ private void init(HiveConf hiveConf) throws IOException { // Only do the lightweight stuff in ctor; by default, LLAP coordinator is created during // HS2 init without the knowledge of LLAP usage (or lack thereof) in the cluster. this.hiveConf = hiveConf; - this.clusterUser = UserGroupInformation.getCurrentUser().getShortUserName(); - // TODO: if two HS2s start at exactly the same time, which could happen during a coordinated - // restart, they could start generating the same IDs. Should we store the startTime - // somewhere like ZK? Try to randomize it a bit for now... - long randomBits = (long)(new Random().nextInt()) << 32; - this.startTime = Math.abs((System.currentTimeMillis() & (long)Integer.MAX_VALUE) | randomBits); - } - - public LlapSigner getLlapSigner(final Configuration jobConf) { - // Note that we create the cluster name from user conf (hence, a user can target a cluster), - // but then we create the signer using hiveConf (hence, we control the ZK config and stuff). - assert UserGroupInformation.isSecurityEnabled(); - final String clusterId = DaemonId.createClusterString( - clusterUser, LlapUtil.generateClusterName(jobConf)); - try { - return signers.get(clusterId, new Callable() { - public LlapSigner call() throws Exception { - return new LlapSignerImpl(hiveConf, clusterId); - } - }); - } catch (ExecutionException e) { - throw new RuntimeException(e); - } - } - - public ApplicationId createExtClientAppId() { - // Note that we cannot allow users to provide app ID, since providing somebody else's appId - // would give one LLAP token (and splits) for that app ID. If we could verify it somehow - // (YARN token? nothing we can do in an UDF), we could get it from client already running on - // YARN. As such, the clients running on YARN will have two app IDs to be aware of. - return ApplicationId.newInstance(startTime, appIdCounter.incrementAndGet()); } public LlapTokenLocalClient getLocalTokenClient( @@ -144,9 +95,7 @@ public LlapTokenLocalClientImpl call() throws Exception { public void close() { try { localClientCache.invalidateAll(); - signers.invalidateAll(); localClientCache.cleanUp(); - signers.cleanUp(); } catch (Exception ex) { LOG.error("Error closing the coordinator; ignoring", ex); } diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/ext/LlapDaemonInfo.java b/llap-client/src/java/org/apache/hadoop/hive/llap/ext/LlapDaemonInfo.java deleted file mode 100644 index b59b3b9e31af..000000000000 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/ext/LlapDaemonInfo.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap.ext; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.Objects; - -import org.apache.hadoop.io.Writable; - -/** - * LlapDaemonInfo - contains llap daemon information - * - host - hostname of llap daemon - * - rpcPort - rpc port of llap daemon to submit fragments - * - outputFormatPort - output port of llap daemon to read data corresponding to the submitted fragment - */ -public class LlapDaemonInfo implements Writable { - - private String host; - private int rpcPort; - private int outputFormatPort; - - public LlapDaemonInfo(String host, int rpcPort, int outputFormatPort) { - this.host = host; - this.rpcPort = rpcPort; - this.outputFormatPort = outputFormatPort; - } - - public LlapDaemonInfo() { - } - - public String getHost() { - return host; - } - - public int getRpcPort() { - return rpcPort; - } - - public int getOutputFormatPort() { - return outputFormatPort; - } - - @Override - public String toString() { - return "LlapDaemonInfo{" + - "host='" + host + '\'' + - ", rpcPort=" + rpcPort + - ", outputFormatPort=" + outputFormatPort + - '}'; - } - - @Override public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - LlapDaemonInfo that = (LlapDaemonInfo) o; - return rpcPort == that.rpcPort && outputFormatPort == that.outputFormatPort && Objects.equals(host, that.host); - } - - @Override public int hashCode() { - return Objects.hash(host, rpcPort, outputFormatPort); - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeUTF(host); - out.writeInt(rpcPort); - out.writeInt(outputFormatPort); - } - - @Override - public void readFields(DataInput in) throws IOException { - host = in.readUTF(); - rpcPort = in.readInt(); - outputFormatPort = in.readInt(); - } -} diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/ext/LlapTaskUmbilicalExternalClient.java b/llap-client/src/java/org/apache/hadoop/hive/llap/ext/LlapTaskUmbilicalExternalClient.java deleted file mode 100644 index 72732a882a01..000000000000 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/ext/LlapTaskUmbilicalExternalClient.java +++ /dev/null @@ -1,618 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hive.llap.ext; - -import org.apache.hadoop.io.Writable; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol.TezAttemptArray; - -import org.apache.hadoop.io.ArrayWritable; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import com.google.common.collect.Lists; -import com.google.protobuf.InvalidProtocolBufferException; - -import org.apache.commons.collections4.ListUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryIdentifierProto; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SignableVertexSpec; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmissionStateProto; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.VertexOrBinary; -import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol; -import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier; -import org.apache.hadoop.hive.llap.tez.Converters; -import org.apache.hadoop.hive.llap.tez.LlapProtocolClientProxy; -import org.apache.hadoop.hive.llap.tezplugins.helpers.LlapTaskUmbilicalServer; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.ipc.ProtocolSignature; -import org.apache.hadoop.security.token.Token; -import org.apache.tez.common.security.JobTokenIdentifier; -import org.apache.tez.dag.api.TezException; -import org.apache.tez.dag.records.TezTaskAttemptID; -import org.apache.tez.runtime.api.impl.EventType; -import org.apache.tez.runtime.api.impl.TezEvent; -import org.apache.tez.runtime.api.impl.TezHeartbeatRequest; -import org.apache.tez.runtime.api.impl.TezHeartbeatResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class LlapTaskUmbilicalExternalClient implements Closeable { - - private static final Logger LOG = LoggerFactory.getLogger(LlapTaskUmbilicalExternalClient.class); - - private static ScheduledThreadPoolExecutor retryExecutor = new ScheduledThreadPoolExecutor(1); - - private final Random rand = new Random(); - private final LlapProtocolClientProxy communicator; - private volatile LlapTaskUmbilicalServer llapTaskUmbilicalServer; - private final Configuration conf; - - protected final String tokenIdentifier; - protected final Token sessionToken; - private LlapTaskUmbilicalExternalResponder responder = null; - private final long connectionTimeout; - private long baseDelay; - private int attemptNum = 0; - private volatile boolean closed = false; - private volatile boolean timeoutsDisabled = false; - private RequestInfo requestInfo; - List tezEvents; - - // Using a shared instance of the umbilical server. - private static class SharedUmbilicalServer { - LlapTaskUmbilicalExternalImpl umbilicalProtocol; - LlapTaskUmbilicalServer llapTaskUmbilicalServer; - - private volatile static SharedUmbilicalServer instance; - private static final Object lock = new Object(); - - static SharedUmbilicalServer getInstance(Configuration conf) { - SharedUmbilicalServer value = instance; - if (value == null) { - synchronized (lock) { - if (instance == null) { - instance = new SharedUmbilicalServer(conf); - } - value = instance; - } - } - return value; - } - - private SharedUmbilicalServer(Configuration conf) { - try { - umbilicalProtocol = new LlapTaskUmbilicalExternalImpl(conf); - llapTaskUmbilicalServer = new LlapTaskUmbilicalServer(conf, umbilicalProtocol, 1); - } catch (Exception err) { - throw new ExceptionInInitializerError(err); - } - } - } - - private enum RequestState { - PENDING, RUNNING - }; - - private static class RequestInfo { - RequestState state; - final SubmitWorkRequestProto request; - final QueryIdentifierProto queryIdentifierProto; - final String taskAttemptId; - final String hostname; - String uniqueNodeId; - final int port; - final AtomicLong lastHeartbeat = new AtomicLong(); - - public RequestInfo(SubmitWorkRequestProto request, QueryIdentifierProto queryIdentifierProto, - String taskAttemptId, String hostname, int port) { - this.state = RequestState.PENDING; - this.request = request; - this.queryIdentifierProto = queryIdentifierProto; - this.taskAttemptId = taskAttemptId; - this.hostname = hostname; - this.port = port; - this.lastHeartbeat.set(System.currentTimeMillis()); - } - } - - public LlapTaskUmbilicalExternalClient(Configuration conf, String tokenIdentifier, - Token sessionToken, LlapTaskUmbilicalExternalResponder responder, - Token llapToken) { - this.conf = conf; - this.tokenIdentifier = tokenIdentifier; - this.sessionToken = sessionToken; - this.responder = responder; - this.connectionTimeout = 3 * HiveConf.getTimeVar(conf, - HiveConf.ConfVars.LLAP_DAEMON_AM_LIVENESS_CONNECTION_TIMEOUT_MS, TimeUnit.MILLISECONDS); - this.baseDelay = HiveConf.getTimeVar(conf, - HiveConf.ConfVars.LLAP_DAEMON_AM_LIVENESS_CONNECTION_SLEEP_BETWEEN_RETRIES_MS, - TimeUnit.MILLISECONDS); - // Add support for configurable threads, however 1 should always be enough. - this.communicator = new LlapProtocolClientProxy(1, conf, llapToken); - this.communicator.init(conf); - } - - private void terminateRequest() { - if (closed || requestInfo == null) { - LOG.warn("No current request to terminate"); - return; - } - - TerminateFragmentRequestProto.Builder builder = TerminateFragmentRequestProto.newBuilder(); - builder.setQueryIdentifier(requestInfo.queryIdentifierProto); - builder.setFragmentIdentifierString(requestInfo.taskAttemptId); - - final String taskAttemptId = requestInfo.taskAttemptId; - communicator.sendTerminateFragment(builder.build(), requestInfo.hostname, requestInfo.port, - new LlapProtocolClientProxy.ExecuteRequestCallback() { - - @Override - public void setResponse(TerminateFragmentResponseProto response) { - LOG.debug("Received terminate response for " + taskAttemptId); - } - - @Override - public void indicateError(Throwable t) { - String msg = "Failed to terminate " + taskAttemptId; - LOG.error(msg, t); - // Don't propagate the error - termination was done as part of closing the client. - } - }); - } - - public InetSocketAddress getAddress() { - return SharedUmbilicalServer.getInstance(conf).llapTaskUmbilicalServer.getAddress(); - } - - - /** - * Submit the work for actual execution. - */ - public void submitWork(SubmitWorkRequestProto request, String llapHost, int llapPort) { - // Register the pending events to be sent for this spec. - VertexOrBinary vob = request.getWorkSpec(); - assert vob.hasVertexBinary() != vob.hasVertex(); - SignableVertexSpec vertex = null; - try { - vertex = vob.hasVertex() ? vob.getVertex() - : SignableVertexSpec.parseFrom(vob.getVertexBinary()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - QueryIdentifierProto queryIdentifierProto = vertex.getQueryIdentifier(); - TezTaskAttemptID attemptId = Converters.createTaskAttemptId(queryIdentifierProto, - vertex.getVertexIndex(), request.getFragmentNumber(), request.getAttemptNumber()); - final String fragmentId = attemptId.toString(); - - this.requestInfo = new RequestInfo(request, queryIdentifierProto, fragmentId, llapHost, llapPort); - - this.tezEvents = Lists.newArrayList(); - registerClient(); - - // Send out the actual SubmitWorkRequest - final LlapTaskUmbilicalExternalClient client = this; - communicator.start(); - submitWork(); - } - - private void submitWork() { - if (!closed) { - communicator.sendSubmitWork(requestInfo.request, - requestInfo.hostname, requestInfo.port, new SubmitWorkCallback(this)); - } - } - - private void retrySubmission() { - attemptNum++; - - // Don't retry immediately - use delay with exponential backoff - long retryDelay = determineRetryDelay(); - LOG.info("Queueing fragment for resubmission {}, attempt {}, delay {}", - this.requestInfo.taskAttemptId, attemptNum, retryDelay); - disableTimeouts(); // Don't timeout because of retry delay - retryExecutor.schedule( - new Runnable() { - @Override - public void run() { - // Re-enable timeouts - enableTimeouts(); - submitWork(); - } - }, - retryDelay, - TimeUnit.MILLISECONDS); - } - - // Helper class to submit fragments to LLAP and retry rejected submissions. - static class SubmitWorkCallback implements LlapProtocolClientProxy.ExecuteRequestCallback { - private LlapTaskUmbilicalExternalClient client; - - public SubmitWorkCallback(LlapTaskUmbilicalExternalClient client) { - this.client = client; - } - - @Override - public void setResponse(SubmitWorkResponseProto response) { - if (response.hasSubmissionState()) { - if (response.getSubmissionState().equals(SubmissionStateProto.REJECTED)) { - String fragmentId = this.client.requestInfo.taskAttemptId; - String msg = "Fragment: " + fragmentId + " rejected. Server Busy."; - LOG.info(msg); - - // taskKill() should also be received during a rejected submission, - // we will let that logic handle retries. - - return; - } - } - if (response.hasUniqueNodeId()) { - client.requestInfo.uniqueNodeId = response.getUniqueNodeId(); - } - } - - @Override - public void indicateError(Throwable t) { - String fragmentId = this.client.requestInfo.taskAttemptId; - String msg = "Failed to submit: " + fragmentId; - LOG.error(msg, t); - Throwable err = new RuntimeException(msg, t); - client.unregisterClient(); - client.responder.submissionFailed(fragmentId, err); - } - } - - @Override - public void close() { - if (!closed) { - terminateRequest(); - unregisterClient(); - } - } - - private void registerClient() { - SharedUmbilicalServer umbilicalServer = SharedUmbilicalServer.getInstance(conf); - LlapTaskUmbilicalExternalClient prevVal = - umbilicalServer.umbilicalProtocol.registeredClients.putIfAbsent(requestInfo.taskAttemptId, this); - if (prevVal != null) { - LOG.warn("Unexpected - fragment " + requestInfo.taskAttemptId + " is already registered!"); - } - umbilicalServer.llapTaskUmbilicalServer.addTokenForJob(tokenIdentifier, sessionToken); - } - - private void unregisterClient() { - if (!closed && requestInfo != null) { - communicator.stop(); - SharedUmbilicalServer umbilicalServer = SharedUmbilicalServer.getInstance(conf); - umbilicalServer.umbilicalProtocol.unregisterClient(requestInfo.taskAttemptId); - umbilicalServer.llapTaskUmbilicalServer.removeTokenForJob(tokenIdentifier); - closed = true; - } - } - - long getLastHeartbeat() { - return this.requestInfo.lastHeartbeat.get(); - } - - void setLastHeartbeat(long lastHeartbeat) { - this.requestInfo.lastHeartbeat.set(lastHeartbeat); - } - - private boolean isTimedOut(long currentTime) { - if (timeoutsDisabled) { - return false; - } - return (currentTime - getLastHeartbeat() >= connectionTimeout); - } - - private void enableTimeouts() { - setLastHeartbeat(System.currentTimeMillis()); - timeoutsDisabled = false; - } - - private void disableTimeouts() { - timeoutsDisabled = true; - } - - private long determineRetryDelay() { - // Delay with exponential backoff - int maxDelay = (int) Math.min(baseDelay * Math.pow(2, attemptNum), 60000); - long retryDelay = rand.nextInt(maxDelay); - return retryDelay; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("LlapTaskUmbilicalExternalClient"); - if (requestInfo != null) { - sb.append("("); - sb.append(requestInfo.taskAttemptId); - sb.append(")"); - } - return sb.toString(); - } - - // Periodic task to time out submitted tasks that have not been updated with umbilical heartbeat. - private static class HeartbeatCheckTask implements Runnable { - LlapTaskUmbilicalExternalImpl umbilicalImpl; - - public HeartbeatCheckTask(LlapTaskUmbilicalExternalImpl umbilicalImpl) { - this.umbilicalImpl = umbilicalImpl; - } - - public void run() { - long currentTime = System.currentTimeMillis(); - List timedOutTasks = new ArrayList(); - - for (Map.Entry entry : umbilicalImpl.registeredClients.entrySet()) { - LlapTaskUmbilicalExternalClient client = entry.getValue(); - if (client.isTimedOut(currentTime)) { - timedOutTasks.add(client); - } - } - - for (LlapTaskUmbilicalExternalClient timedOutTask : timedOutTasks) { - String taskAttemptId = timedOutTask.requestInfo.taskAttemptId; - LOG.info("Running taskAttemptId " + taskAttemptId + " timed out"); - timedOutTask.unregisterClient(); - timedOutTask.responder.heartbeatTimeout(taskAttemptId); - } - } - } - - public interface LlapTaskUmbilicalExternalResponder { - void submissionFailed(String fragmentId, Throwable throwable); - void heartbeat(TezHeartbeatRequest request); - void taskKilled(TezTaskAttemptID taskAttemptId); - void heartbeatTimeout(String fragmentId); - } - - private static class LlapTaskUmbilicalExternalImpl implements LlapTaskUmbilicalProtocol { - - final ConcurrentMap registeredClients = new ConcurrentHashMap<>(); - private final ScheduledThreadPoolExecutor timer; - - public LlapTaskUmbilicalExternalImpl(Configuration conf) { - long taskInterval = HiveConf.getTimeVar(conf, - HiveConf.ConfVars.LLAP_DAEMON_AM_LIVENESS_CONNECTION_TIMEOUT_MS, TimeUnit.MILLISECONDS); - // Setup timer task to check for hearbeat timeouts - this.timer = new ScheduledThreadPoolExecutor(1); - timer.scheduleAtFixedRate(new HeartbeatCheckTask(this), - taskInterval, taskInterval, TimeUnit.MILLISECONDS); - } - - @Override - public boolean canCommit(TezTaskAttemptID taskid) throws IOException { - // Expecting only a single instance of a task to be running. - return true; - } - - @Override - public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOException, - TezException { - // Keep-alive information. The client should be informed and will have to take care of re-submitting the work. - // Some parts of fault tolerance go here. - - // This also provides completion information, and a possible notification when task actually starts running (first heartbeat) - - LOG.debug("Received heartbeat from container, request={}", request); - - // Incoming events can be ignored until the point when shuffle needs to be handled, instead of just scans. - TezHeartbeatResponse response = new TezHeartbeatResponse(); - - response.setLastRequestId(request.getRequestId()); - // Assuming TaskAttemptId and FragmentIdentifierString are the same. Verify this. - TezTaskAttemptID taskAttemptId = request.getCurrentTaskAttemptID(); - String taskAttemptIdString = taskAttemptId.toString(); - updateHeartbeatInfo(taskAttemptIdString); - - List tezEvents = null; - LlapTaskUmbilicalExternalClient client = registeredClients.get(taskAttemptIdString); - if (client == null) { - // Heartbeat is from a task that we are not currently tracking. - LOG.info("Unexpected heartbeat from " + taskAttemptIdString); - response.setShouldDie(); // Do any of the other fields need to be set? - return response; - } - - if (client.requestInfo.state == RequestState.PENDING) { - client.requestInfo.state = RequestState.RUNNING; - tezEvents = client.tezEvents; - } else { - tezEvents = Collections.emptyList(); - } - - boolean shouldUnregisterClient = false; - - response.setLastRequestId(request.getRequestId()); - // Irrelevant from eventIds. This can be tracked in the AM itself, instead of polluting the task. - // Also since we have all the MRInput events here - they'll all be sent in together. - response.setNextFromEventId(0); // Irrelevant. See comment above. - response.setNextPreRoutedEventId(0); //Irrelevant. See comment above. - response.setEvents(tezEvents); - - List inEvents = request.getEvents(); - if (LOG.isDebugEnabled()) { - LOG.debug("Heartbeat from " + taskAttemptIdString + - " events: " + (inEvents != null ? inEvents.size() : -1)); - } - for (TezEvent tezEvent : ListUtils.emptyIfNull(inEvents)) { - EventType eventType = tezEvent.getEventType(); - switch (eventType) { - case TASK_ATTEMPT_COMPLETED_EVENT: - LOG.debug("Task completed event for " + taskAttemptIdString); - shouldUnregisterClient = true; - break; - case TASK_ATTEMPT_FAILED_EVENT: - LOG.debug("Task failed event for " + taskAttemptIdString); - shouldUnregisterClient = true; - break; - case TASK_STATUS_UPDATE_EVENT: - // If we want to handle counters - LOG.debug("Task update event for " + taskAttemptIdString); - break; - default: - LOG.warn("Unhandled event type " + eventType); - break; - } - } - - if (shouldUnregisterClient) { - client.unregisterClient(); - } - - // Pass the request on to the responder - try { - if (client.responder != null) { - client.responder.heartbeat(request); - } - } catch (Exception err) { - LOG.error("Error during responder execution", err); - } - - return response; - } - - @Override - public void nodeHeartbeat(Text hostname, Text uniqueId, int port, TezAttemptArray aw, - BooleanArray guaranteed) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("Node heartbeat from " + hostname + ":" + port + ", " + uniqueId); - } - // External client currently cannot use guaranteed. - updateHeartbeatInfo(hostname.toString(), uniqueId.toString(), port, aw); - // No need to propagate to this to the responder - } - - @Override - public void taskKilled(TezTaskAttemptID taskAttemptId) throws IOException { - String taskAttemptIdString = taskAttemptId.toString(); - LlapTaskUmbilicalExternalClient client = registeredClients.get(taskAttemptIdString); - if (client != null) { - if (client.requestInfo.state == RequestState.PENDING) { - // A task kill while the request is still in PENDING state means the request should be retried. - LOG.info("Received task kill for {} which is still in pending state. Retry submission.", taskAttemptIdString); - client.retrySubmission(); - } else { - try { - LOG.error("Task killed - " + taskAttemptIdString); - client.unregisterClient(); - if (client.responder != null) { - client.responder.taskKilled(taskAttemptId); - } - } catch (Exception err) { - LOG.error("Error during responder execution", err); - } - } - } else { - LOG.info("Received task killed notification for task which is not currently being tracked: " + taskAttemptId); - } - } - - @Override - public long getProtocolVersion(String protocol, long clientVersion) throws IOException { - return 0; - } - - @Override - public ProtocolSignature getProtocolSignature(String protocol, long clientVersion, - int clientMethodsHash) throws IOException { - return ProtocolSignature.getProtocolSignature(this, protocol, - clientVersion, clientMethodsHash); - } - - private void unregisterClient(String taskAttemptId) { - registeredClients.remove(taskAttemptId); - } - - private void updateHeartbeatInfo(String taskAttemptId) { - int updateCount = 0; - - LlapTaskUmbilicalExternalClient registeredClient = registeredClients.get(taskAttemptId); - if (registeredClient != null) { - registeredClient.setLastHeartbeat(System.currentTimeMillis()); - updateCount++; - } - - if (updateCount == 0) { - LOG.warn("No tasks found for heartbeat from taskAttemptId " + taskAttemptId); - } - } - - private void updateHeartbeatInfo( - String hostname, String uniqueId, int port, TezAttemptArray tasks) { - int updateCount = 0; - HashSet attempts = new HashSet<>(); - for (Writable w : tasks.get()) { - attempts.add((TezTaskAttemptID)w); - } - - String error = ""; - for (Map.Entry entry : registeredClients.entrySet()) { - LlapTaskUmbilicalExternalClient registeredClient = entry.getValue(); - if (doesClientMatchHeartbeat(registeredClient, hostname, uniqueId, port)) { - TezTaskAttemptID ta = TezTaskAttemptID.fromString(registeredClient.requestInfo.taskAttemptId); - if (attempts.contains(ta)) { - registeredClient.setLastHeartbeat(System.currentTimeMillis()); - updateCount++; - } else { - error += (registeredClient.requestInfo.taskAttemptId + ", "); - } - } - } - if (!error.isEmpty()) { - LOG.info("The tasks we expected to be on the node are not there: " + error); - } - - if (updateCount == 0) { - LOG.info("No tasks found for heartbeat from hostname " + hostname + ", port " + port); - } - } - - private static boolean doesClientMatchHeartbeat(LlapTaskUmbilicalExternalClient client, - String hostname, String uniqueId, int port) { - return (hostname.equals(client.requestInfo.hostname) - && port == client.requestInfo.port - && uniqueId.equals(client.requestInfo.uniqueNodeId)); - } - } -} diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java index 75c00970e8de..b912b12276cf 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java @@ -19,9 +19,6 @@ package org.apache.hadoop.hive.llap.registry; -import com.google.common.base.Preconditions; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.llap.LlapUtil; import org.apache.hadoop.hive.registry.ServiceInstance; import org.apache.hadoop.yarn.api.records.Resource; @@ -48,31 +45,6 @@ public interface LlapServiceInstance extends ServiceInstance { * @return */ public String getServicesAddress(); - /** - * OutputFormat endpoint for service instance - * - * @return - */ - public int getOutputFormatPort(); - - /** - * External host, usually needed in cloud envs where we cannot access internal host from outside - * - * @return - */ - String getExternalHostname(); - - /** - * RPC endpoint for external clients - tcp traffic on this port should be opened on cloud. - * - * @return - */ - int getExternalClientsRpcPort(); - - - default void ensureCloudEnv(Configuration conf) { - Preconditions.checkState(LlapUtil.isCloudDeployment(conf), "Only supported in cloud based deployments"); - } /** * Memory and Executors available for the LLAP tasks diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java index 58948fa68d65..a65e6cb99a2c 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java @@ -56,26 +56,11 @@ public int getShufflePort() { throw new UnsupportedOperationException(); } - @Override - public String getExternalHostname() { - throw new UnsupportedOperationException(); - } - - @Override - public int getExternalClientsRpcPort() { - throw new UnsupportedOperationException(); - } - @Override public String getServicesAddress() { throw new UnsupportedOperationException(); } - @Override - public int getOutputFormatPort() { - throw new UnsupportedOperationException(); - } - @Override public Map getProperties() { return Collections.emptyMap(); diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java index af65301a2198..73d126c0b25f 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java @@ -40,11 +40,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.llap.LlapUtil; import org.apache.hadoop.hive.llap.registry.LlapServiceInstance; import org.apache.hadoop.hive.llap.registry.LlapServiceInstanceSet; import org.apache.hadoop.hive.llap.registry.ServiceRegistry; -import org.apache.hadoop.hive.registry.ServiceInstance; import org.apache.hadoop.hive.registry.ServiceInstanceStateChangeListener; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.StringUtils; @@ -65,9 +63,6 @@ public class LlapFixedRegistryImpl implements ServiceRegistry capacityValues = new HashMap<>(2); @@ -196,15 +176,9 @@ public String register() throws IOException { } registerServiceRecord(daemonZkRecord, uniqueId); - if (LlapUtil.isCloudDeployment(conf)) { - LOG.info("Registered node. Created a znode on ZooKeeper for LLAP instance: rpc: {}, external client rpc : {} " - + "shuffle: {}, webui: {}, mgmt: {}, znodePath: {}", rpcEndpoint, externalRpcEndpoint, - getShuffleEndpoint(), getServicesEndpoint(), getMngEndpoint(), getRegistrationZnodePath()); - } else { - LOG.info("Registered node. Created a znode on ZooKeeper for LLAP instance: rpc: {}, " - + "shuffle: {}, webui: {}, mgmt: {}, znodePath: {}", rpcEndpoint, getShuffleEndpoint(), - getServicesEndpoint(), getMngEndpoint(), getRegistrationZnodePath()); - } + LOG.info("Registered node. Created a znode on ZooKeeper for LLAP instance: rpc: {}, " + + "shuffle: {}, webui: {}, mgmt: {}, znodePath: {}", rpcEndpoint, getShuffleEndpoint(), + getServicesEndpoint(), getMngEndpoint(), getRegistrationZnodePath()); return uniqueId; } @@ -239,12 +213,8 @@ public class DynamicServiceInstance extends ServiceInstanceBase implements LlapServiceInstance { private final int mngPort; private final int shufflePort; - private final int outputFormatPort; private final String serviceAddress; - private String externalHost; - private int externalClientsRpcPort; - private final Resource resource; public DynamicServiceInstance(ServiceRecord srv) throws IOException { @@ -252,7 +222,6 @@ public DynamicServiceInstance(ServiceRecord srv) throws IOException { final Endpoint shuffle = srv.getInternalEndpoint(IPC_SHUFFLE); final Endpoint mng = srv.getInternalEndpoint(IPC_MNG); - final Endpoint outputFormat = srv.getInternalEndpoint(IPC_OUTPUTFORMAT); final Endpoint services = srv.getExternalEndpoint(IPC_SERVICES); this.mngPort = @@ -261,21 +230,9 @@ public DynamicServiceInstance(ServiceRecord srv) throws IOException { this.shufflePort = Integer.parseInt(RegistryTypeUtils.getAddressField(shuffle.addresses.get(0), AddressTypes.ADDRESS_PORT_FIELD)); - this.outputFormatPort = - Integer.valueOf(RegistryTypeUtils.getAddressField(outputFormat.addresses.get(0), - AddressTypes.ADDRESS_PORT_FIELD)); this.serviceAddress = RegistryTypeUtils.getAddressField(services.addresses.get(0), AddressTypes.ADDRESS_URI); - if (LlapUtil.isCloudDeployment(conf)) { - final Endpoint externalRpc = srv.getExternalEndpoint(IPC_EXTERNAL_LLAP); - this.externalHost = RegistryTypeUtils.getAddressField(externalRpc.addresses.get(0), - AddressTypes.ADDRESS_HOSTNAME_FIELD); - this.externalClientsRpcPort = Integer.parseInt( - RegistryTypeUtils.getAddressField(externalRpc.addresses.get(0), - AddressTypes.ADDRESS_PORT_FIELD)); - } - String memStr = srv.get(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, ""); String coreStr = srv.get(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS, ""); try { @@ -296,18 +253,6 @@ public String getServicesAddress() { return serviceAddress; } - @Override - public String getExternalHostname() { - ensureCloudEnv(LlapZookeeperRegistryImpl.this.conf); - return externalHost; - } - - @Override - public int getExternalClientsRpcPort() { - ensureCloudEnv(LlapZookeeperRegistryImpl.this.conf); - return externalClientsRpcPort; - } - @Override public Resource getResource() { return resource; @@ -324,11 +269,6 @@ public String toString() { public int getManagementPort() { return mngPort; } - - @Override - public int getOutputFormatPort() { - return outputFormatPort; - } } diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/LlapTaskUmbilicalServer.java b/llap-client/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/LlapTaskUmbilicalServer.java deleted file mode 100644 index 4b184620e403..000000000000 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/LlapTaskUmbilicalServer.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hive.llap.tezplugins.helpers; - -import java.io.IOException; -import java.net.BindException; -import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.Validator.RangeValidator; -import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.ipc.Server; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.authorize.PolicyProvider; -import org.apache.hadoop.security.authorize.Service; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.hive.common.IPStackUtils; -import org.apache.tez.common.security.JobTokenIdentifier; -import org.apache.tez.common.security.JobTokenSecretManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LlapTaskUmbilicalServer { - - private static final Logger LOG = LoggerFactory.getLogger(LlapTaskUmbilicalServer.class); - - protected volatile Server server; - private final InetSocketAddress address; - private final AtomicBoolean started = new AtomicBoolean(true); - private JobTokenSecretManager jobTokenSecretManager; - private Map tokenRefMap = new HashMap(); - - public LlapTaskUmbilicalServer(Configuration conf, LlapTaskUmbilicalProtocol umbilical, int numHandlers) throws IOException { - jobTokenSecretManager = new JobTokenSecretManager(conf); - - String[] portRange = - conf.get(HiveConf.ConfVars.LLAP_TASK_UMBILICAL_SERVER_PORT.varname) - .split("-"); - boolean isHadoopSecurityAuthorizationEnabled = conf.getBoolean( - CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false); - - int minPort = Integer.parseInt(portRange[0]); - boolean portFound = false; - IOException e = null; - if (portRange.length == 1) { - // Single port specified, not Range. - startServer(conf, umbilical, numHandlers, minPort, - isHadoopSecurityAuthorizationEnabled); - portFound = true; - LOG.info("Successfully bound to port {}", minPort); - } else { - int maxPort = Integer.parseInt(portRange[1]); - // Validate the range specified is valid. i.e the ports lie between - // 1024 and 65535. - validatePortRange(portRange[0], portRange[1]); - - for (int i = minPort; i < maxPort; i++) { - try { - startServer(conf, umbilical, numHandlers, i, - isHadoopSecurityAuthorizationEnabled); - portFound = true; - LOG.info("Successfully bound to port {}", i); - break; - } catch (BindException be) { - // Ignore and move ahead, in search of a free port. - LOG.warn("Unable to bind to port {}", i, be); - e = be; - } - } - } - if (!portFound) { - throw e; - } - - this.address = NetUtils.getConnectAddress(server); - LOG.info( - "Started TaskUmbilicalServer: " + umbilical.getClass().getName() + " at address: " + address + - " with numHandlers=" + numHandlers); - } - - private void validatePortRange(String minPort, String maxPort) - throws IOException { - RangeValidator rangeValidator = new RangeValidator(1024L, 65535L); - String valMin = rangeValidator.validate(minPort); - String valMax = rangeValidator.validate(maxPort); - if (valMin == null & valMax == null) { - throw new IOException("Invalid minimum range value: " + minPort + " and " - + "maximum range value: " + maxPort + " for " - + HiveConf.ConfVars.LLAP_TASK_UMBILICAL_SERVER_PORT.varname - + ". The value should be between 1024 and 65535."); - } - if (valMin != null) { - throw new IOException("Invalid minimum range value :" + minPort + " for " - + HiveConf.ConfVars.LLAP_TASK_UMBILICAL_SERVER_PORT.varname - + ". The value should be between 1024 and 65535."); - } - if (valMax != null) { - throw new IOException("Invalid maximum range value:" + maxPort + " for " - + HiveConf.ConfVars.LLAP_TASK_UMBILICAL_SERVER_PORT.varname - + ". The value should be between 1024 and 65535."); - } - } - - private void startServer(Configuration conf, - LlapTaskUmbilicalProtocol umbilical, int numHandlers, int port, - boolean isHadoopSecurityAuthorizationEnabled) throws IOException { - server = new RPC.Builder(conf).setProtocol(LlapTaskUmbilicalProtocol.class) - .setBindAddress(IPStackUtils.resolveWildcardAddress()).setPort(port).setInstance(umbilical) - .setNumHandlers(numHandlers).setSecretManager(jobTokenSecretManager) - .build(); - if (isHadoopSecurityAuthorizationEnabled) { - server.refreshServiceAcl(conf, new LlapUmbilicalExternalPolicyProvider()); - } - server.start(); - } - - public InetSocketAddress getAddress() { - return this.address; - } - - public int getNumOpenConnections() { - return server.getNumOpenConnections(); - } - - public synchronized void addTokenForJob(String tokenIdentifier, Token token) { - // Maintain count of outstanding requests for tokenIdentifier. - int[] refCount = tokenRefMap.get(tokenIdentifier); - if (refCount == null) { - refCount = new int[] { 0 }; - tokenRefMap.put(tokenIdentifier, refCount); - // Should only need to insert the token the first time. - jobTokenSecretManager.addTokenForJob(tokenIdentifier, token); - } - refCount[0]++; - } - - public synchronized void removeTokenForJob(String tokenIdentifier) { - // Maintain count of outstanding requests for tokenIdentifier. - // If count goes to 0, it is safe to remove the token. - int[] refCount = tokenRefMap.get(tokenIdentifier); - if (refCount == null) { - LOG.warn("No refCount found for tokenIdentifier " + tokenIdentifier); - } else { - refCount[0]--; - if (refCount[0] <= 0) { - tokenRefMap.remove(tokenIdentifier); - jobTokenSecretManager.removeTokenForJob(tokenIdentifier); - } - } - } - - public void shutdownServer() { - if (started.get()) { // Primarily to avoid multiple shutdowns. - started.set(false); - server.stop(); - } - } - - public static class LlapUmbilicalExternalPolicyProvider extends PolicyProvider { - - private static final Service[] services = { - new Service( - MRJobConfig.MR_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL, - LlapTaskUmbilicalProtocol.class) - }; - - @Override - public Service[] getServices() { - return services.clone(); - } - } -} diff --git a/llap-common/pom.xml b/llap-common/pom.xml index 15012499451f..20689348bf95 100644 --- a/llap-common/pom.xml +++ b/llap-common/pom.xml @@ -48,18 +48,6 @@ com.google.guava guava - - io.jsonwebtoken - jjwt-api - - - io.jsonwebtoken - jjwt-impl - - - io.jsonwebtoken - jjwt-jackson - org.apache.commons commons-lang3 diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/FieldDesc.java b/llap-common/src/java/org/apache/hadoop/hive/llap/FieldDesc.java deleted file mode 100644 index 20ab13d22cf7..000000000000 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/FieldDesc.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.io.Writable; - -public class FieldDesc implements Writable { - private String name; - private TypeInfo typeInfo; - - public FieldDesc() { - } - - public FieldDesc(String name, TypeInfo typeInfo) { - this.name = name; - this.typeInfo = typeInfo; - } - - public String getName() { - return name; - } - - public TypeInfo getTypeInfo() { - return typeInfo; - } - - @Override - public String toString() { - return getName() + ":" + getTypeInfo().toString(); - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeUTF(name); - out.writeUTF(typeInfo.toString()); - } - - @Override - public void readFields(DataInput in) throws IOException { - name = in.readUTF(); - typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(in.readUTF()); - } -} \ No newline at end of file diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java index ce8e188f6e8d..c35840e649b5 100644 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java +++ b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java @@ -297,24 +297,4 @@ public static Credentials credentialsFromByteArray(byte[] binaryCredentials) credentials.readTokenStorageStream(dib); return credentials; } - - /** - * @return returns the value of LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED - * @param conf - */ - public static boolean isCloudDeployment(Configuration conf) { - return HiveConf.getBoolVar(conf, ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED, false); - } - - /** - * @return returns the value of PUBLIC_HOSTNAME from either environment variable or system properties - */ - public static String getPublicHostname() { - String publicHostname = System.getenv("PUBLIC_HOSTNAME"); - if (publicHostname == null) { - publicHostname = System.getProperty("PUBLIC_HOSTNAME"); - } - return publicHostname; - } - } diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/Row.java b/llap-common/src/java/org/apache/hadoop/hive/llap/Row.java deleted file mode 100644 index 8c0797c61db6..000000000000 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/Row.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hive.llap; - -import java.math.BigDecimal; -import java.sql.Date; -import java.sql.Timestamp; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.google.common.base.Preconditions; - -public class Row { - private final Schema schema; - private final Object[] colValues; - private Map nameToIndexMapping; - - public Row(Schema schema) { - this.schema = schema; - this.colValues = new Object[schema.getColumns().size()]; - this.nameToIndexMapping = new HashMap(schema.getColumns().size()); - - List colDescs = schema.getColumns(); - for (int idx = 0; idx < colDescs.size(); ++idx) { - FieldDesc colDesc = colDescs.get(idx); - nameToIndexMapping.put(colDesc.getName(), idx); - } - } - - public Object getValue(int colIndex) { - return colValues[colIndex]; - } - - public Object getValue(String colName) { - Integer idx = nameToIndexMapping.get(colName); - Preconditions.checkArgument(idx != null); - return getValue(idx); - } - - public Boolean getBoolean(int idx) { - return (Boolean) getValue(idx); - } - - public Boolean getBoolean(String colName) { - return (Boolean) getValue(colName); - } - - public Byte getByte(int idx) { - return (Byte) getValue(idx); - } - - public Byte getByte(String colName) { - return (Byte) getValue(colName); - } - - public Short getShort(int idx) { - return (Short) getValue(idx); - } - - public Short getShort(String colName) { - return (Short) getValue(colName); - } - - public Integer getInt(int idx) { - return (Integer) getValue(idx); - } - - public Integer getInt(String colName) { - return (Integer) getValue(colName); - } - - public Long getLong(int idx) { - return (Long) getValue(idx); - } - - public Long getLong(String colName) { - return (Long) getValue(colName); - } - - public Float getFloat(int idx) { - return (Float) getValue(idx); - } - - public Float getFloat(String colName) { - return (Float) getValue(colName); - } - - public Double getDouble(int idx) { - return (Double) getValue(idx); - } - - public Double getDouble(String colName) { - return (Double) getValue(colName); - } - - public String getString(int idx) { - return (String) getValue(idx); - } - - public String getString(String colName) { - return (String) getValue(colName); - } - - public Date getDate(int idx) { - return (Date) getValue(idx); - } - - public Date getDate(String colName) { - return (Date) getValue(colName); - } - - public Timestamp getTimestamp(int idx) { - return (Timestamp) getValue(idx); - } - - public Timestamp getTimestamp(String colName) { - return (Timestamp) getValue(colName); - } - - public byte[] getBytes(int idx) { - return (byte[]) getValue(idx); - } - - public byte[] getBytes(String colName) { - return (byte[]) getValue(colName); - } - - public BigDecimal getDecimal(int idx) { - return (BigDecimal) getValue(idx); - } - - public BigDecimal getDecimal(String colName) { - return (BigDecimal) getValue(colName); - } - - public List getList(int idx) { - return (List) getValue(idx); - } - - public List getList(String colName) { - return (List) getValue(colName); - } - - public Map getMap(int idx) { - return (Map) getValue(idx); - } - - public Map getMap(String colName) { - return (Map) getValue(colName); - } - - // Struct value is simply a list of values. - // The schema can be used to map the field name to the position in the list. - public List getStruct(int idx) { - return (List) getValue(idx); - } - - public List getStruct(String colName) { - return (List) getValue(colName); - } - - public Schema getSchema() { - return schema; - } - - void setValue(int colIdx, Object obj) { - colValues[colIdx] = obj; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("["); - for (int idx = 0; idx < schema.getColumns().size(); ++idx) { - if (idx > 0) { - sb.append(", "); - } - Object val = getValue(idx); - sb.append(val == null ? "null" : val.toString()); - } - sb.append("]"); - return sb.toString(); - } -} diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/Schema.java b/llap-common/src/java/org/apache/hadoop/hive/llap/Schema.java deleted file mode 100644 index c0d22ca97a23..000000000000 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/Schema.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import org.apache.hadoop.io.Writable; - -public class Schema implements Writable { - - private final List columns; - - public Schema(List columns) { - this.columns = columns; - } - - public Schema() { - columns = new ArrayList(); - } - - public List getColumns() { - return columns; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - boolean first = true; - for (FieldDesc colDesc : getColumns()) { - if (!first) { - sb.append(","); - } - sb.append(colDesc.toString()); - first = false; - } - return sb.toString(); - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeInt(columns.size()); - for (FieldDesc column : columns) { - column.write(out); - } - } - - @Override - public void readFields(DataInput in) throws IOException { - int numColumns = in.readInt(); - columns.clear(); - for (int idx = 0; idx < numColumns; ++idx) { - FieldDesc colDesc = new FieldDesc(); - colDesc.readFields(in); - columns.add(colDesc); - } - } -} diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/io/ChunkedInputStream.java b/llap-common/src/java/org/apache/hadoop/hive/llap/io/ChunkedInputStream.java deleted file mode 100644 index f0c7436c738c..000000000000 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/io/ChunkedInputStream.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap.io; - -import java.io.DataInputStream; -import java.io.IOException; -import java.io.InputStream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -// Data is expected to be a series of data chunks in the form -// The final data chunk should be a 0-length chunk which will indicate end of input. -public class ChunkedInputStream extends InputStream { - - static final private Logger LOG = LoggerFactory.getLogger(ChunkedInputStream.class); - - private DataInputStream din; - private int unreadBytes = 0; // Bytes remaining in the current chunk of data - private byte[] singleByte = new byte[1]; - private boolean endOfData = false; - private String id; - - public ChunkedInputStream(InputStream in, String id) { - din = new DataInputStream(in); - this.id = id; - LOG.debug("Creating chunked input for {}", id); - } - - @Override - public void close() throws IOException { - LOG.debug("{}: Closing chunked input.", id); - din.close(); - } - - @Override - public int read() throws IOException { - int bytesRead = read(singleByte, 0, 1); - return (bytesRead == -1) ? -1 : (int) singleByte[0]; - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - int bytesRead = 0; - - if (len < 0) { - throw new IllegalArgumentException(id + ": Negative read length"); - } else if (len == 0) { - return 0; - } - - // If there is a current unread chunk, read from that, or else get the next chunk. - if (unreadBytes == 0) { - try { - // Find the next chunk size - unreadBytes = din.readInt(); - if (LOG.isDebugEnabled()) { - LOG.debug("{}: Chunk size {}", id, unreadBytes); - } - if (unreadBytes == 0) { - LOG.debug("{}: Hit end of data", id); - endOfData = true; - return -1; - } - } catch (IOException err) { - throw new IOException(id + ": Error while attempting to read chunk length", err); - } - } - - int bytesToRead = Math.min(len, unreadBytes); - try { - din.readFully(b, off, bytesToRead); - } catch (IOException err) { - throw new IOException(id + ": Error while attempting to read " + bytesToRead + " bytes from current chunk", err); - } - unreadBytes -= bytesToRead; - bytesRead += bytesToRead; - - return bytesRead; - } - - public boolean isEndOfData() { - return endOfData; - } -} diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/io/ChunkedOutputStream.java b/llap-common/src/java/org/apache/hadoop/hive/llap/io/ChunkedOutputStream.java deleted file mode 100644 index 124a90184262..000000000000 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/io/ChunkedOutputStream.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap.io; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.OutputStream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -// Writes data out as a series of chunks in the form -// Closing the output stream will send a final 0-length chunk which will indicate end of input. -public class ChunkedOutputStream extends OutputStream { - - static final private Logger LOG = LoggerFactory.getLogger(ChunkedOutputStream.class); - - private DataOutputStream dout; - private byte[] singleByte = new byte[1]; - private byte[] buffer; - private int bufPos = 0; - private String id; - - public ChunkedOutputStream(OutputStream out, int bufSize, String id) { - LOG.debug("Creating chunked input stream: {}", id); - if (bufSize <= 0) { - throw new IllegalArgumentException("Positive bufSize required, was " + bufSize); - } - buffer = new byte[bufSize]; - dout = new DataOutputStream(out); - this.id = id; - } - - @Override - public void write(int b) throws IOException { - singleByte[0] = (byte) b; - write(singleByte, 0, 1); - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - int bytesWritten = 0; - while (bytesWritten < len) { - // Copy the data to the buffer - int bytesToWrite = Math.min(len - bytesWritten, buffer.length - bufPos); - System.arraycopy(b, off + bytesWritten, buffer, bufPos, bytesToWrite); - bytesWritten += bytesToWrite; - bufPos += bytesToWrite; - - // If we've filled the buffer, write it out - if (bufPos == buffer.length) { - writeChunk(); - } - } - } - - @Override - public void close() throws IOException { - flush(); - - // Write final 0-length chunk - writeChunk(); - - LOG.debug("{}: Closing underlying output stream.", id); - dout.close(); - } - - @Override - public void flush() throws IOException { - // Write any remaining bytes to the out stream. - if (bufPos > 0) { - writeChunk(); - dout.flush(); - } - } - - private void writeChunk() throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("{}: Writing chunk of size {}", id, bufPos); - } - - // First write chunk length - dout.writeInt(bufPos); - - // Then write chunk bytes - dout.write(buffer, 0, bufPos); - - bufPos = 0; // reset buffer - } -} diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/security/DefaultJwtSharedSecretProvider.java b/llap-common/src/java/org/apache/hadoop/hive/llap/security/DefaultJwtSharedSecretProvider.java deleted file mode 100644 index 1e04c2a58bfd..000000000000 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/security/DefaultJwtSharedSecretProvider.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap.security; - -import com.google.common.base.Preconditions; -import io.jsonwebtoken.security.Keys; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.StandardCharsets; -import java.security.Key; - -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET; - -/** - * Default implementation of {@link JwtSecretProvider}. - * - * 1. It first tries to get shared secret from conf {@link HiveConf.ConfVars#LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET} - * using {@link Configuration#getPassword(String)}. - * - * 2. If not found, it tries to read from env var {@link #LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR}. - * - * If secret is not found even after 1) and 2), {@link #init(Configuration)} methods throws {@link IllegalStateException}. - * - * Length of shared secret provided in 1) or 2) should be > 32 bytes. - * - * It uses the same encryption and decryption secret which can be used to sign and verify JWT. - */ -public class DefaultJwtSharedSecretProvider implements JwtSecretProvider { - - public static final String LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR = - "LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR"; - - private Key jwtEncryptionKey; - - @Override public Key getEncryptionSecret() { - return jwtEncryptionKey; - } - - @Override public Key getDecryptionSecret() { - return jwtEncryptionKey; - } - - @Override public void init(final Configuration conf) { - char[] sharedSecret; - byte[] sharedSecretBytes = null; - - // try getting secret from conf first - // if not found, get from env var - LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR - try { - sharedSecret = conf.getPassword(LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET.varname); - } catch (IOException e) { - throw new RuntimeException("Unable to get password [hive.llap.external.client.cloud.jwt.shared.secret] - " - + e.getMessage(), e); - } - if (sharedSecret != null) { - ByteBuffer bb = StandardCharsets.UTF_8.encode(CharBuffer.wrap(sharedSecret)); - sharedSecretBytes = new byte[bb.remaining()]; - bb.get(sharedSecretBytes); - } else { - String sharedSecredFromEnv = System.getenv(LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR); - if (StringUtils.isNotBlank(sharedSecredFromEnv)) { - sharedSecretBytes = sharedSecredFromEnv.getBytes(); - } - } - - Preconditions.checkState(sharedSecretBytes != null, - "With: " + LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED.varname + " = true, \n" - + "To use: org.apache.hadoop.hive.llap.security.DefaultJwtSharedSecretProvider, \n" - + "1. a non-null value of 'hive.llap.external.client.cloud.jwt.shared.secret' must be provided OR \n" - + "2. alternatively environment variable " - + "LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR can also be set. \n" - + "Length of the secret provided in 1) or 2) should be > 32 bytes."); - - this.jwtEncryptionKey = Keys.hmacShaKeyFor(sharedSecretBytes); - } - -} diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/security/JwtSecretProvider.java b/llap-common/src/java/org/apache/hadoop/hive/llap/security/JwtSecretProvider.java deleted file mode 100644 index c401f182c64b..000000000000 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/security/JwtSecretProvider.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap.security; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; - -import java.security.Key; - -/** - * JwtSecretProvider - * - * - provides encryption and decryption secrets for generating and parsing JWTs. - * - * - Hive internally uses method initAndGet() which initializes providers based on the value of config - * {@link HiveConf.ConfVars#LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_PROVIDER}. - * It expects implementations to provide default constructor and {@link #init(Configuration)} method. - */ -public interface JwtSecretProvider { - - /** - * returns secret for signing JWT. - */ - Key getEncryptionSecret(); - - /** - * returns secret for parsing JWT. - */ - Key getDecryptionSecret(); - - /** - * Initializes the provider. - * Should also contain any validations that we want to put on secret, helps us to fail fast. - * @param conf configuration - */ - void init(Configuration conf); - - /** - * Hive internally uses this method to obtain instance of {@link JwtSecretProvider} - * - * @param conf configuration - * @return implementation of {@link HiveConf.ConfVars#LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_PROVIDER} - */ - static JwtSecretProvider initAndGet(Configuration conf) { - final String providerClass = - HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_PROVIDER); - JwtSecretProvider provider; - try { - provider = (JwtSecretProvider) Class.forName(providerClass).newInstance(); - provider.init(conf); - } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { - throw new RuntimeException("Unable to instantiate provider: " + providerClass, e); - } - return provider; - } -} diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapExtClientJwtHelper.java b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapExtClientJwtHelper.java deleted file mode 100644 index 55500546a9df..000000000000 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapExtClientJwtHelper.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap.security; - -import io.jsonwebtoken.Claims; -import io.jsonwebtoken.Jws; -import io.jsonwebtoken.Jwts; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.records.ApplicationId; - -import java.util.Date; -import java.util.UUID; - -/** - * Contains helper methods for generating and verifying JWTs for external llap clients. - * Initializes and uses {@link JwtSecretProvider} to obtain encryption and decryption secret. - */ -public class LlapExtClientJwtHelper { - - public static final String LLAP_JWT_SUBJECT = "llap"; - public static final String LLAP_EXT_CLIENT_APP_ID = "llap_ext_client_app_id"; - private final JwtSecretProvider jwtSecretProvider; - - public LlapExtClientJwtHelper(Configuration conf) { - this.jwtSecretProvider = JwtSecretProvider.initAndGet(conf); - } - - /** - * @param extClientAppId application Id - application Id injected by get_splits - * @return JWT signed with {@link JwtSecretProvider#getEncryptionSecret()}. - * As of now this JWT contains extClientAppId in claims. - */ - public String buildJwtForLlap(ApplicationId extClientAppId) { - return Jwts.builder() - .setSubject(LLAP_JWT_SUBJECT) - .setIssuedAt(new Date()) - .setId(UUID.randomUUID().toString()) - .claim(LLAP_EXT_CLIENT_APP_ID, extClientAppId.toString()) - .signWith(jwtSecretProvider.getEncryptionSecret()) - .compact(); - } - - /** - * - * @param jwt signed JWT String - * @return claims present in JWT, this method parses jwt using {@link JwtSecretProvider#getDecryptionSecret()} - */ - public Jws parseClaims(String jwt) { - return Jwts.parser() - .setSigningKey(jwtSecretProvider.getDecryptionSecret()) - .parseClaimsJws(jwt); - } - -} diff --git a/llap-common/src/protobuf/LlapDaemonProtocol.proto b/llap-common/src/protobuf/LlapDaemonProtocol.proto index 8b15f4392eb5..6e86fee31a8e 100644 --- a/llap-common/src/protobuf/LlapDaemonProtocol.proto +++ b/llap-common/src/protobuf/LlapDaemonProtocol.proto @@ -132,8 +132,10 @@ message SubmitWorkRequestProto { optional bytes initial_event_signature = 11; optional bool is_guaranteed = 12 [default = false]; - optional string jwt = 13; - optional bool is_external_client_request = 14 [default = false]; + + // HIVE-28932: We decommissioned the external client support + reserved 13, 14; + reserved "jwt", "is_external_client_request"; } message RegisterDagRequestProto { @@ -200,12 +202,6 @@ message GetTokenResponseProto { optional bytes token = 1; } -// The message sent by external client to claim the output from the output socket. -message LlapOutputSocketInitMessage { - required string fragment_id = 1; - optional bytes token = 2; -} - message PurgeCacheRequestProto { } diff --git a/llap-common/src/test/org/apache/hadoop/hive/llap/TestRow.java b/llap-common/src/test/org/apache/hadoop/hive/llap/TestRow.java deleted file mode 100644 index 9520b07f6ff6..000000000000 --- a/llap-common/src/test/org/apache/hadoop/hive/llap/TestRow.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap; - -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -import org.apache.commons.lang3.RandomStringUtils; - -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; - -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import static org.junit.Assert.*; - -public class TestRow { - - @Test - public void testUsage() { - Schema schema = createTestSchema(); - Row row = new Row(schema); - - Random rand = new Random(); - int iterations = 100; - for (int idx = 0; idx < iterations; ++idx) { - // Set the row values - boolean isNullCol0 = (rand.nextDouble() <= 0.25); - String col0 = RandomStringUtils.random(10); - row.setValue(0, isNullCol0 ? null : col0); - - boolean isNullCol1 = (rand.nextDouble() <= 0.25); - Integer col1 = Integer.valueOf(rand.nextInt()); - row.setValue(1, isNullCol1 ? null : col1); - - // Validate the row values - if (isNullCol0) { - assertTrue(row.getValue(0) == null); - assertTrue(row.getValue("col0") == null); - } else { - assertTrue(row.getValue(0) != null); - assertEquals(col0, row.getValue(0)); - assertEquals(col0, row.getValue("col0")); - } - - if (isNullCol1) { - assertTrue(row.getValue(1) == null); - assertTrue(row.getValue("col1") == null); - } else { - assertTrue(row.getValue(1) != null); - assertEquals(col1, row.getValue(1)); - assertEquals(col1, row.getValue("col1")); - } - } - } - - private Schema createTestSchema() { - List colDescs = new ArrayList(); - - colDescs.add(new FieldDesc("col0", - TypeInfoFactory.stringTypeInfo)); - - colDescs.add(new FieldDesc("col1", - TypeInfoFactory.intTypeInfo)); - - Schema schema = new Schema(colDescs); - return schema; - } -} \ No newline at end of file diff --git a/llap-common/src/test/org/apache/hadoop/hive/llap/io/TestChunkedInputStream.java b/llap-common/src/test/org/apache/hadoop/hive/llap/io/TestChunkedInputStream.java deleted file mode 100644 index b90a2d57e88d..000000000000 --- a/llap-common/src/test/org/apache/hadoop/hive/llap/io/TestChunkedInputStream.java +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap.io; - -import java.io.FilterInputStream; -import java.io.FilterOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.PipedInputStream; -import java.io.PipedOutputStream; -import java.util.Arrays; -import java.util.List; -import java.util.Random; - -import org.apache.hadoop.hive.serde2.RandomTypeUtil; -import org.junit.Test; -import static org.junit.Assert.*; - -public class TestChunkedInputStream { - - static int bufferSize = 128; - static Random rand = new Random(); - static String alphabet = "abcdefghijklmnopqrstuvwxyz"; - - static class StreamTester { - Exception error = null; - - public Exception getError() { - return error; - } - - public void setError(Exception error) { - this.error = error; - } - } - - // Test class to write a series of values to the designated output stream - static class BasicUsageWriter extends StreamTester implements Runnable { - TestStreams streams; - boolean flushCout; - boolean closePoutEarly; - - public BasicUsageWriter(TestStreams streams, boolean flushCout, boolean closePoutEarly) { - this.streams = streams; - this.flushCout = flushCout; - this.closePoutEarly = closePoutEarly; - } - - @Override - public void run() { - try { - // Write the items to the output stream. - for (byte[] value: streams.values) { - streams.out.write(value, 0, value.length); - } - - if (flushCout) { - streams.out.flush(); - } - if (closePoutEarly) { - // Close the inner output stream before closing the outer output stream. - // For chunked output this means we don't write end-of-data indicator. - streams.pout.close(); - } - // This will throw error if we close pout early. - streams.out.close(); - } catch (Exception err) { - err.printStackTrace(); - this.error = err; - } - } - } - - // Test class to read a series of values to the designated input stream - static class BasicUsageReader extends StreamTester implements Runnable { - TestStreams streams; - boolean allValuesRead = false; - - public BasicUsageReader(TestStreams streams) { - this.streams = streams; - } - - // Continue reading from the input stream until the desired number of byte has been read - void readFully(InputStream in, byte[] readValue, int numBytes) throws IOException { - int bytesRead = 0; - while (bytesRead < numBytes) { - int read = in.read(readValue, bytesRead, numBytes - bytesRead); - if (read <= 0) { - throw new IOException("Unexpected read length " + read); - } - bytesRead += read; - } - } - - @Override - public void run() { - try { - // Read the items from the input stream and confirm they match - for (byte[] value : streams.values) { - byte[] readValue = new byte[value.length]; - readFully(streams.in, readValue, readValue.length); - assertArrayEquals(value, readValue); - } - - allValuesRead = true; - - // Check that the output is done - assertEquals(-1, streams.in.read()); - } catch (Exception err) { - err.printStackTrace(); - this.error = err; - } - } - } - - static class MyFilterInputStream extends FilterInputStream { - public MyFilterInputStream(InputStream in) { - super(in); - } - } - - // Helper class to set up a ChunkedInput/Output stream for testing - static class TestStreams { - PipedOutputStream pout; - OutputStream out; - PipedInputStream pin; - InputStream in; - List values; - - public TestStreams(boolean useChunkedStream) throws Exception { - pout = new PipedOutputStream(); - pin = new PipedInputStream(pout); - if (useChunkedStream) { - out = new ChunkedOutputStream(pout, bufferSize, "test"); - in = new ChunkedInputStream(pin, "test"); - } else { - // Test behavior with non-chunked streams - out = new FilterOutputStream(pout); - in = new MyFilterInputStream(pin); - } - } - - public void close() { - try { - pout.close(); - } catch (Exception err) { - // ignore - } - - try { - pin.close(); - } catch (Exception err) { - // ignore - } - } - } - - static void runTest(Runnable writer, Runnable reader, TestStreams streams) throws Exception { - Thread writerThread = new Thread(writer); - Thread readerThread = new Thread(reader); - - writerThread.start(); - readerThread.start(); - - writerThread.join(); - readerThread.join(); - } - - @Test - public void testBasicUsage() throws Exception { - List values = Arrays.asList( - new byte[]{(byte) 1}, - new byte[]{(byte) 2}, - RandomTypeUtil.getRandString(rand, alphabet, 99).getBytes(), - RandomTypeUtil.getRandString(rand, alphabet, 1024).getBytes() - ); - - // Try the basic test with non-chunked stream - TestStreams nonChunkedStreams = new TestStreams(false); - nonChunkedStreams.values = values; - BasicUsageWriter writer1 = new BasicUsageWriter(nonChunkedStreams, false, false); - BasicUsageReader reader1 = new BasicUsageReader(nonChunkedStreams); - runTest(writer1, reader1, nonChunkedStreams); - assertTrue(reader1.allValuesRead); - assertNull(writer1.getError()); - assertNull(reader1.getError()); - - // Try with chunked streams - TestStreams chunkedStreams = new TestStreams(true); - chunkedStreams.values = values; - BasicUsageWriter writer2 = new BasicUsageWriter(chunkedStreams, false, false); - BasicUsageReader reader2 = new BasicUsageReader(chunkedStreams); - runTest(writer2, reader2, chunkedStreams); - assertTrue(reader2.allValuesRead); - assertTrue(((ChunkedInputStream) chunkedStreams.in).isEndOfData()); - assertNull(writer2.getError()); - assertNull(reader2.getError()); - } - - @Test - public void testAbruptlyClosedOutput() throws Exception { - List values = Arrays.asList( - new byte[]{(byte) 1}, - new byte[]{(byte) 2}, - RandomTypeUtil.getRandString(rand, alphabet, 99).getBytes(), - RandomTypeUtil.getRandString(rand, alphabet, 1024).getBytes() - ); - - // Close the PipedOutputStream before we close the outermost OutputStream. - - // Try non-chunked stream. There should be no issues assuming we flushed the streams before closing. - TestStreams nonChunkedStreams = new TestStreams(false); - nonChunkedStreams.values = values; - BasicUsageWriter writer1 = new BasicUsageWriter(nonChunkedStreams, true, true); - BasicUsageReader reader1 = new BasicUsageReader(nonChunkedStreams); - runTest(writer1, reader1, nonChunkedStreams); - assertTrue(reader1.allValuesRead); - assertNull(writer1.getError()); - assertNull(reader1.getError()); - - // Try with chunked stream. Here the chunked output didn't get a chance to write the end-of-data - // indicator, so the chunked input does not know to stop reading. - TestStreams chunkedStreams = new TestStreams(true); - chunkedStreams.values = values; - BasicUsageWriter writer2 = new BasicUsageWriter(chunkedStreams, true, true); - BasicUsageReader reader2 = new BasicUsageReader(chunkedStreams); - runTest(writer2, reader2, chunkedStreams); - assertTrue(reader2.allValuesRead); - assertFalse(((ChunkedInputStream) chunkedStreams.in).isEndOfData()); - // Closing the chunked output stream early gives an error - assertNotNull(writer2.getError()); - // In this case we should expect the test to have failed at the very last read() check. - assertNotNull(reader2.getError()); - } -} diff --git a/llap-ext-client/pom.xml b/llap-ext-client/pom.xml deleted file mode 100644 index 95d1fa224715..000000000000 --- a/llap-ext-client/pom.xml +++ /dev/null @@ -1,198 +0,0 @@ - - - - 4.0.0 - - org.apache.hive - hive - 4.3.0-SNAPSHOT - ../pom.xml - - hive-llap-ext-client - jar - Hive Llap External Client - - .. - - - - - - org.apache.hive - hive-exec - ${project.version} - - - org.apache.hive - hive-llap-client - ${project.version} - - - - org.apache.hadoop - hadoop-common - true - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-reload4j - - - ch.qos.reload4j - reload4j - - - commons-beanutils - commons-beanutils - - - commons-logging - commons-logging - - - - - org.apache.hadoop - hadoop-mapreduce-client-core - true - - - org.apache.hadoop - hadoop-yarn-registry - true - - - org.apache.tez - tez-api - true - - - org.slf4j - slf4j-log4j12 - - - commons-logging - commons-logging - - - com.sun.jersey - jersey-json - - - com.sun.jersey - jersey-client - - - - - org.apache.tez - tez-runtime-internals - true - - - org.slf4j - slf4j-log4j12 - - - commons-logging - commons-logging - - - - - - junit - junit - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - org.junit.vintage - junit-vintage-engine - test - - - org.apache.commons - commons-lang3 - ${commons-lang3.version} - test - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - tests - test - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-reload4j - - - ch.qos.reload4j - reload4j - - - commons-beanutils - commons-beanutils - - - commons-logging - commons-logging - - - - - - ${basedir}/src/java - ${basedir}/src/test - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-source - generate-sources - - add-source - - - - src/gen/thrift/gen-javabean - - - - - - - - diff --git a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java deleted file mode 100644 index 7ab8032889db..000000000000 --- a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java +++ /dev/null @@ -1,552 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hive.llap; - -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.nio.ByteBuffer; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.UUID; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.regex.Pattern; - -import org.apache.commons.collections4.ListUtils; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.llap.LlapBaseRecordReader.ReaderEvent; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentRuntimeInfo; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.LlapOutputSocketInitMessage; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SignableVertexSpec; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.VertexOrBinary; -import org.apache.hadoop.hive.llap.ext.LlapDaemonInfo; -import org.apache.hadoop.hive.llap.ext.LlapTaskUmbilicalExternalClient; -import org.apache.hadoop.hive.llap.ext.LlapTaskUmbilicalExternalClient.LlapTaskUmbilicalExternalResponder; -import org.apache.hadoop.hive.llap.registry.LlapServiceInstance; -import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier; -import org.apache.hadoop.hive.llap.tez.Converters; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.DataInputBuffer; -import org.apache.hadoop.io.DataOutputBuffer; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.mapred.InputSplit; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.RecordReader; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.TaskAttemptID; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hive.common.util.ShutdownHookManager; -import org.apache.tez.common.security.JobTokenIdentifier; -import org.apache.tez.common.security.TokenCache; -import org.apache.tez.dag.records.TezTaskAttemptID; -import org.apache.tez.runtime.api.events.TaskAttemptFailedEvent; -import org.apache.tez.runtime.api.impl.EventType; -import org.apache.tez.runtime.api.impl.TezEvent; -import org.apache.tez.runtime.api.impl.TezHeartbeatRequest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.protobuf.ByteString; - - -/** - * Base LLAP input format to handle requesting of splits and communication with LLAP daemon. - */ -public class LlapBaseInputFormat> - implements InputFormat { - - private static final Logger LOG = LoggerFactory.getLogger(LlapBaseInputFormat.class); - - private static final Object lock = new Object(); - private static final Map> connectionMap = new HashMap>(); - - private String url; // "jdbc:hive2://localhost:10000/default" - private String user; // "hive", - private String pwd; // "" - private String query; - private final Random rand = new Random(); - - public static final String URL_KEY = "llap.if.hs2.connection"; - public static final String QUERY_KEY = "llap.if.query"; - public static final String USER_KEY = "llap.if.user"; - public static final String PWD_KEY = "llap.if.pwd"; - public static final String HANDLE_ID = "llap.if.handleid"; - public static final String DB_KEY = "llap.if.database"; - public static final String USE_NEW_SPLIT_FORMAT = "llap.if.use.new.split.format"; - public static final String SESSION_QUERIES_FOR_GET_NUM_SPLITS = "llap.session.queries.for.get.num.splits"; - public static final Pattern SET_QUERY_PATTERN = Pattern.compile("^\\s*set\\s+.*=.+$", Pattern.CASE_INSENSITIVE); - - public static final String SPLIT_QUERY = "select get_llap_splits(\"%s\",%d)"; - - @SuppressWarnings("unchecked") - @Override - public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException { - - LlapInputSplit llapSplit = (LlapInputSplit) split; - - // Set conf to use LLAP user rather than current user for LLAP Zk registry. - HiveConf.setVar(job, HiveConf.ConfVars.LLAP_ZK_REGISTRY_USER, llapSplit.getLlapUser()); - SubmitWorkInfo submitWorkInfo = SubmitWorkInfo.fromBytes(llapSplit.getPlanBytes()); - - // llapSplit.getLlapDaemonInfos() will never be empty as of now, also validated this in GenericUDTFGetSplits while populating. - final LlapDaemonInfo llapDaemonInfo = llapSplit.getLlapDaemonInfos()[0]; - final String host = llapDaemonInfo.getHost(); - final int outputPort = llapDaemonInfo.getOutputFormatPort(); - final int llapSubmitPort = llapDaemonInfo.getRpcPort(); - - LOG.info("Will try to submit request to first Llap Daemon in the split - {}", llapDaemonInfo); - - byte[] llapTokenBytes = llapSplit.getTokenBytes(); - Token llapToken = null; - if (llapTokenBytes != null) { - DataInputBuffer in = new DataInputBuffer(); - in.reset(llapTokenBytes, 0, llapTokenBytes.length); - llapToken = new Token(); - llapToken.readFields(in); - } - - LlapRecordReaderTaskUmbilicalExternalResponder umbilicalResponder = - new LlapRecordReaderTaskUmbilicalExternalResponder(); - LlapTaskUmbilicalExternalClient llapClient = - new LlapTaskUmbilicalExternalClient(job, submitWorkInfo.getTokenIdentifier(), - submitWorkInfo.getToken(), umbilicalResponder, llapToken); - - int attemptNum = 0; - final int taskNum; - // Use task attempt number, task number from conf if provided - TaskAttemptID taskAttemptId = TaskAttemptID.forName(job.get(MRJobConfig.TASK_ATTEMPT_ID)); - if (taskAttemptId != null) { - attemptNum = taskAttemptId.getId(); - taskNum = taskAttemptId.getTaskID().getId(); - if (LOG.isDebugEnabled()) { - LOG.debug("Setting attempt number to: {}, task number to: {} from given taskAttemptId: {} in conf", - attemptNum, taskNum, taskAttemptId); - } - } else { - taskNum = llapSplit.getSplitNum(); - } - - SubmitWorkRequestProto request = constructSubmitWorkRequestProto( - submitWorkInfo, taskNum, attemptNum, llapClient.getAddress(), - submitWorkInfo.getToken(), llapSplit, job); - - SignableVertexSpec vertex = SignableVertexSpec.parseFrom(submitWorkInfo.getVertexBinary()); - String fragmentId = - Converters.createTaskAttemptId(vertex.getQueryIdentifier(), vertex.getVertexIndex(), - request.getFragmentNumber(), request.getAttemptNumber()).toString(); - - LOG.info("Submitting fragment:{} to llap [host = {}, port = {}] ", fragmentId, host, llapSubmitPort); - - llapClient.submitWork(request, host, llapSubmitPort); - - Socket socket = new Socket(host, outputPort); - - OutputStream socketStream = socket.getOutputStream(); - LlapOutputSocketInitMessage.Builder builder = - LlapOutputSocketInitMessage.newBuilder().setFragmentId(fragmentId); - if (llapSplit.getTokenBytes() != null) { - builder.setToken(ByteString.copyFrom(llapSplit.getTokenBytes())); - } - - LOG.info("Registering fragment:{} to llap [host = {}, output port = {}] to read output", - fragmentId, host, outputPort); - builder.build().writeDelimitedTo(socketStream); - socketStream.flush(); - - LOG.info("Registered id: " + fragmentId); - - @SuppressWarnings("rawtypes") - LlapBaseRecordReader recordReader = new LlapBaseRecordReader(socket.getInputStream(), - llapSplit.getSchema(), BytesWritable.class, job, llapClient, socket); - umbilicalResponder.setRecordReader(recordReader); - return recordReader; - } - - /** - * Calling getSplits() will open a HiveServer2 connection which should be closed by the calling application - * using LlapBaseInputFormat.close() when the application is done with the splits. - */ - @Override - public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { - List ins = new ArrayList(); - - if (url == null) url = job.get(URL_KEY); - if (query == null) query = job.get(QUERY_KEY); - if (user == null) user = job.get(USER_KEY); - if (pwd == null) pwd = job.get(PWD_KEY); - String database = job.get(DB_KEY); - - if (url == null || query == null) { - throw new IllegalStateException(); - } - - String handleId = job.get(HANDLE_ID); - if (handleId == null) { - handleId = UUID.randomUUID().toString(); - LOG.info("Handle ID not specified - generated handle ID {}", handleId); - } - - LOG.info("Handle ID {}: query={}", handleId, query); - String escapedQuery = StringUtils.escapeString(query, ESCAPE_CHAR, escapedChars); - String sql = String.format(SPLIT_QUERY, escapedQuery, numSplits); - try { - Connection conn = DriverManager.getConnection(url,user,pwd); - try ( - Statement stmt = conn.createStatement(); - ) { - if (database != null && !database.isEmpty()) { - stmt.execute("USE " + database); - } - String sessionQueries = job.get(SESSION_QUERIES_FOR_GET_NUM_SPLITS); - if (sessionQueries != null && !sessionQueries.trim().isEmpty()) { - String[] queries = sessionQueries.trim().split(","); - for (String q : queries) { - //allow only set queries - if (SET_QUERY_PATTERN.matcher(q).matches()) { - LOG.debug("Executing session query: {}", q); - stmt.execute(q); - } else { - LOG.warn("Only SET queries are allowed, not executing this query: {}", q); - } - } - } - - // In case of USE_NEW_SPLIT_FORMAT=true, following format is used - // type split - // schema-split LlapInputSplit -- contains only schema - // plan-split LlapInputSplit -- contains only planBytes[] - // 0 LlapInputSplit -- actual split 1 - // 1 LlapInputSplit -- actual split 2 - // ... ... - boolean useNewSplitFormat = job.getBoolean(USE_NEW_SPLIT_FORMAT, false); - - ResultSet res = stmt.executeQuery(sql); - int count = 0; - LlapInputSplit schemaSplit = null; - LlapInputSplit planSplit = null; - while (res.next()) { - // deserialize split - DataInput in = new DataInputStream(res.getBinaryStream(2)); - LlapInputSplit is = new LlapInputSplit(); - is.readFields(in); - if (useNewSplitFormat) { - ins.add(is); - } else { - // to keep the old format, populate schema and planBytes[] in actual splits - if (count == 0) { - schemaSplit = is; - if (numSplits == 0) { - ins.add(schemaSplit); - } - } else if (count == 1) { - planSplit = is; - } else { - is.setSchema(schemaSplit.getSchema()); - assert planSplit != null; - is.setPlanBytes(planSplit.getPlanBytes()); - ins.add(is); - } - count++; - } - } - res.close(); - } catch (Exception e) { - LOG.error("Closing connection due to error", e); - conn.close(); - throw e; - } - - // Keep connection open to hang on to associated resources (temp tables, locks). - // Save to connectionMap so it can be closed at user's convenience. - addConnection(handleId, conn); - } catch (Exception e) { - throw new IOException(e); - } - return ins.toArray(new InputSplit[ins.size()]); - } - - private void addConnection(String handleId, Connection connection) { - synchronized (lock) { - List handleConnections = connectionMap.get(handleId); - if (handleConnections == null) { - handleConnections = new ArrayList(); - connectionMap.put(handleId, handleConnections); - } - handleConnections.add(connection); - } - } - - /** - * Close the connection associated with the handle ID, if getSplits() was configured with a handle ID. - * Call when the application is done using the splits generated by getSplits(). - * @param handleId Handle ID used in configuration for getSplits() - * @throws IOException - */ - public static void close(String handleId) throws IOException { - List handleConnections; - synchronized (lock) { - handleConnections = connectionMap.remove(handleId); - } - closeConnections(handleId, handleConnections); - } - - private static void closeConnections(String handleId, List handleConnections) { - if (handleConnections != null) { - LOG.debug("Closing {} connections for handle ID {}", handleConnections.size(), handleId); - for (Connection conn : handleConnections) { - try { - conn.close(); - } catch (Exception err) { - LOG.error("Error while closing connection for " + handleId, err); - } - } - } else { - LOG.debug("No connection found for handle ID {}", handleId); - } - } - - /** - * Close all outstanding connections created by getSplits() calls - */ - public static void closeAll() { - LOG.debug("Closing all handles"); - synchronized (lock) { - Iterator>> itr = connectionMap.entrySet().iterator(); - Map.Entry> connHandle = null; - while (itr.hasNext()) { - connHandle = itr.next(); - closeConnections(connHandle.getKey(), connHandle.getValue()); - itr.remove(); - } - } - } - - static { - // Shutdown hook to clean up resources at process end. - ShutdownHookManager.addShutdownHook(new Runnable() { - @Override - public void run() { - closeAll(); - } - }); - } - - private SubmitWorkRequestProto constructSubmitWorkRequestProto(SubmitWorkInfo submitWorkInfo, - int taskNum, int attemptNum, InetSocketAddress address, Token token, - LlapInputSplit llapInputSplit, JobConf job) throws IOException { - byte[] fragmentBytes = llapInputSplit.getFragmentBytes(); - byte[] fragmentBytesSignature = llapInputSplit.getFragmentBytesSignature(); - - ApplicationId appId = submitWorkInfo.getFakeAppId(); - - // This works, assuming the executor is running within YARN. - String user = System.getenv(ApplicationConstants.Environment.USER.name()); - LOG.info("Setting user in submitWorkRequest to: " + user); - - ContainerId containerId = - ContainerId.newInstance(ApplicationAttemptId.newInstance(appId, attemptNum), taskNum); - - // Credentials can change across DAGs. Ideally construct only once per DAG. - Credentials credentials = new Credentials(); - TokenCache.setSessionToken(token, credentials); - ByteBuffer credentialsBinary = serializeCredentials(credentials); - - FragmentRuntimeInfo.Builder runtimeInfo = FragmentRuntimeInfo.newBuilder(); - runtimeInfo.setCurrentAttemptStartTime(System.currentTimeMillis()); - runtimeInfo.setWithinDagPriority(0); - runtimeInfo.setDagStartTime(submitWorkInfo.getCreationTime()); - runtimeInfo.setFirstAttemptStartTime(submitWorkInfo.getCreationTime()); - runtimeInfo.setNumSelfAndUpstreamTasks(submitWorkInfo.getVertexParallelism()); - runtimeInfo.setNumSelfAndUpstreamCompletedTasks(0); - - SubmitWorkRequestProto.Builder builder = SubmitWorkRequestProto.newBuilder(); - - VertexOrBinary.Builder vertexBuilder = VertexOrBinary.newBuilder(); - vertexBuilder.setVertexBinary(ByteString.copyFrom(submitWorkInfo.getVertexBinary())); - if (submitWorkInfo.getVertexSignature() != null) { - // Unsecure case? - builder.setWorkSpecSignature(ByteString.copyFrom(submitWorkInfo.getVertexSignature())); - } - builder.setWorkSpec(vertexBuilder.build()); - builder.setFragmentNumber(taskNum); - builder.setAttemptNumber(attemptNum); - builder.setContainerIdString(containerId.toString()); - builder.setAmHost(LlapUtil.getAmHostNameFromAddress(address, job)); - builder.setAmPort(address.getPort()); - builder.setCredentialsBinary(ByteString.copyFrom(credentialsBinary)); - builder.setFragmentRuntimeInfo(runtimeInfo.build()); - builder.setInitialEventBytes(ByteString.copyFrom(fragmentBytes)); - if (fragmentBytesSignature != null) { - builder.setInitialEventSignature(ByteString.copyFrom(fragmentBytesSignature)); - } - builder.setJwt(llapInputSplit.getJwt()); - builder.setIsExternalClientRequest(true); - return builder.build(); - } - - private ByteBuffer serializeCredentials(Credentials credentials) throws IOException { - Credentials containerCredentials = new Credentials(); - containerCredentials.addAll(credentials); - DataOutputBuffer containerTokens_dob = new DataOutputBuffer(); - containerCredentials.writeTokenStorageToStream(containerTokens_dob); - return ByteBuffer.wrap(containerTokens_dob.getData(), 0, containerTokens_dob.getLength()); - } - - private static final char ESCAPE_CHAR = '\\'; - - private static final char[] escapedChars = { - '"', ESCAPE_CHAR - }; - - private static class LlapRecordReaderTaskUmbilicalExternalResponder implements LlapTaskUmbilicalExternalResponder { - protected LlapBaseRecordReader recordReader = null; - protected LinkedBlockingQueue queuedEvents = new LinkedBlockingQueue(); - - public LlapRecordReaderTaskUmbilicalExternalResponder() { - } - - @Override - public void submissionFailed(String fragmentId, Throwable throwable) { - try { - sendOrQueueEvent(ReaderEvent.errorEvent( - "Received submission failed event for fragment ID " + fragmentId + ": " + throwable.toString())); - } catch (Exception err) { - LOG.error("Error during heartbeat responder:", err); - } - } - - @Override - public void heartbeat(TezHeartbeatRequest request) { - List inEvents = request.getEvents(); - for (TezEvent tezEvent : ListUtils.emptyIfNull(inEvents)) { - EventType eventType = tezEvent.getEventType(); - try { - switch (eventType) { - case TASK_ATTEMPT_COMPLETED_EVENT: - sendOrQueueEvent(ReaderEvent.doneEvent()); - break; - case TASK_ATTEMPT_FAILED_EVENT: - TaskAttemptFailedEvent taskFailedEvent = (TaskAttemptFailedEvent) tezEvent.getEvent(); - sendOrQueueEvent(ReaderEvent.errorEvent(taskFailedEvent.getDiagnostics())); - break; - case TASK_STATUS_UPDATE_EVENT: - // If we want to handle counters - break; - default: - LOG.warn("Unhandled event type " + eventType); - break; - } - } catch (Exception err) { - LOG.error("Error during heartbeat responder:", err); - } - } - } - - @Override - public void taskKilled(TezTaskAttemptID taskAttemptId) { - try { - sendOrQueueEvent(ReaderEvent.errorEvent( - "Received task killed event for task ID " + taskAttemptId)); - } catch (Exception err) { - LOG.error("Error during heartbeat responder:", err); - } - } - - @Override - public void heartbeatTimeout(String taskAttemptId) { - try { - sendOrQueueEvent(ReaderEvent.errorEvent( - "Timed out waiting for heartbeat for task ID " + taskAttemptId)); - } catch (Exception err) { - LOG.error("Error during heartbeat responder:", err); - } - } - - public synchronized LlapBaseRecordReader getRecordReader() { - return recordReader; - } - - public synchronized void setRecordReader(LlapBaseRecordReader recordReader) { - this.recordReader = recordReader; - - if (recordReader == null) { - return; - } - - // If any events were queued by the responder, give them to the record reader now. - while (!queuedEvents.isEmpty()) { - ReaderEvent readerEvent = queuedEvents.poll(); - LOG.debug("Sending queued event to record reader: " + readerEvent.getEventType()); - recordReader.handleEvent(readerEvent); - } - } - - /** - * Send the ReaderEvents to the record reader, if it is registered to this responder. - * If there is no registered record reader, add them to a list of pending reader events - * since we don't want to drop these events. - * @param readerEvent - */ - protected synchronized void sendOrQueueEvent(ReaderEvent readerEvent) { - LlapBaseRecordReader recordReader = getRecordReader(); - if (recordReader != null) { - recordReader.handleEvent(readerEvent); - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("No registered record reader, queueing event " + readerEvent.getEventType() - + " with message " + readerEvent.getMessage()); - } - - try { - queuedEvents.put(readerEvent); - } catch (Exception err) { - throw new RuntimeException("Unexpected exception while queueing reader event", err); - } - } - } - - /** - * Clear the list of queued reader events if we are not interested in sending any pending events to any registering record reader. - */ - public void clearQueuedEvents() { - queuedEvents.clear(); - } - } -} diff --git a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapDump.java b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapDump.java deleted file mode 100644 index 9c0362b941d6..000000000000 --- a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapDump.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hive.llap; - -import java.util.Arrays; -import java.util.Properties; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; - -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapred.RecordReader; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.InputSplit; -import org.apache.hadoop.util.ExitUtil; - -/** - * Utility to test query and data retrieval via the LLAP input format. - * llapdump --hiveconf hive.zookeeper.quorum=localhost --hiveconf hive.zookeeper.client.port=2181\ - * --hiveconf hive.llap.daemon.service.hosts=@llap_MiniLlapCluster 'select * from employee where employee_id < 10' - * - */ -public class LlapDump { - - private static final Logger LOG = LoggerFactory.getLogger(LlapDump.class); - - private static String url = "jdbc:hive2://localhost:10000/default"; - private static String user = "hive"; - private static String pwd = ""; - private static String query = null; - private static String numSplits = "1"; - - public static void main(String[] args) throws Exception { - Options opts = createOptions(); - CommandLine cli = new GnuParser().parse(opts, args); - - if (cli.hasOption('h')) { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("llapdump", opts); - return; - } - - if (cli.hasOption('l')) { - url = cli.getOptionValue("l"); - } - - if (cli.hasOption('u')) { - user = cli.getOptionValue("u"); - } - - if (cli.hasOption('p')) { - pwd = cli.getOptionValue("p"); - } - - if (cli.hasOption('n')) { - numSplits = cli.getOptionValue("n"); - } - - Properties configProps = cli.getOptionProperties("hiveconf"); - - if (cli.getArgs().length > 0) { - query = cli.getArgs()[0]; - } - - if (query == null) { - throw new IllegalArgumentException("No query string specified"); - } - - System.out.println("url: "+url); - System.out.println("user: "+user); - System.out.println("query: "+query); - - LlapRowInputFormat format = new LlapRowInputFormat(); - - JobConf job = new JobConf(); - job.set(LlapBaseInputFormat.URL_KEY, url); - job.set(LlapBaseInputFormat.USER_KEY, user); - job.set(LlapBaseInputFormat.PWD_KEY, pwd); - job.set(LlapBaseInputFormat.QUERY_KEY, query); - - // Additional conf settings specified on the command line - for (String key: configProps.stringPropertyNames()) { - job.set(key, configProps.getProperty(key)); - } - - InputSplit[] splits = format.getSplits(job, Integer.parseInt(numSplits)); - - if (splits.length == 0) { - System.out.println("No splits returned - empty scan"); - System.out.println("Results: "); - } else { - boolean first = true; - - for (InputSplit s: splits) { - LOG.info("Processing input split s from " + Arrays.toString(s.getLocations())); - RecordReader reader = format.getRecordReader(s, job, null); - - if (reader instanceof LlapRowRecordReader && first) { - Schema schema = ((LlapRowRecordReader)reader).getSchema(); - System.out.println(""+schema); - } - - if (first) { - System.out.println("Results: "); - System.out.println(""); - first = false; - } - - Row value = reader.createValue(); - while (reader.next(NullWritable.get(), value)) { - printRow(value); - } - } - ExitUtil.terminate(0); - } - } - - private static void printRow(Row row) { - Schema schema = row.getSchema(); - StringBuilder sb = new StringBuilder(); - int length = schema.getColumns().size(); - for (int idx = 0; idx < length; ++idx) { - sb.append(row.getValue(idx)); - if (idx != length - 1) { - sb.append(", "); - } - } - System.out.println(sb.toString()); - } - - static Options createOptions() { - Options result = new Options(); - - result.addOption(OptionBuilder - .withLongOpt("location") - .withDescription("HS2 url") - .hasArg() - .create('l')); - - result.addOption(OptionBuilder - .withLongOpt("user") - .withDescription("user name") - .hasArg() - .create('u')); - - result.addOption(OptionBuilder - .withLongOpt("pwd") - .withDescription("password") - .hasArg() - .create('p')); - - result.addOption(OptionBuilder - .withLongOpt("num") - .withDescription("number of splits") - .hasArg() - .create('n')); - - result.addOption(OptionBuilder - .withValueSeparator() - .hasArgs(2) - .withArgName("property=value") - .withLongOpt("hiveconf") - .withDescription("Use value for given property") - .create()); - - result.addOption(OptionBuilder - .withLongOpt("help") - .withDescription("help") - .hasArg(false) - .create('h')); - - return result; - } -} diff --git a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapRowInputFormat.java b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapRowInputFormat.java deleted file mode 100644 index 43ed20475e1e..000000000000 --- a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapRowInputFormat.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap; - -import java.io.IOException; - -import org.apache.hadoop.hive.llap.LlapBaseRecordReader; -import org.apache.hadoop.hive.llap.LlapInputSplit; -import org.apache.hadoop.hive.llap.LlapRowRecordReader; -import org.apache.hadoop.hive.llap.Row; -import org.apache.hadoop.hive.llap.Schema; - -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.mapred.InputSplit; -import org.apache.hadoop.mapred.RecordReader; -import org.apache.hadoop.mapred.Reporter; - -public class LlapRowInputFormat implements InputFormat { - - private final LlapBaseInputFormat baseInputFormat = new LlapBaseInputFormat<>(); - - @Override - public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { - return baseInputFormat.getSplits(job, numSplits); - } - - @Override - public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter reporter) - throws IOException { - LlapInputSplit llapSplit = (LlapInputSplit) split; - LlapBaseRecordReader reader = - (LlapBaseRecordReader) baseInputFormat.getRecordReader(llapSplit, job, reporter); - return new LlapRowRecordReader(job, reader.getSchema(), reader); - } -} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/LlapStorageHandler.java b/llap-server/src/java/org/apache/hadoop/hive/llap/LlapStorageHandler.java deleted file mode 100644 index 558bcd5b1596..000000000000 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/LlapStorageHandler.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap; - -import org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler; -import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.mapred.OutputFormat; - -public class LlapStorageHandler extends DefaultStorageHandler { - @Override - public Class getInputFormatClass() { - throw new RuntimeException("Should not be called."); - } - - @Override - public Class getOutputFormatClass() { - return LlapOutputFormat.class; - } -} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java index cc87e79dbf4d..f4819c586f86 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java @@ -68,9 +68,6 @@ public Void call() throws Exception { io.netty.handler.codec.http.HttpObjectAggregator.class, // netty-all com.google.flatbuffers.Table.class, //flatbuffers com.carrotsearch.hppc.ByteArrayDeque.class, //hppc - io.jsonwebtoken.security.Keys.class, //jjwt-api - io.jsonwebtoken.impl.DefaultJws.class, //jjwt-impl - io.jsonwebtoken.io.JacksonSerializer.class, //jjwt-jackson }; for (Class c : dependencies) { diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java index 2c7b729f6986..0dadbf982b01 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java @@ -19,14 +19,11 @@ package org.apache.hadoop.hive.llap.daemon.impl; -import org.apache.hadoop.hive.llap.LlapUtil; import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol.BooleanArray; import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol.TezAttemptArray; import java.util.ArrayList; import java.util.List; -import java.util.HashSet; -import java.util.Set; import javax.net.SocketFactory; @@ -198,7 +195,7 @@ public void serviceStop() { } } - public AMNodeInfo registerTask(boolean externalClientRequest, String amLocation, int port, String umbilicalUser, + public AMNodeInfo registerTask(String amLocation, int port, String umbilicalUser, Token jobToken, QueryIdentifier queryIdentifier, TezTaskAttemptID attemptId, boolean isGuaranteed) { if (LOG.isTraceEnabled()) { @@ -220,7 +217,6 @@ public AMNodeInfo registerTask(boolean externalClientRequest, String amLocation, if (amNodeInfo == null) { amNodeInfo = new AMNodeInfo(amNodeId, umbilicalUser, jobToken, queryIdentifier, retryPolicy, retryTimeout, socketFactory, conf); - amNodeInfo.setIsExternalClientRequest(externalClientRequest); amNodeInfoPerQuery.put(amNodeId, amNodeInfo); // Add to the queue only the first time this is registered, and on // subsequent instances when it's taken off the queue. @@ -413,15 +409,8 @@ protected Void callInternal() { BooleanArray guaranteed = new BooleanArray(); guaranteed.set(tasks.guaranteed.toArray(new BooleanWritable[tasks.guaranteed.size()])); - if (LlapUtil.isCloudDeployment(conf) && amNodeInfo.isExternalClientRequest()) { - String hostname = amNodeInfo.amNodeId.getHostname(); - int externalClientCloudRpcPort = amNodeInfo.amNodeId.getPort(); - amNodeInfo.getUmbilical().nodeHeartbeat(new Text(hostname), - new Text(daemonId.getUniqueNodeIdInCluster()), externalClientCloudRpcPort, aw, guaranteed); - } else { - amNodeInfo.getUmbilical().nodeHeartbeat(new Text(nodeId.getHostname()), - new Text(daemonId.getUniqueNodeIdInCluster()), nodeId.getPort(), aw, guaranteed); - } + amNodeInfo.getUmbilical().nodeHeartbeat(new Text(nodeId.getHostname()), + new Text(daemonId.getUniqueNodeIdInCluster()), nodeId.getPort(), aw, guaranteed); } catch (IOException e) { QueryIdentifier currentQueryIdentifier = amNodeInfo.getQueryIdentifier(); amNodeInfo.setAmFailed(true); @@ -480,7 +469,6 @@ protected class AMNodeInfo implements Delayed { private LlapTaskUmbilicalProtocol umbilical; private long nextHeartbeatTime; private final AtomicBoolean isDone = new AtomicBoolean(false); - private final AtomicBoolean isExternalClientRequest = new AtomicBoolean(false); public AMNodeInfo(LlapNodeId amNodeId, String umbilicalUser, @@ -552,14 +540,6 @@ boolean isDone() { return isDone.get(); } - void setIsExternalClientRequest(boolean val) { - isExternalClientRequest.set(val); - } - - boolean isExternalClientRequest() { - return isExternalClientRequest.get(); - } - /** * @return A snapshot of the tasks running at this daemon from this AM. * Doesn't have to be consistent between multiple tasks; whether some task makes it into diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java index d9be6c0b0975..fb25bc5c8c3d 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java @@ -40,10 +40,6 @@ import com.google.common.cache.RemovalListener; import com.google.common.cache.RemovalNotification; -import io.jsonwebtoken.Claims; -import io.jsonwebtoken.Jws; -import io.jsonwebtoken.JwtException; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.llap.LlapUgiManager; import org.apache.hadoop.hive.conf.HiveConf; @@ -83,7 +79,6 @@ import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SetCapacityResponseProto; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.VertexOrBinary; import org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorMetrics; -import org.apache.hadoop.hive.llap.security.LlapExtClientJwtHelper; import org.apache.hadoop.hive.llap.security.LlapSignerImpl; import org.apache.hadoop.hive.llap.tez.Converters; import org.apache.hadoop.hive.llap.tezplugins.LlapTezUtils; @@ -241,8 +236,6 @@ public SubmitWorkResponseProto submitWork(SubmitWorkRequestProto request) throws QueryIdentifierProto qIdProto = vertex.getQueryIdentifier(); - verifyJwtForExternalClient(request, qIdProto.getApplicationIdString(), fragmentIdString); - LOG.info("Queueing container for execution: fragemendId={}, {}", fragmentIdString, stringifySubmitRequest(request, vertex)); @@ -351,39 +344,6 @@ public SubmitWorkResponseProto submitWork(SubmitWorkRequestProto request) throws .build(); } - // if request is coming from llap external client, verify the JWT - // as of now, JWT contains applicationId - private void verifyJwtForExternalClient(SubmitWorkRequestProto request, String extClientAppIdFromSplit, - String fragmentIdString) { - LOG.info("Checking if request[{}] is from llap external client in a cloud based deployment", - extClientAppIdFromSplit); - if (request.getIsExternalClientRequest() && LlapUtil.isCloudDeployment(getConfig())) { - LOG.info("Llap external client request - {}, verifying JWT", extClientAppIdFromSplit); - Preconditions.checkState(request.hasJwt(), "JWT not found in request, fragmentId: " + fragmentIdString); - - LlapExtClientJwtHelper llapExtClientJwtHelper = new LlapExtClientJwtHelper(getConfig()); - Jws claimsJws; - try { - claimsJws = llapExtClientJwtHelper.parseClaims(request.getJwt()); - } catch (JwtException e) { - LOG.error("Cannot verify JWT provided with the request, fragmentId: {}, {}", fragmentIdString, e); - throw e; - } - - String extClientAppIdFromJwt = (String) claimsJws.getBody().get(LlapExtClientJwtHelper.LLAP_EXT_CLIENT_APP_ID); - - // this should never happen ideally. - // extClientAppId is injected in JWT and fragment request by initial get_splits() call. - // so both of these - extClientAppIdFromJwt and extClientAppIdFromSplit should be equal eventually if the signed JWT is valid for this request. - // In get_splits, this extClientAppId is obtained via LlapCoordinator#createExtClientAppId which generates a - // application Id to be used by external clients. - Preconditions.checkState(extClientAppIdFromJwt.equals(extClientAppIdFromSplit), - String.format("applicationId[%s] in request does not match to applicationId[%s] in JWT", - extClientAppIdFromSplit, extClientAppIdFromJwt)); - LOG.info("Llap external client request - {}, JWT verification successful", extClientAppIdFromSplit); - } - } - private SignableVertexSpec extractVertexSpec(SubmitWorkRequestProto request, LlapTokenInfo tokenInfo) throws InvalidProtocolBufferException, IOException { VertexOrBinary vob = request.getWorkSpec(); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index 4d3a5d06e6c0..0312ece5afce 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -49,7 +49,6 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.DaemonId; import org.apache.hadoop.hive.llap.LlapDaemonInfo; -import org.apache.hadoop.hive.llap.LlapOutputFormatService; import org.apache.hadoop.hive.llap.LlapUgiManager; import org.apache.hadoop.hive.llap.LlapUtil; import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration; @@ -76,7 +75,6 @@ import org.apache.hadoop.hive.llap.metrics.LlapMetricsSystem; import org.apache.hadoop.hive.llap.metrics.MetricsUtils; import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService; -import org.apache.hadoop.hive.llap.security.LlapExtClientJwtHelper; import org.apache.hadoop.hive.llap.security.SecretManager; import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler; import org.apache.hadoop.hive.ql.ServiceContext; @@ -141,7 +139,6 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemoryBytes, boolean ioEnabled, boolean isDirectCache, long ioMemoryBytes, String[] localDirs, int srvPort, - boolean externalClientCloudSetupEnabled, int externalClientsRpcPort, int mngPort, int shufflePort, int webPort, String appName) { super("LlapDaemon"); @@ -150,11 +147,6 @@ public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemor Preconditions.checkArgument(numExecutors > 0); Preconditions.checkArgument(srvPort == 0 || (srvPort > 1024 && srvPort < 65536), "Server RPC Port must be between 1025 and 65535, or 0 automatic selection"); - if (externalClientCloudSetupEnabled) { - Preconditions.checkArgument( - externalClientsRpcPort == 0 || (externalClientsRpcPort > 1024 && externalClientsRpcPort < 65536), - "Server RPC port for external clients must be between 1025 and 65535, or 0 automatic selection"); - } Preconditions.checkArgument(mngPort == 0 || (mngPort > 1024 && mngPort < 65536), "Management RPC Port must be between 1025 and 65535, or 0 automatic selection"); @@ -162,10 +154,6 @@ public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemor "Work dirs must be specified"); Preconditions.checkArgument(shufflePort == 0 || (shufflePort > 1024 && shufflePort < 65536), "Shuffle Port must be between 1024 and 65535, or 0 for automatic selection"); - int outputFormatServicePort = HiveConf.getIntVar(daemonConf, HiveConf.ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT); - Preconditions.checkArgument(outputFormatServicePort == 0 - || (outputFormatServicePort > 1024 && outputFormatServicePort < 65536), - "OutputFormatService Port must be between 1024 and 65535, or 0 for automatic selection"); String hosts = HiveConf.getTrimmedVar(daemonConf, ConfVars.LLAP_DAEMON_SERVICE_HOSTS); if (hosts.startsWith("@")) { String zkHosts = HiveConf.getTrimmedVar(daemonConf, ConfVars.HIVE_ZOOKEEPER_QUORUM); @@ -241,11 +229,8 @@ public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemor ", llapIoEnabled=" + ioEnabled + ", llapIoCacheIsDirect=" + isDirectCache + ", rpcListenerPort=" + srvPort + - ", externalClientCloudSetupEnabled=" + externalClientCloudSetupEnabled + - ", rpcListenerPortForExternalClients=" + externalClientsRpcPort + ", mngListenerPort=" + mngPort + ", webPort=" + webPort + - ", outputFormatSvcPort=" + outputFormatServicePort + ", workDirs=" + Arrays.toString(localDirs) + ", shufflePort=" + shufflePort + ", waitQueueSize= " + waitQueueSize + @@ -340,7 +325,7 @@ public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemor this.secretManager = sm; this.server = new LlapProtocolServerImpl(secretManager, numHandlers, this, srvAddress, mngAddress, srvPort, - externalClientsRpcPort, mngPort, daemonId, metrics).withTokenManager(this.llapTokenManager); + mngPort, daemonId, metrics).withTokenManager(this.llapTokenManager); LlapUgiManager llapUgiManager = LlapUgiManager.getInstance(daemonConf); @@ -492,7 +477,6 @@ public void serviceStart() throws Exception { this.shufflePort.set(ShuffleHandler.get().getPort()); getConfig() .setInt(ConfVars.LLAP_DAEMON_YARN_SHUFFLE_PORT.varname, ShuffleHandler.get().getPort()); - LlapOutputFormatService.initializeAndStart(getConfig(), secretManager); super.serviceStart(); // Setup the actual ports in the configuration. @@ -501,16 +485,6 @@ public void serviceStart() throws Exception { if (webServices != null) { getConfig().setInt(ConfVars.LLAP_DAEMON_WEB_PORT.varname, webServices.getPort()); } - getConfig().setInt(ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT.varname, LlapOutputFormatService.get().getPort()); - if (LlapUtil.isCloudDeployment(getConfig())) { - - // this invokes JWT secret provider and tries to get shared secret. - // meant to validate shared secret as well. - new LlapExtClientJwtHelper(getConfig()); - - getConfig().setInt(ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT.varname, - server.getExternalClientsRpcServerBindAddress().getPort()); - } // Ensure this is set in the config so that the AM can read it. getConfig() @@ -544,7 +518,6 @@ public void serviceStop() throws Exception { super.serviceStop(); ShuffleHandler.shutdown(); shutdown(); - LlapOutputFormatService.get().stop(); LOG.info("LlapDaemon shutdown complete"); } @@ -624,8 +597,6 @@ public static void main(String[] args) throws Exception { String[] localDirs = (localDirList == null || localDirList.isEmpty()) ? new String[0] : StringUtils.getTrimmedStrings(localDirList); int rpcPort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_PORT); - int externalClientCloudRpcPort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT); - boolean externalClientCloudSetupEnabled = LlapUtil.isCloudDeployment(daemonConf); int mngPort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_MANAGEMENT_RPC_PORT); int shufflePort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_YARN_SHUFFLE_PORT); int webPort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_WEB_PORT); @@ -643,8 +614,8 @@ public static void main(String[] args) throws Exception { LlapDaemon.initializeLogging(daemonConf); llapDaemon = new LlapDaemon(daemonConf, numExecutors, executorMemoryBytes, isLlapIo, isDirectCache, - ioMemoryBytes, localDirs, rpcPort, externalClientCloudSetupEnabled, - externalClientCloudRpcPort, mngPort, shufflePort, webPort, appName); + ioMemoryBytes, localDirs, rpcPort, + mngPort, shufflePort, webPort, appName); LOG.info("Adding shutdown hook for LlapDaemon"); ShutdownHookManager.addShutdownHook(new CompositeServiceShutdownHook(llapDaemon), 1); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java index b1e8db5287d8..caa5d390f884 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java @@ -83,8 +83,8 @@ private enum TokenRequiresSigning { private final int numHandlers; private final ContainerRunner containerRunner; - private final int srvPort, mngPort, externalClientsRpcPort; - private RPC.Server server, mngServer, externalClientsRpcServer; + private final int srvPort, mngPort; + private RPC.Server server, mngServer; private final AtomicReference srvAddress, mngAddress; private final SecretManager secretManager; private String clusterUser = null; @@ -95,7 +95,7 @@ private enum TokenRequiresSigning { public LlapProtocolServerImpl(SecretManager secretManager, int numHandlers, ContainerRunner containerRunner, AtomicReference srvAddress, - AtomicReference mngAddress, int srvPort, int externalClientsRpcPort, + AtomicReference mngAddress, int srvPort, int mngPort, DaemonId daemonId, LlapDaemonExecutorMetrics executorMetrics) { super("LlapDaemonProtocolServerImpl"); @@ -105,7 +105,6 @@ public LlapProtocolServerImpl(SecretManager secretManager, int numHandlers, this.srvAddress = srvAddress; this.srvPort = srvPort; this.mngAddress = mngAddress; - this.externalClientsRpcPort = externalClientsRpcPort; this.mngPort = mngPort; this.executorMetrics = executorMetrics; LOG.info("Creating: " + LlapProtocolServerImpl.class.getSimpleName() + @@ -241,15 +240,6 @@ private void startProtocolServers( server = LlapUtil.startProtocolServer(srvPort, numHandlers, srvAddress, conf, daemonImpl, LlapProtocolBlockingPB.class, secretManager, pp, ConfVars.LLAP_SECURITY_ACL, ConfVars.LLAP_SECURITY_ACL_DENY); - // for cloud deployments, start a separate RPC server on the port - // which we can open to accept requests from external clients. - if (LlapUtil.isCloudDeployment(conf)) { - externalClientsRpcServer = LlapUtil.startProtocolServer(externalClientsRpcPort, numHandlers, null, conf, daemonImpl, - LlapProtocolBlockingPB.class, secretManager, pp, ConfVars.LLAP_SECURITY_ACL, - ConfVars.LLAP_SECURITY_ACL_DENY); - - LOG.info("Started externalClientsRpcServer for cloud based deployments : {}, {}", externalClientsRpcServer.getListenerAddress(), externalClientsRpcServer); - } mngServer = LlapUtil.startProtocolServer(mngPort, 2, mngAddress, conf, managementImpl, LlapManagementProtocolPB.class, secretManager, pp, ConfVars.LLAP_MANAGEMENT_ACL, ConfVars.LLAP_MANAGEMENT_ACL_DENY); @@ -261,9 +251,6 @@ public void serviceStop() { if (server != null) { server.stop(); } - if (externalClientsRpcServer != null) { - externalClientsRpcServer.stop(); - } if (mngServer != null) { mngServer.stop(); } @@ -279,11 +266,6 @@ InetSocketAddress getManagementBindAddress() { return mngAddress.get(); } - @InterfaceAudience.Private - InetSocketAddress getExternalClientsRpcServerBindAddress() { - return externalClientsRpcServer.getListenerAddress(); - } - @Override public GetTokenResponseProto getDelegationToken(RpcController controller, GetTokenRequestProto request) throws ServiceException { diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java index 9f6ff74ecc9c..eddf2834eced 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java @@ -160,7 +160,7 @@ public TaskRunnerCallable(SubmitWorkRequestProto request, QueryFragmentInfo frag this.amReporter = amReporter; // Register with the AMReporter when the callable is setup. Unregister once it starts running. if (amReporter != null && jobToken != null) { - this.amNodeInfo = amReporter.registerTask(request.getIsExternalClientRequest(), request.getAmHost(), request.getAmPort(), + this.amNodeInfo = amReporter.registerTask(request.getAmHost(), request.getAmPort(), vertex.getTokenIdentifier(), jobToken, fragmentInfo.getQueryInfo().getQueryIdentifier(), attemptId, isGuaranteed); } else { diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/LlapDaemonExtension.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/LlapDaemonExtension.java index 0b6ddb25daee..0d1afc569b9e 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/LlapDaemonExtension.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/LlapDaemonExtension.java @@ -64,7 +64,7 @@ public void beforeEach(ExtensionContext context) throws Exception { HiveConf.setVar(conf, HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS, "llap"); LlapDaemonInfo.initialize(appName, conf); daemon = - new LlapDaemon(conf, 1, LlapDaemon.getTotalHeapSize(), false, false, -1, new String[1], 0, false, 0, 0, 0, 0, + new LlapDaemon(conf, 1, LlapDaemon.getTotalHeapSize(), false, false, -1, new String[1], 0, 0, 0, 0, appName); daemon.init(conf); daemon.start(); diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java index f8cd79d6826f..b23c7e32b96a 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java @@ -24,7 +24,6 @@ import java.io.File; import java.io.IOException; -import org.apache.hadoop.hive.llap.LlapUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -37,8 +36,6 @@ import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.Service; -import org.apache.hadoop.util.Shell; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hive.testutils.MiniZooKeeperCluster; import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; @@ -142,22 +139,17 @@ private MiniLlapCluster(String clusterName, @Nullable MiniZooKeeperCluster miniZ @Override public void serviceInit(Configuration conf) throws IOException, InterruptedException { int rpcPort = 0; - int externalClientCloudRpcPort = 0; int mngPort = 0; int shufflePort = 0; int webPort = 0; - int outputFormatServicePort = 0; boolean usePortsFromConf = conf.getBoolean("minillap.usePortsFromConf", false); LOG.info("MiniLlap configured to use ports from conf: {}", usePortsFromConf); if (usePortsFromConf) { rpcPort = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_DAEMON_RPC_PORT); - externalClientCloudRpcPort = HiveConf.getIntVar(conf, ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT); mngPort = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_MANAGEMENT_RPC_PORT); shufflePort = conf.getInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, ShuffleHandler.DEFAULT_SHUFFLE_PORT); webPort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_WEB_PORT); - outputFormatServicePort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT); } - HiveConf.setIntVar(conf, ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT, outputFormatServicePort); if (ownZkCluster) { miniZooKeeperCluster = new MiniZooKeeperCluster(); @@ -174,12 +166,10 @@ public void serviceInit(Configuration conf) throws IOException, InterruptedExcep clusterSpecificConfiguration.set(ConfVars.HIVE_ZOOKEEPER_QUORUM.varname, "localhost"); clusterSpecificConfiguration.setInt(ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT.varname, miniZooKeeperCluster.getClientPort()); - boolean externalClientCloudSetupEnabled = LlapUtil.isCloudDeployment(conf); - LOG.info("Initializing {} llap instances for MiniLlapCluster with name={}", numInstances, clusterNameTrimmed); for (int i = 0 ;i < numInstances ; i++) { llapDaemons[i] = new LlapDaemon(conf, numExecutorsPerService, execBytesPerService, llapIoEnabled, - ioIsDirect, ioBytesPerService, localDirs, rpcPort, externalClientCloudSetupEnabled, externalClientCloudRpcPort, + ioIsDirect, ioBytesPerService, localDirs, rpcPort, mngPort, shufflePort, webPort, clusterNameTrimmed); llapDaemons[i].init(new Configuration(conf)); } diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java index 44351b9d419b..43fdfe195d37 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java @@ -106,7 +106,7 @@ public void testEnforceProperNumberOfIOThreads() throws IOException { HiveConf.setIntVar(hiveConf, HiveConf.ConfVars.LLAP_IO_THREADPOOL_SIZE, 3); daemon = new LlapDaemon(hiveConf, 4, LlapDaemon.getTotalHeapSize(), true, false, - -1, new String[1], 0, false, 0,0, 0, defaultWebPort, "TestLlapDaemon"); + -1, new String[1], 0,0, 0, defaultWebPort, "TestLlapDaemon"); } @Test @@ -120,7 +120,7 @@ public void testLocalDirCleaner() throws IOException, InterruptedException { createFile(localDirs[0] + "/file3"); daemon = new LlapDaemon(hiveConf, 1, LlapDaemon.getTotalHeapSize(), false, false, - -1, localDirs, 0, false, 0,0, 0, defaultWebPort, "TestLlapDaemon"); + -1, localDirs, 0, 0, 0, defaultWebPort, "TestLlapDaemon"); daemon.init(hiveConf); assertFileExists(localDirs[0] + "/hive/appcache/file1", true); @@ -155,7 +155,7 @@ public void testUpdateRegistration() throws IOException { int enabledQueue = 2; daemon = new LlapDaemon(hiveConf, 1, LlapDaemon.getTotalHeapSize(), false, false, - -1, new String[1], 0, false, 0,0, 0, defaultWebPort, "TestLlapDaemon"); + -1, new String[1], 0,0, 0, defaultWebPort, "TestLlapDaemon"); trySetMock(daemon, LlapRegistryService.class, mockRegistry); diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java index a3802ecda44b..8fefb3bc7243 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java @@ -60,7 +60,7 @@ public void testSimpleCall() throws ServiceException, IOException { LlapProtocolServerImpl server = new LlapProtocolServerImpl(null, numHandlers, containerRunnerMock, new AtomicReference(), new AtomicReference(), - 0, 0, 0, null, null); + 0, 0, null, null); when(containerRunnerMock.submitWork(any(SubmitWorkRequestProto.class))).thenReturn( SubmitWorkResponseProto .newBuilder() @@ -95,7 +95,7 @@ public void testGetDaemonMetrics() throws ServiceException, IOException { LlapProtocolServerImpl server = new LlapProtocolServerImpl(null, numHandlers, null, new AtomicReference(), new AtomicReference(), - 0, 0, 0, null, executorMetrics); + 0, 0, null, executorMetrics); executorMetrics.addMetricsFallOffFailedTimeLost(10); executorMetrics.addMetricsFallOffKilledTimeLost(11); executorMetrics.addMetricsFallOffSuccessTimeLost(12); @@ -166,7 +166,7 @@ public void testSetCapacity() throws ServiceException, IOException { LlapProtocolServerImpl server = new LlapProtocolServerImpl(null, numHandlers, containerRunnerMock, new AtomicReference(), new AtomicReference(), - 0, 0, 0, null, executorMetrics); + 0, 0, null, executorMetrics); try { server.init(new Configuration()); diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/comparator/TestAMReporter.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/comparator/TestAMReporter.java index 5070dfc62a1b..56b3e8b69399 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/comparator/TestAMReporter.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/comparator/TestAMReporter.java @@ -72,9 +72,9 @@ public void testMultipleAM() throws InterruptedException { String am2Location = "am2"; String umbilicalUser = "user"; QueryIdentifier queryId = new QueryIdentifier("app", 0); - amReporter.registerTask(false,am1Location, am1Port, umbilicalUser, null, queryId, + amReporter.registerTask(am1Location, am1Port, umbilicalUser, null, queryId, mock(TezTaskAttemptID.class), false); - amReporter.registerTask(false,am2Location, am2Port, umbilicalUser, null, queryId, + amReporter.registerTask(am2Location, am2Port, umbilicalUser, null, queryId, mock(TezTaskAttemptID.class), false); Thread.currentThread().sleep(2000); diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestLlapOrcCacheLoader.java b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestLlapOrcCacheLoader.java index fa0e59b04a2c..e98afaa776a8 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestLlapOrcCacheLoader.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestLlapOrcCacheLoader.java @@ -18,7 +18,6 @@ */ package org.apache.hadoop.hive.llap.io.encoded; -import io.jsonwebtoken.lang.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -37,6 +36,7 @@ import org.apache.hive.common.util.FixedSizedObjectPool; import org.junit.Before; import org.junit.Test; +import org.junit.jupiter.api.Assertions; import java.io.IOException; @@ -89,7 +89,7 @@ public void testLoadFooter() throws IOException { loader.loadFileFooter(); } MetadataCache.LlapBufferOrBuffers metadata = metaCache.getFileMetadata(key); - Assert.notNull(metadata); + Assertions.assertNotNull(metadata); } @@ -108,7 +108,7 @@ public void testLoadUncompressedRanges() throws IOException { cache.getFileData(key, range, 0, mockDiskRangeListFactory, null, gotAllData); - Assert.isTrue(gotAllData.value); + Assertions.assertTrue(gotAllData.value); } } @@ -126,7 +126,7 @@ public void testLoadValidRanges() throws IOException { cache.getFileData(key, range, 0, mockDiskRangeListFactory, null, gotAllData); - Assert.isTrue(gotAllData.value); + Assertions.assertTrue(gotAllData.value); } @Test @@ -143,7 +143,7 @@ public void testLoadAlreadyLoadedRange() throws IOException { DataCache.BooleanRef gotAllData = new DataCache.BooleanRef(); cache.getFileData(key, range, 0, mockDiskRangeListFactory, null, gotAllData); - Assert.isTrue(gotAllData.value); + Assertions.assertTrue(gotAllData.value); DiskRangeList range2 = new DiskRangeList(ORC_PADDING,14); try(LlapOrcCacheLoader loader = new LlapOrcCacheLoader(path, key, conf, mockDataCache, metaCache, @@ -154,7 +154,7 @@ public void testLoadAlreadyLoadedRange() throws IOException { gotAllData.value = false; cache.getFileData(key, range, 0, mockDiskRangeListFactory, null, gotAllData); - Assert.isTrue(gotAllData.value); + Assertions.assertTrue(gotAllData.value); } @Test @@ -171,7 +171,7 @@ public void testLoadBadlyEstimatedRanges() throws IOException { cache.getFileData(key, range, 0, mockDiskRangeListFactory, null, gotAllData); - Assert.isTrue(!gotAllData.value); + Assertions.assertFalse(gotAllData.value); } diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java index ce5d4dbaccf2..56881047227a 100644 --- a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java +++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java @@ -888,7 +888,6 @@ private void resetCurrentDag(int newDagId, String hiveQueryId) { // is likely already happening. } - // Needed for GenericUDTFGetSplits, where TaskSpecs are generated private String extractQueryId(TaskSpec taskSpec) throws IOException { UserPayload processorPayload = taskSpec.getProcessorDescriptor().getUserPayload(); Configuration conf = TezUtils.createConfFromUserPayload(processorPayload); diff --git a/packaging/pom.xml b/packaging/pom.xml index 4171fa28994c..496c21ab20c3 100644 --- a/packaging/pom.xml +++ b/packaging/pom.xml @@ -384,11 +384,6 @@ hive-llap-client ${project.version} - - org.apache.hive - hive-llap-ext-client - ${project.version} - org.apache.hive hive-hplsql diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java index a144fc121219..3ebe79c3e068 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java @@ -532,9 +532,6 @@ private StatefulSet doBuildStatefulSet(HiveCluster hiveCluster, LlapSpec llap, I int webPort = ConfigUtils.getInt(llap.configOverrides(), ConfigUtils.HIVE_LLAP_DAEMON_WEB_PORT_KEY, null, ConfigUtils.HIVE_LLAP_DAEMON_WEB_PORT_DEFAULT); - int outputPort = ConfigUtils.getInt(llap.configOverrides(), - ConfigUtils.HIVE_LLAP_DAEMON_OUTPUT_SERVICE_PORT_KEY, null, - ConfigUtils.HIVE_LLAP_DAEMON_OUTPUT_SERVICE_PORT_DEFAULT); List ports = new ArrayList<>(); ports.add(new ContainerPortBuilder() @@ -546,9 +543,6 @@ private StatefulSet doBuildStatefulSet(HiveCluster hiveCluster, LlapSpec llap, I ports.add(new ContainerPortBuilder() .withName("web").withContainerPort(webPort) .withProtocol("TCP").build()); - ports.add(new ContainerPortBuilder() - .withName("output").withContainerPort(outputPort) - .withProtocol("TCP").build()); Probe readinessProbe = buildTcpProbe(managementPort, llap.readinessProbe(), 15, 10, 3); diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java index 6253681455b2..3684361c3853 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java @@ -135,9 +135,6 @@ public static String tezAmComponentKey(String llapName) { public static final String HIVE_LLAP_DAEMON_WEB_PORT_KEY = "hive.llap.daemon.web.port"; public static final int HIVE_LLAP_DAEMON_WEB_PORT_DEFAULT = 15002; - public static final String HIVE_LLAP_DAEMON_OUTPUT_SERVICE_PORT_KEY = "hive.llap.daemon.output.service.port"; - public static final int HIVE_LLAP_DAEMON_OUTPUT_SERVICE_PORT_DEFAULT = 15003; - public static final String HIVE_LLAP_DAEMON_UMBILICAL_PORT_KEY = "hive.llap.daemon.umbilical.port"; public static final String HIVE_LLAP_DAEMON_UMBILICAL_PORT_DEFAULT = "0"; diff --git a/packaging/src/main/assembly/src.xml b/packaging/src/main/assembly/src.xml index fac8a76515f7..ef5cdf93284c 100644 --- a/packaging/src/main/assembly/src.xml +++ b/packaging/src/main/assembly/src.xml @@ -88,7 +88,6 @@ metastore/**/* llap-common/**/* llap-client/**/* - llap-ext-client/**/* llap-tez/**/* llap-server/**/* lib/**/* diff --git a/pom.xml b/pom.xml index 6d8fbeeb5aa4..6ba426e338c4 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,6 @@ streaming llap-common llap-client - llap-ext-client llap-tez llap-server shims @@ -227,7 +226,6 @@ 5.7.1 3.0.0 2.9.0 - 0.10.5 1.2 2.0.1 2.9.0 @@ -445,21 +443,6 @@ truffle-runtime ${graalvm.version} - - io.jsonwebtoken - jjwt-api - ${jjwt.version} - - - io.jsonwebtoken - jjwt-impl - ${jjwt.version} - - - io.jsonwebtoken - jjwt-jackson - ${jjwt.version} - io.netty netty-all diff --git a/ql/src/java/org/apache/hadoop/hive/llap/ChannelOutputStream.java b/ql/src/java/org/apache/hadoop/hive/llap/ChannelOutputStream.java deleted file mode 100644 index 2a1fa9f8da9c..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/llap/ChannelOutputStream.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hive.llap; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.concurrent.Semaphore; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * OutputStream to write to the Netty Channel - */ -public class ChannelOutputStream extends OutputStream { - - private static final Logger LOG = LoggerFactory.getLogger(ChannelOutputStream.class); - - private ChannelHandlerContext chc; - private int bufSize; - private String id; - private ByteBuf buf; - private byte[] singleByte = new byte[1]; - private boolean closed = false; - private final int maxPendingWrites; - private final Semaphore writeResources; - - private ChannelFutureListener writeListener = new ChannelFutureListener() { - @Override - public void operationComplete(ChannelFuture future) { - writeResources.release(); - - if (future.isCancelled()) { - LOG.error("Write cancelled on ID " + id); - } else if (!future.isSuccess()) { - LOG.error("Write error on ID " + id, future.cause()); - } - } - }; - - private ChannelFutureListener closeListener = new ChannelFutureListener() { - @Override - public void operationComplete(ChannelFuture future) { - if (future.isCancelled()) { - LOG.error("Close cancelled on ID " + id); - } else if (!future.isSuccess()) { - LOG.error("Close failed on ID " + id, future.cause()); - } - } - }; - - public ChannelOutputStream(ChannelHandlerContext chc, String id, int bufSize, int maxOutstandingWrites) { - this.chc = chc; - this.id = id; - this.bufSize = bufSize; - this.buf = chc.alloc().buffer(bufSize); - this.maxPendingWrites = maxOutstandingWrites; - this.writeResources = new Semaphore(maxPendingWrites); - } - - @Override - public void write(int b) throws IOException { - singleByte[0] = (byte) b; - write(singleByte, 0, 1); - } - - @Override - public void write(byte[] b) throws IOException { - write(b, 0, b.length); - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - int currentOffset = off; - int bytesRemaining = len; - - while (bytesRemaining + buf.readableBytes() > bufSize) { - int iterationLen = bufSize - buf.readableBytes(); - writeInternal(b, currentOffset, iterationLen); - currentOffset += iterationLen; - bytesRemaining -= iterationLen; - } - - if (bytesRemaining > 0) { - writeInternal(b, currentOffset, bytesRemaining); - } - } - - @Override - public void flush() throws IOException { - if (buf.isReadable()) { - writeToChannel(); - } - chc.flush(); - } - - @Override - public void close() throws IOException { - if (closed) { - throw new IOException("Already closed: " + id); - } - - try { - flush(); - } catch (IOException err) { - LOG.error("Error flushing stream before close on " + id, err); - } - - closed = true; - - // Wait for all writes to finish before we actually close. - takeWriteResources(maxPendingWrites); - - try { - chc.close().addListener(closeListener); - } finally { - buf.release(); - buf = null; - chc = null; - closed = true; - } - } - - // Attempt to acquire write resources, waiting if they are not available. - private void takeWriteResources(int numResources) throws IOException { - try { - writeResources.acquire(numResources); - } catch (InterruptedException ie) { - throw new IOException("Interrupted while waiting for write resources for " + id); - } - } - - private void writeToChannel() throws IOException { - if (closed) { - throw new IOException("Already closed: " + id); - } - - takeWriteResources(1); - chc.writeAndFlush(buf.copy()).addListener(writeListener); - buf.clear(); - } - - private void writeInternal(byte[] b, int off, int len) throws IOException { - if (closed) { - throw new IOException("Already closed: " + id); - } - - buf.writeBytes(b, off, len); - if (buf.readableBytes() >= bufSize) { - writeToChannel(); - } - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormat.java b/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormat.java deleted file mode 100644 index 7de07ae4aecc..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormat.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hive.llap; - -import java.io.IOException; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.OutputFormat; -import org.apache.hadoop.mapred.RecordWriter; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.util.Progressable; -import org.apache.hadoop.hive.llap.io.api.LlapProxy; -import org.apache.hadoop.hive.ql.io.StreamingOutputFormat; - -import com.google.common.base.Preconditions; - -public class LlapOutputFormat - implements OutputFormat, StreamingOutputFormat { - - public static final String LLAP_OF_ID_KEY = "llap.of.id"; - - @Override - public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException { - } - - @Override - public RecordWriter getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException { - if (!LlapProxy.isDaemon()) { - throw new IOException("LlapOutputFormat can only be used inside Llap"); - } - try { - return LlapOutputFormatService.get().getWriter(job.get(LLAP_OF_ID_KEY)); - } catch (InterruptedException e) { - throw new IOException(e); - } - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormatService.java b/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormatService.java deleted file mode 100644 index 5e9238c98e88..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormatService.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hive.llap; - -import java.util.Map; -import java.util.HashMap; -import java.io.IOException; -import java.net.InetSocketAddress; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.RecordWriter; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.LlapOutputSocketInitMessage; -import org.apache.hadoop.hive.llap.io.ChunkedOutputStream; -import org.apache.hadoop.hive.llap.security.SecretManager; - -import com.google.common.base.Preconditions; - -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.protobuf.ProtobufDecoder; -import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder; -import io.netty.handler.codec.string.StringEncoder; - - -/** - * Responsible for sending back result set data to the connections - * made by external clients via the LLAP input format. - */ -public class LlapOutputFormatService { - - private static final Logger LOG = LoggerFactory.getLogger(LlapOutputFormat.class); - - private static final AtomicBoolean started = new AtomicBoolean(false); - private static final AtomicBoolean initing = new AtomicBoolean(false); - private static LlapOutputFormatService INSTANCE; - - // TODO: the global lock might be to coarse here. - private final Object lock = new Object(); - private final Map> writers = new HashMap>(); - private final Map errors = new HashMap(); - private final Configuration conf; - private static final int WAIT_TIME = 5; - - private EventLoopGroup eventLoopGroup; - private ServerBootstrap serverBootstrap; - private ChannelFuture listeningChannelFuture; - private int port; - private final SecretManager sm; - private final long writerTimeoutMs; - - private LlapOutputFormatService(Configuration conf, SecretManager sm) throws IOException { - this.sm = sm; - this.conf = conf; - this.writerTimeoutMs = HiveConf.getTimeVar( - conf, ConfVars.LLAP_DAEMON_OUTPUT_STREAM_TIMEOUT, TimeUnit.MILLISECONDS); - } - - public static void initializeAndStart(Configuration conf, SecretManager sm) throws Exception { - if (!initing.getAndSet(true)) { - INSTANCE = new LlapOutputFormatService(conf, sm); - INSTANCE.start(); - started.set(true); - } - } - - public static LlapOutputFormatService get() throws IOException { - Preconditions.checkState(started.get(), - "LlapOutputFormatService must be started before invoking get"); - return INSTANCE; - } - - public void start() throws IOException { - LOG.info("Starting LlapOutputFormatService"); - - int portFromConf = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT); - int sendBufferSize = HiveConf.getIntVar(conf, - HiveConf.ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_SEND_BUFFER_SIZE); - // Netty defaults to no of processors * 2. Can be changed via -Dio.netty.eventLoopThreads - eventLoopGroup = new NioEventLoopGroup(); - serverBootstrap = new ServerBootstrap(); - serverBootstrap.group(eventLoopGroup); - serverBootstrap.channel(NioServerSocketChannel.class); - serverBootstrap.childHandler(new LlapOutputFormatServiceChannelHandler(sendBufferSize)); - try { - listeningChannelFuture = serverBootstrap.bind(portFromConf).sync(); - this.port = ((InetSocketAddress) listeningChannelFuture.channel().localAddress()).getPort(); - LOG.info("LlapOutputFormatService: Binding to port: {} with send buffer size: {} ", this.port, - sendBufferSize); - } catch (InterruptedException err) { - throw new IOException("LlapOutputFormatService: Error binding to port " + portFromConf, err); - } - } - - public void stop() throws IOException, InterruptedException { - LOG.info("Stopping LlapOutputFormatService"); - - if (listeningChannelFuture != null) { - listeningChannelFuture.channel().close().sync(); - listeningChannelFuture = null; - } else { - LOG.warn("LlapOutputFormatService does not appear to have a listening port to close."); - } - - eventLoopGroup.shutdownGracefully(1, WAIT_TIME, TimeUnit.SECONDS).sync(); - } - - @SuppressWarnings("unchecked") - public RecordWriter getWriter(String id) throws IOException, InterruptedException { - RecordWriter writer = null; - synchronized (lock) { - long startTime = System.nanoTime(); - boolean isFirst = true; - while ((writer = writers.get(id)) == null) { - String error = errors.remove(id); - if (error != null) { - throw new IOException(error); - } - if (isFirst) { - LOG.info("Waiting for writer for " + id); - isFirst = false; - } - if (((System.nanoTime() - startTime) / 1000000) > writerTimeoutMs) { - throw new IOException("The writer for " + id + " has timed out after " - + writerTimeoutMs + "ms"); - } - lock.wait(writerTimeoutMs); - } - } - LOG.info("Returning writer for: "+id); - return (RecordWriter) writer; - } - - public int getPort() { - return port; - } - - protected class LlapOutputFormatServiceHandler - extends SimpleChannelInboundHandler { - private final int sendBufferSize; - - public LlapOutputFormatServiceHandler(final int sendBufferSize) { - this.sendBufferSize = sendBufferSize; - } - - @Override - public void channelRead0(ChannelHandlerContext ctx, LlapOutputSocketInitMessage msg) { - String id = msg.getFragmentId(); - byte[] tokenBytes = msg.hasToken() ? msg.getToken().toByteArray() : null; - try { - registerReader(ctx, id, tokenBytes); - } catch (Throwable t) { - // Make sure we fail the channel if something goes wrong. - // We internally handle all the "expected" exceptions, so log a lot of information here. - failChannel(ctx, id, StringUtils.stringifyException(t)); - } - } - - private void registerReader(ChannelHandlerContext ctx, String id, byte[] tokenBytes) { - if (sm != null) { - try { - sm.verifyToken(tokenBytes); - } catch (SecurityException | IOException ex) { - failChannel(ctx, id, ex.getMessage()); - return; - } - } - LOG.debug("registering socket for: " + id); - int maxPendingWrites = HiveConf.getIntVar(conf, - HiveConf.ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_MAX_PENDING_WRITES); - @SuppressWarnings("rawtypes") - RecordWriter writer = new LlapRecordWriter(id, - new ChunkedOutputStream( - new ChannelOutputStream(ctx, id, sendBufferSize, maxPendingWrites), sendBufferSize, id)); - boolean isFailed = true; - synchronized (lock) { - if (!writers.containsKey(id)) { - isFailed = false; - writers.put(id, writer); - // Add listener to handle any cleanup for when the connection is closed - ctx.channel().closeFuture().addListener(new LlapOutputFormatChannelCloseListener(id)); - lock.notifyAll(); - } - } - if (isFailed) { - failChannel(ctx, id, "Writer already registered for " + id); - } - } - - /** Do not call under lock. */ - private void failChannel(ChannelHandlerContext ctx, String id, String error) { - // TODO: write error to the channel? there's no mechanism for that now. - ctx.close(); - synchronized (lock) { - errors.put(id, error); - lock.notifyAll(); - } - LOG.error(error); - } - } - - protected class LlapOutputFormatChannelCloseListener implements ChannelFutureListener { - private String id; - - LlapOutputFormatChannelCloseListener(String id) { - this.id = id; - } - - @Override - public void operationComplete(ChannelFuture future) throws Exception { - RecordWriter writer = null; - synchronized (INSTANCE) { - writer = writers.remove(id); - } - - if (writer == null) { - LOG.warn("Did not find a writer for ID " + id); - } - } - } - - protected class LlapOutputFormatServiceChannelHandler extends ChannelInitializer { - private final int sendBufferSize; - public LlapOutputFormatServiceChannelHandler(final int sendBufferSize) { - this.sendBufferSize = sendBufferSize; - } - - @Override - public void initChannel(SocketChannel ch) throws Exception { - ch.pipeline().addLast( - new ProtobufVarint32FrameDecoder(), - new ProtobufDecoder(LlapOutputSocketInitMessage.getDefaultInstance()), - new StringEncoder(), - new LlapOutputFormatServiceHandler(sendBufferSize)); - } - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/llap/LlapRecordWriter.java b/ql/src/java/org/apache/hadoop/hive/llap/LlapRecordWriter.java deleted file mode 100644 index ad1a9e0b09d6..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/llap/LlapRecordWriter.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap; - -import java.io.IOException; -import java.io.OutputStream; -import java.io.DataOutputStream; - -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapred.RecordWriter; -import org.apache.hadoop.mapred.Reporter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LlapRecordWriter - implements RecordWriter { - public static final Logger LOG = LoggerFactory.getLogger(LlapRecordWriter.class); - - String id; - DataOutputStream dos; - - public LlapRecordWriter(String id, OutputStream out) { - this.id = id; - dos = new DataOutputStream(out); - } - - @Override - public void close(Reporter reporter) throws IOException { - LOG.info("CLOSING the record writer output stream for " + id); - dos.close(); - } - - @Override - public void write(K key, V value) throws IOException { - value.write(dos); - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index 81b8f30d6527..1bff2fefbc54 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -658,8 +658,6 @@ public final class FunctionRegistry { system.registerGenericUDTF("parse_url_tuple", GenericUDTFParseUrlTuple.class); system.registerGenericUDTF("posexplode", GenericUDTFPosExplode.class); system.registerGenericUDTF("stack", GenericUDTFStack.class); - system.registerGenericUDTF("get_splits", GenericUDTFGetSplits.class); - system.registerGenericUDTF("get_llap_splits", GenericUDTFGetSplits2.class); system.registerGenericUDTF("get_sql_schema", GenericUDTFGetSQLSchema.class); //PTF declarations diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java index 67ef4cb5ac01..41261720adf7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java @@ -38,10 +38,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.AbstractMapOperator; -import org.apache.hadoop.hive.llap.io.api.LlapProxy; -import org.apache.hadoop.hive.llap.tez.Converters; import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.llap.LlapOutputFormat; import org.apache.hadoop.hive.ql.exec.DummyStoreOperator; import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator; import org.apache.hadoop.hive.ql.exec.MapOperator; @@ -102,9 +99,6 @@ public class MapRecordProcessor extends RecordProcessor { public MapRecordProcessor(final JobConf jconf, final ProcessorContext context) throws Exception { super(jconf, context); String queryId = HiveConf.getVar(jconf, HiveConf.ConfVars.HIVE_QUERY_ID); - if (LlapProxy.isDaemon()) { - setLlapOfFragmentId(context); - } cache = ObjectCacheFactory.getCache(jconf, queryId, true); dynamicValueCache = ObjectCacheFactory.getCache(jconf, queryId, false, true); execContext = new ExecMapperContext(jconf); @@ -113,13 +107,6 @@ public MapRecordProcessor(final JobConf jconf, final ProcessorContext context) t HiveConf.getVar(jconf, HiveConf.ConfVars.SPLIT_GROUPING_MODE)); } - private void setLlapOfFragmentId(final ProcessorContext context) { - // TODO: could we do this only if the OF is actually used? - String attemptId = Converters.createTaskAttemptId(context).toString(); - LOG.debug("Setting the LLAP fragment ID for OF to {}", attemptId); - jconf.set(LlapOutputFormat.LLAP_OF_ID_KEY, attemptId); - } - @Override void init(MRTaskReporter mrReporter, Map inputs, Map outputs) throws Exception { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 6bcbd346b235..aac9c35692c7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -290,7 +290,6 @@ import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; -import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe2; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; @@ -8024,8 +8023,6 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // Set the fetch formatter to be a no-op for the ListSinkOperator, since we'll // write out formatted thrift objects to SequenceFile conf.set(SerDeUtils.LIST_SINK_OUTPUT_FORMATTER, NoOpFetchFormatter.class.getName()); - } else if (fileFormat.equals(PlanUtils.LLAP_OUTPUT_FORMAT_KEY)) { - serdeClass = LazyBinarySerDe2.class; } tableDescriptor = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, fileFormat, serdeClass); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index 81c08194ebd8..407b0846b730 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -43,7 +43,6 @@ import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.llap.LlapOutputFormat; import org.apache.hadoop.hive.metastore.HiveMetaStoreUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; @@ -102,9 +101,6 @@ public final class PlanUtils { private static long countForMapJoinDumpFilePrefix = 0; - public static final String LLAP_OUTPUT_FORMAT_KEY = "Llap"; - private static final String LLAP_OF_SH_CLASS = "org.apache.hadoop.hive.llap.LlapStorageHandler"; - public static synchronized long getCountForMapJoinDumpFilePrefix() { return countForMapJoinDumpFilePrefix++; } @@ -286,11 +282,6 @@ public static TableDesc getTableDesc( inputFormat = RCFileInputFormat.class; outputFormat = RCFileOutputFormat.class; assert serdeClass == ColumnarSerDe.class; - } else if (LLAP_OUTPUT_FORMAT_KEY.equalsIgnoreCase(fileFormat)) { - inputFormat = TextInputFormat.class; - outputFormat = LlapOutputFormat.class; - properties.setProperty( - hive_metastoreConstants.META_TABLE_STORAGE, LLAP_OF_SH_CLASS); } else { // use TextFile by default inputFormat = TextInputFormat.class; outputFormat = IgnoreKeyTextOutputFormat.class; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java deleted file mode 100644 index d135dc2f9fcc..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java +++ /dev/null @@ -1,817 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.ql.udf.generic; - -import java.io.ByteArrayOutputStream; -import java.io.Closeable; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Set; -import java.util.UUID; - -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidTxnWriteIdList; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.llap.FieldDesc; -import org.apache.hadoop.hive.llap.LlapInputSplit; -import org.apache.hadoop.hive.llap.LlapUtil; -import org.apache.hadoop.hive.llap.NotTezEventHelper; -import org.apache.hadoop.hive.llap.Schema; -import org.apache.hadoop.hive.llap.SubmitWorkInfo; -import org.apache.hadoop.hive.llap.coordinator.LlapCoordinator; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryIdentifierProto; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SignableVertexSpec; -import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService; -import org.apache.hadoop.hive.llap.ext.LlapDaemonInfo; -import org.apache.hadoop.hive.llap.registry.LlapServiceInstance; -import org.apache.hadoop.hive.llap.registry.LlapServiceInstanceSet; -import org.apache.hadoop.hive.llap.security.LlapExtClientJwtHelper; -import org.apache.hadoop.hive.llap.security.LlapSigner; -import org.apache.hadoop.hive.llap.security.LlapSigner.Signable; -import org.apache.hadoop.hive.llap.security.LlapSigner.SignedMessage; -import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier; -import org.apache.hadoop.hive.llap.security.LlapTokenLocalClient; -import org.apache.hadoop.hive.llap.tez.Converters; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.Context; -import org.apache.hadoop.hive.ql.Driver; -import org.apache.hadoop.hive.ql.QueryPlan; -import org.apache.hadoop.hive.ql.QueryState; -import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.exec.UDFArgumentException; -import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; -import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; -import org.apache.hadoop.hive.ql.exec.tez.DagUtils; -import org.apache.hadoop.hive.ql.exec.tez.HiveSplitGenerator; -import org.apache.hadoop.hive.ql.exec.tez.TezTask; -import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; -import org.apache.hadoop.hive.ql.lockmgr.TxnManagerFactory; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.parse.ParseException; -import org.apache.hadoop.hive.ql.parse.ParseUtils; -import org.apache.hadoop.hive.ql.plan.MapWork; -import org.apache.hadoop.hive.ql.plan.PlanUtils; -import org.apache.hadoop.hive.ql.plan.TezWork; -import org.apache.hadoop.hive.ql.processors.CommandProcessorException; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.ql.udf.UDFType; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.InputSplit; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.SplitLocationInfo; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.tez.common.security.JobTokenIdentifier; -import org.apache.tez.common.security.JobTokenSecretManager; -import org.apache.tez.dag.api.DAG; -import org.apache.tez.dag.api.InputDescriptor; -import org.apache.tez.dag.api.InputInitializerDescriptor; -import org.apache.tez.dag.api.RootInputLeafOutput; -import org.apache.tez.dag.api.TaskLocationHint; -import org.apache.tez.dag.api.TaskSpecBuilder; -import org.apache.tez.dag.api.Vertex; -import org.apache.tez.mapreduce.grouper.TezSplitGrouper; -import org.apache.tez.runtime.api.Event; -import org.apache.tez.runtime.api.events.InputConfigureVertexTasksEvent; -import org.apache.tez.runtime.api.events.InputDataInformationEvent; -import org.apache.tez.runtime.api.impl.TaskSpec; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Preconditions; - -/** - * GenericUDTFGetSplits. - * - */ -@Description(name = "get_splits", value = "_FUNC_(string,int) - " - + "Returns an array of length int serialized splits for the referenced tables string." - + " Passing length 0 returns only schema data for the compiled query.") -@UDFType(deterministic = false) -public class GenericUDTFGetSplits extends GenericUDTF { - private static final Logger LOG = LoggerFactory.getLogger(GenericUDTFGetSplits.class); - private static String sha = null; - - protected transient StringObjectInspector stringOI; - protected transient IntObjectInspector intOI; - protected transient JobConf jc; - private boolean limitQuery; - protected ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); - protected DataOutput dos = new DataOutputStream(bos); - protected String inputArgQuery; - protected int inputArgNumSplits; - protected boolean schemaSplitOnly; - - @Override - public StructObjectInspector initialize(ObjectInspector[] arguments) - throws UDFArgumentException { - LOG.debug("initializing GenericUDFGetSplits"); - validateInput(arguments); - - List names = Arrays.asList("split"); - List fieldOIs = Arrays - . asList(PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector); - StructObjectInspector outputOI = ObjectInspectorFactory - .getStandardStructObjectInspector(names, fieldOIs); - - LOG.debug("done initializing GenericUDFGetSplits"); - return outputOI; - } - - protected void validateInput(ObjectInspector[] arguments) - throws UDFArgumentLengthException, UDFArgumentTypeException { - - if (SessionState.get() == null || SessionState.get().getConf() == null) { - throw new IllegalStateException("Cannot run get splits outside HS2"); - } - - LOG.debug("Initialized conf, jc and metastore connection"); - - if (arguments.length != 2) { - throw new UDFArgumentLengthException( - "The function GET_SPLITS accepts 2 arguments."); - } else if (!(arguments[0] instanceof StringObjectInspector)) { - LOG.error("Got " + arguments[0].getTypeName() + " instead of string."); - throw new UDFArgumentTypeException(0, "\"" - + "string\" is expected at function GET_SPLITS, " + "but \"" - + arguments[0].getTypeName() + "\" is found"); - } else if (!(arguments[1] instanceof IntObjectInspector)) { - LOG.error("Got " + arguments[1].getTypeName() + " instead of int."); - throw new UDFArgumentTypeException(1, "\"" - + "int\" is expected at function GET_SPLITS, " + "but \"" - + arguments[1].getTypeName() + "\" is found"); - } - - stringOI = (StringObjectInspector) arguments[0]; - intOI = (IntObjectInspector) arguments[1]; - } - - public static class PlanFragment { - public JobConf jc; - public TezWork work; - public Schema schema; - - public PlanFragment(TezWork work, Schema schema, JobConf jc) { - this.work = work; - this.schema = schema; - this.jc = jc; - } - } - - @Override - public void process(Object[] arguments) throws HiveException { - initArgs(arguments); - try { - SplitResult splitResult = getSplitResult(false); - InputSplit[] splits = schemaSplitOnly ? new InputSplit[]{splitResult.schemaSplit} : splitResult.actualSplits; - for (InputSplit s : splits) { - Object[] os = new Object[1]; - bos.reset(); - s.write(dos); - byte[] frozen = bos.toByteArray(); - os[0] = frozen; - forward(os); - } - } catch (Exception e) { - throw new HiveException(e); - } - } - - protected void initArgs(Object[] arguments) { - inputArgQuery = stringOI.getPrimitiveJavaObject(arguments[0]); - inputArgNumSplits = intOI.get(arguments[1]); - schemaSplitOnly = inputArgNumSplits == 0; - } - - protected SplitResult getSplitResult(boolean generateLightWeightSplits) - throws HiveException, IOException { - - // Generate extClientAppId for the LLAP splits - LlapCoordinator coordinator = LlapCoordinator.getInstance(); - if (coordinator == null) { - throw new HiveException("LLAP coordinator is not initialized; must be running in HS2 with " - + ConfVars.LLAP_HS2_ENABLE_COORDINATOR.varname + " enabled"); - } - ApplicationId extClientAppId = coordinator.createExtClientAppId(); - String externalDagName = SessionState.get().getConf().getVar(ConfVars.HIVE_QUERY_NAME); - - StringBuilder sb = new StringBuilder(); - sb.append("Generated appID ").append(extClientAppId.toString()).append(" for LLAP splits"); - if (externalDagName != null) { - sb.append(", with externalID ").append(externalDagName); - } - LOG.info(sb.toString()); - - PlanFragment fragment = createPlanFragment(inputArgQuery, extClientAppId); - TezWork tezWork = fragment.work; - Schema schema = fragment.schema; - - SplitResult splitResult = getSplits(jc, tezWork, schema, extClientAppId, generateLightWeightSplits); - validateSplitResult(splitResult, generateLightWeightSplits); - return splitResult; - } - - private void validateSplitResult(SplitResult splitResult, boolean generateLightWeightSplits) { - Preconditions.checkNotNull(splitResult.schemaSplit, "schema split cannot be null"); - if (!schemaSplitOnly) { - InputSplit[] splits = splitResult.actualSplits; - if (splits.length > 0 && generateLightWeightSplits) { - Preconditions.checkNotNull(splitResult.planSplit, "plan split cannot be null"); - } - LOG.info("Generated {} splits for query {}", splits.length, inputArgQuery); - } - } - - private PlanFragment createPlanFragment(String query, ApplicationId splitsAppId) - throws HiveException { - - HiveConf conf = new HiveConf(SessionState.get().getConf()); - HiveConf.setVar(conf, ConfVars.HIVE_FETCH_TASK_CONVERSION, "none"); - HiveConf.setVar(conf, ConfVars.HIVE_QUERY_RESULT_FILEFORMAT, PlanUtils.LLAP_OUTPUT_FORMAT_KEY); - - String originalMode = HiveConf.getVar(conf, - ConfVars.HIVE_EXECUTION_MODE); - HiveConf.setVar(conf, ConfVars.HIVE_EXECUTION_MODE, "llap"); - HiveConf.setBoolVar(conf, ConfVars.HIVE_TEZ_GENERATE_CONSISTENT_SPLITS, true); - HiveConf.setBoolVar(conf, ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS, true); - conf.setBoolean(TezSplitGrouper.TEZ_GROUPING_NODE_LOCAL_ONLY, true); - // Tez/LLAP requires RPC query plan - HiveConf.setBoolVar(conf, ConfVars.HIVE_RPC_QUERY_PLAN, true); - HiveConf.setBoolVar(conf, ConfVars.HIVE_QUERY_RESULTS_CACHE_ENABLED, false); - - if (schemaSplitOnly) { - //Schema only - try { - List fieldSchemas = ParseUtils.parseQueryAndGetSchema(conf, query); - Schema schema = new Schema(convertSchema(fieldSchemas)); - return new PlanFragment(null, schema, null); - } catch (ParseException e) { - throw new HiveException(e); - } - } - - try { - jc = DagUtils.getInstance().createConfiguration(conf); - } catch (IOException e) { - throw new HiveException(e); - } - - // Instantiate Driver to compile the query passed in. - // This UDF is running as part of an existing query, which may already be using the - // SessionState TxnManager. If this new Driver also tries to use the same TxnManager - // then this may mess up the existing state of the TxnManager. - // So initialize the new Driver with a new TxnManager so that it does not use the - // Session TxnManager that is already in use. - HiveTxnManager txnManager = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); - Driver driver = new Driver(new QueryState.Builder().withHiveConf(conf).nonIsolated().build(), null, txnManager); - DriverCleanup driverCleanup = new DriverCleanup(driver, txnManager, splitsAppId.toString()); - boolean needsCleanup = true; - try { - try { - driver.compileAndRespond(query, false); - } catch (CommandProcessorException e) { - throw new HiveException("Failed to compile query", e); - } - - QueryPlan plan = driver.getPlan(); - limitQuery = plan.getQueryProperties().getOuterQueryLimit() != -1; - List> roots = plan.getRootTasks(); - Schema schema = convertSchema(plan.getResultSchema()); - boolean fetchTask = plan.getFetchTask() != null; - TezWork tezWork; - if (roots == null || roots.size() != 1 || !(roots.get(0) instanceof TezTask)) { - // fetch task query - if (fetchTask) { - tezWork = null; - } else { - throw new HiveException("Was expecting a single TezTask or FetchTask."); - } - } else { - tezWork = ((TezTask) roots.get(0)).getWork(); - } - // A simple limit query (select * from table limit n) generates only mapper work (no reduce phase). - // This can create multiple splits ignoring limit constraint, and multiple llap daemons working on those splits - // return more than "n" rows. Therefore, a limit query needs to be materialized. - if (tezWork == null || tezWork.getAllWork().size() != 1 || limitQuery) { - String tableName = "table_" + UUID.randomUUID().toString().replaceAll("-", ""); - - String storageFormatString = getTempTableStorageFormatString(conf); - String ctas = "create temporary table " + tableName + " " + storageFormatString + " as " + query; - LOG.info("Materializing the query for LLAPIF; CTAS: " + ctas); - driver.releaseLocksAndCommitOrRollback(false); - driver.releaseResources(); - HiveConf.setVar(conf, ConfVars.HIVE_EXECUTION_MODE, originalMode); - try { - driver.run(ctas); - } catch (CommandProcessorException e) { - throw new HiveException("Failed to create temp table [" + tableName + "]", e); - } - - HiveConf.setVar(conf, ConfVars.HIVE_EXECUTION_MODE, "llap"); - query = "select * from " + tableName; - try { - driver.compileAndRespond(query, true); - } catch (CommandProcessorException e) { - throw new HiveException("Failed to select from table [" + tableName + "]", e); - } - - plan = driver.getPlan(); - roots = plan.getRootTasks(); - schema = convertSchema(plan.getResultSchema()); - - if (roots == null || roots.size() != 1 || !(roots.get(0) instanceof TezTask)) { - throw new HiveException("Was expecting a single TezTask."); - } - - tezWork = ((TezTask)roots.get(0)).getWork(); - } else { - // Table will be queried directly by LLAP - // Acquire locks if necessary - they will be released during session cleanup. - // The read will have READ_COMMITTED level semantics. - try { - driver.lockAndRespond(); - } catch (CommandProcessorException cpr1) { - throw new HiveException("Failed to acquire locks", cpr1); - } - - // Attach the resources to the session cleanup. - SessionState.get().addCleanupItem(driverCleanup); - needsCleanup = false; - } - - // Pass the ValidTxnList and ValidTxnWriteIdList snapshot configurations corresponding to the input query - HiveConf driverConf = driver.getConf(); - String validTxnString = driverConf.get(ValidTxnList.VALID_TXNS_KEY); - if (validTxnString != null) { - jc.set(ValidTxnList.VALID_TXNS_KEY, validTxnString); - } - String validWriteIdString = driverConf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY); - if (validWriteIdString != null) { - assert validTxnString != null; - jc.set(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY, validWriteIdString); - } - - return new PlanFragment(tezWork, schema, jc); - } finally { - if (needsCleanup) { - if (driverCleanup != null) { - try { - driverCleanup.close(); - } catch (IOException err) { - throw new HiveException(err); - } - } else if (driver != null) { - driver.close(); - driver.destroy(); - } - } - } - } - - - // generateLightWeightSplits - if true then - // 1) schema and planBytes[] in each LlapInputSplit are not populated - // 2) schemaSplit(contains only schema) and planSplit(contains only planBytes[]) are populated in SplitResult - private SplitResult getSplits(JobConf job, TezWork work, Schema schema, ApplicationId extClientAppId, - boolean generateLightWeightSplits) throws IOException { - - SplitResult splitResult = new SplitResult(); - splitResult.schemaSplit = new LlapInputSplit( - 0, new byte[0], new byte[0], new byte[0], - new SplitLocationInfo[0], new LlapDaemonInfo[0], schema, "", new byte[0], ""); - if (schemaSplitOnly) { - // schema only - return splitResult; - } - - DAG dag = DAG.create(work.getName()); - dag.setCredentials(job.getCredentials()); - - DagUtils utils = DagUtils.getInstance(); - Context ctx = new Context(job); - MapWork mapWork = (MapWork) work.getAllWork().get(0); - // bunch of things get setup in the context based on conf but we need only the MR tmp directory - // for the following method. - JobConf wxConf = utils.initializeVertexConf(job, ctx, mapWork); - // TODO: should we also whitelist input formats here? from mapred.input.format.class - Path scratchDir = utils.createTezDir(ctx.getMRScratchDir(), job); - try { - LocalResource appJarLr = createJarLocalResource(utils.getExecJarPathLocal(ctx.getConf()), utils, job); - - LlapCoordinator coordinator = LlapCoordinator.getInstance(); - if (coordinator == null) { - throw new IOException("LLAP coordinator is not initialized; must be running in HS2 with " - + ConfVars.LLAP_HS2_ENABLE_COORDINATOR.varname + " enabled"); - } - - // Update the queryId to use the generated extClientAppId. See comment below about - // why this is done. - HiveConf.setVar(wxConf, HiveConf.ConfVars.HIVE_QUERY_ID, extClientAppId.toString()); - Vertex wx = utils.createVertex(wxConf, mapWork, scratchDir, work, - DagUtils.createTezLrMap(appJarLr, null)); - String vertexName = wx.getName(); - dag.addVertex(wx); - utils.addCredentials(mapWork, dag, job); - - - // we have the dag now proceed to get the splits: - Preconditions.checkState(HiveConf.getBoolVar(wxConf, - ConfVars.HIVE_TEZ_GENERATE_CONSISTENT_SPLITS)); - Preconditions.checkState(HiveConf.getBoolVar(wxConf, - ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS)); - - // we're not interested in split fs serialization optimization in this case - // it was implemented for split generation in TezAM - // this can be removed any time when it turns out that's needed here too - HiveConf.setIntVar(wxConf, HiveConf.ConfVars.HIVE_TEZ_INPUT_FS_SERIALIZATION_THRESHOLD, -1); - - HiveSplitGenerator splitGenerator = - new HiveSplitGenerator(wxConf, mapWork, false, inputArgNumSplits); - List eventList = splitGenerator.initialize(); - int numGroupedSplitsGenerated = eventList.size() - 1; - InputSplit[] result = new InputSplit[numGroupedSplitsGenerated]; - - InputConfigureVertexTasksEvent configureEvent - = (InputConfigureVertexTasksEvent) eventList.get(0); - - List hints = configureEvent.getLocationHint().getTaskLocationHints(); - - Preconditions.checkState(hints.size() == numGroupedSplitsGenerated); - - if (LOG.isDebugEnabled()) { - LOG.debug("NumEvents=" + eventList.size() + ", NumSplits=" + result.length); - } - - // This assumes LLAP cluster owner is always the HS2 user. - String llapUser = LlapRegistryService.currentUser(); - - String queryUser = null; - byte[] tokenBytes = null; - LlapSigner signer = null; - if (UserGroupInformation.isSecurityEnabled()) { - signer = coordinator.getLlapSigner(job); - - // 1. Generate the token for query user (applies to all splits). - queryUser = SessionState.getUserFromAuthenticator(); - if (queryUser == null) { - queryUser = UserGroupInformation.getCurrentUser().getUserName(); - LOG.warn("Cannot determine the session user; using " + queryUser + " instead"); - } - LlapTokenLocalClient tokenClient = coordinator.getLocalTokenClient(job, llapUser); - // We put the query user, not LLAP user, into the message and token. - Token token = tokenClient.createToken( - extClientAppId.toString(), queryUser, true); - LOG.info("Created the token for remote user: {}", token); - bos.reset(); - token.write(dos); - tokenBytes = bos.toByteArray(); - } else { - queryUser = UserGroupInformation.getCurrentUser().getUserName(); - } - - // Generate umbilical token (applies to all splits) - Token umbilicalToken = JobTokenCreator.createJobToken(extClientAppId); - - LOG.info("Number of splits: " + numGroupedSplitsGenerated); - SignedMessage signedSvs = null; - byte[] submitWorkBytes = null; - final byte[] emptySubmitWorkBytes = new byte[0]; - final Schema emptySchema = new Schema(); - for (int i = 0; i < numGroupedSplitsGenerated; i++) { - TaskSpec taskSpec = new TaskSpecBuilder().constructTaskSpec(dag, vertexName, - numGroupedSplitsGenerated, extClientAppId, i); - - // 2. Generate the vertex/submit information for all events. - if (i == 0) { - // The queryId could either be picked up from the current request being processed, or - // generated. The current request isn't exactly correct since the query is 'done' once we - // return the results. Generating a new one has the added benefit of working once this - // is moved out of a UDTF into a proper API. - // Setting this to the generated AppId which is unique. - // Despite the differences in TaskSpec, the vertex spec should be the same. - signedSvs = createSignedVertexSpec(signer, taskSpec, extClientAppId, queryUser, - extClientAppId.toString()); - SubmitWorkInfo submitWorkInfo = new SubmitWorkInfo(extClientAppId, - System.currentTimeMillis(), numGroupedSplitsGenerated, signedSvs.message, - signedSvs.signature, umbilicalToken); - submitWorkBytes = SubmitWorkInfo.toBytes(submitWorkInfo); - if (generateLightWeightSplits) { - splitResult.planSplit = new LlapInputSplit( - 0, submitWorkBytes, new byte[0], new byte[0], - new SplitLocationInfo[0], new LlapDaemonInfo[0], new Schema(), "", new byte[0], ""); - } - } - - // 3. Generate input event. - SignedMessage eventBytes = makeEventBytes(wx, vertexName, eventList.get(i + 1), signer); - - // 4. Make location hints. - SplitLocationInfo[] locations = makeLocationHints(hints.get(i)); - - // 5. populate info about llap daemons(to help client submit request and read data) - LlapDaemonInfo[] llapDaemonInfos = populateLlapDaemonInfos(job, locations); - - // 6. Generate JWT for external clients if it's a cloud deployment - // we inject extClientAppId in JWT which is same as what fragment contains. - // extClientAppId in JWT and in fragment are compared on LLAP when a fragment is submitted. - // see method ContainerRunnerImpl#verifyJwtForExternalClient - String jwt = ""; - if (LlapUtil.isCloudDeployment(job)) { - LlapExtClientJwtHelper llapExtClientJwtHelper = new LlapExtClientJwtHelper(job); - jwt = llapExtClientJwtHelper.buildJwtForLlap(extClientAppId); - } - - if (generateLightWeightSplits) { - result[i] = new LlapInputSplit(i, emptySubmitWorkBytes, eventBytes.message, - eventBytes.signature, locations, llapDaemonInfos, emptySchema, llapUser, tokenBytes, jwt); - } else { - result[i] = new LlapInputSplit(i, submitWorkBytes, eventBytes.message, - eventBytes.signature, locations, llapDaemonInfos, schema, llapUser, tokenBytes, jwt); - } - } - splitResult.actualSplits = result; - return splitResult; - } catch (Exception e) { - throw new IOException(e); - } - } - - static class SplitResult { - InputSplit schemaSplit; - InputSplit planSplit; - InputSplit[] actualSplits; - } - - private static class DriverCleanup implements Closeable { - private final Driver driver; - private final HiveTxnManager txnManager; - private final String applicationId; - - public DriverCleanup(Driver driver, HiveTxnManager txnManager, String applicationId) { - this.driver = driver; - this.txnManager = txnManager; - this.applicationId = applicationId; - } - - @Override - public void close() throws IOException { - try { - LOG.info("DriverCleanup for LLAP splits: {}", applicationId); - driver.releaseLocksAndCommitOrRollback(true); - driver.close(); - driver.destroy(); - txnManager.closeTxnManager(); - } catch (Exception err) { - LOG.error("Error closing driver resources", err); - throw new IOException(err); - } - } - - @Override - public String toString() { - return "DriverCleanup for LLAP splits: " + applicationId; - } - } - - private static class JobTokenCreator { - private static Token createJobToken(ApplicationId applicationId) { - String tokenIdentifier = applicationId.toString(); - JobTokenIdentifier identifier = new JobTokenIdentifier(new Text( - tokenIdentifier)); - Token sessionToken = new Token(identifier, - new JobTokenSecretManager(new Configuration())); - sessionToken.setService(identifier.getJobId()); - return sessionToken; - } - } - - private SplitLocationInfo[] makeLocationHints(TaskLocationHint hint) { - Set hosts = hint.getHosts(); - if (hosts == null) { - LOG.warn("No hosts"); - return new SplitLocationInfo[0]; - } - if (hosts.size() != 1) { - LOG.warn("Bad # of locations: " + hosts.size()); - } - SplitLocationInfo[] locations = new SplitLocationInfo[hosts.size()]; - int j = 0; - for (String host : hosts) { - locations[j++] = new SplitLocationInfo(host, false); - } - return locations; - } - - private LlapDaemonInfo[] populateLlapDaemonInfos(JobConf job, SplitLocationInfo[] locations) throws IOException { - LlapRegistryService registryService = LlapRegistryService.getClient(job); - LlapServiceInstanceSet instanceSet = registryService.getInstances(); - Collection llapServiceInstances = null; - - //this means a valid location, see makeLocationHints() - if (locations.length == 1 && locations[0].getLocation() != null) { - llapServiceInstances = instanceSet.getByHost(locations[0].getLocation()); - } - - //okay, so we were unable to find any llap instance by hostname - //let's populate them all so that we can fetch data from any of them. - if (CollectionUtils.isEmpty(llapServiceInstances)) { - llapServiceInstances = instanceSet.getAll(); - } - - Preconditions.checkState(llapServiceInstances.size() > 0, - "Unable to find any of the llap instances in zk registry"); - - LlapDaemonInfo[] llapDaemonInfos = new LlapDaemonInfo[llapServiceInstances.size()]; - int count = 0; - for (LlapServiceInstance inst : llapServiceInstances) { - LlapDaemonInfo info; - if (LlapUtil.isCloudDeployment(job)) { - info = new LlapDaemonInfo(inst.getExternalHostname(), inst.getExternalClientsRpcPort(), inst.getOutputFormatPort()); - } else { - info = new LlapDaemonInfo(inst.getHost(), inst.getRpcPort(), inst.getOutputFormatPort()); - } - llapDaemonInfos[count++] = info; - } - return llapDaemonInfos; - } - - private SignedMessage makeEventBytes(Vertex wx, String vertexName, - Event event, LlapSigner signer) throws IOException { - assert event instanceof InputDataInformationEvent; - List> inputs = - TaskSpecBuilder.getVertexInputs(wx); - Preconditions.checkState(inputs.size() == 1); - - Signable signableNte = NotTezEventHelper.createSignableNotTezEvent( - (InputDataInformationEvent)event, vertexName, inputs.get(0).getName()); - if (signer != null) { - return signer.serializeAndSign(signableNte); - } else { - SignedMessage sm = new SignedMessage(); - sm.message = signableNte.serialize(); - return sm; - } - } - - private SignedMessage createSignedVertexSpec(LlapSigner signer, TaskSpec taskSpec, - ApplicationId applicationId, String queryUser, String queryIdString) throws IOException { - QueryIdentifierProto queryIdentifierProto = - QueryIdentifierProto.newBuilder().setApplicationIdString(applicationId.toString()) - .setDagIndex(taskSpec.getDagIdentifier()).setAppAttemptNumber(0).build(); - final SignableVertexSpec.Builder svsb = Converters.constructSignableVertexSpec( - taskSpec, queryIdentifierProto, applicationId.toString(), queryUser, queryIdString); - svsb.setIsExternalSubmission(true); - if (signer == null) { - SignedMessage result = new SignedMessage(); - result.message = serializeVertexSpec(svsb); - return result; - } - return signer.serializeAndSign(new Signable() { - @Override - public void setSignInfo(int masterKeyId) { - svsb.setSignatureKeyId(masterKeyId); - } - - @Override - public byte[] serialize() throws IOException { - return serializeVertexSpec(svsb); - } - }); - } - - private static byte[] serializeVertexSpec(SignableVertexSpec.Builder svsb) throws IOException { - ByteArrayOutputStream os = new ByteArrayOutputStream(); - svsb.build().writeTo(os); - return os.toByteArray(); - } - - /** - * Returns a local resource representing a jar. This resource will be used to - * execute the plan on the cluster. - * - * @param localJarPath - * Local path to the jar to be localized. - * @return LocalResource corresponding to the localized hive exec resource. - * @throws IOException - * when any file system related call fails. - * @throws URISyntaxException - * when current jar location cannot be determined. - */ - private LocalResource createJarLocalResource(String localJarPath, - DagUtils utils, Configuration conf) throws IOException, - IllegalArgumentException, FileNotFoundException { - FileStatus destDirStatus = utils.getHiveJarDirectory(conf); - assert destDirStatus != null; - Path destDirPath = destDirStatus.getPath(); - - Path localFile = new Path(localJarPath); - if (sha == null || !destDirPath.toString().contains(sha)) { - sha = getSha(localFile, conf); - } - - String destFileName = localFile.getName(); - - // Now, try to find the file based on SHA and name. Currently we require - // exact name match. - // We could also allow cutting off versions and other stuff provided that - // SHA matches... - destFileName = FilenameUtils.removeExtension(destFileName) + "-" + sha - + FilenameUtils.EXTENSION_SEPARATOR - + FilenameUtils.getExtension(destFileName); - - // TODO: if this method is ever called on more than one jar, getting the dir - // and the - // list need to be refactored out to be done only once. - Path destFile = new Path(destDirPath.toString() + "/" + destFileName); - return utils.localizeResource(localFile, destFile, LocalResourceType.FILE, - conf); - } - - private String getSha(Path localFile, Configuration conf) throws IOException, - IllegalArgumentException { - InputStream is = null; - try { - FileSystem localFs = FileSystem.getLocal(conf); - is = localFs.open(localFile); - return DigestUtils.sha256Hex(is); - } finally { - if (is != null) { - is.close(); - } - } - } - - private List convertSchema(List fieldSchemas) { - List colDescs = new ArrayList(); - for (FieldSchema fs : fieldSchemas) { - String colName = fs.getName(); - String typeString = fs.getType(); - colDescs.add(new FieldDesc(colName, TypeInfoUtils.getTypeInfoFromTypeString(typeString))); - } - return colDescs; - } - - private Schema convertSchema(org.apache.hadoop.hive.metastore.api.Schema schema) { - return new Schema(convertSchema(schema.getFieldSchemas())); - } - - private String getTempTableStorageFormatString(HiveConf conf) { - String formatString = ""; - String storageFormatOption = - conf.getVar(HiveConf.ConfVars.LLAP_EXTERNAL_SPLITS_TEMP_TABLE_STORAGE_FORMAT).toLowerCase(); - if (storageFormatOption.equals("text")) { - formatString = "stored as textfile"; - } else if (storageFormatOption.equals("orc")) { - formatString = "stored as orc"; - } - return formatString; - } - - @Override - public void close() throws HiveException { - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits2.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits2.java deleted file mode 100644 index f703c21f3748..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits2.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.ql.udf.generic; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.UDFArgumentException; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.udf.UDFType; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.mapred.InputSplit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * GenericUDTFGetSplits2 - Memory efficient version of GenericUDTFGetSplits. - * It separates out information like schema and planBytes[] which is common to all the splits. - * This produces output in following format. - *

- * type split - * ---------------------------------------------------- - * schema-split LlapInputSplit -- contains only schema - * plan-split LlapInputSplit -- contains only planBytes[] - * 0 LlapInputSplit -- actual split 1 - * 1 LlapInputSplit -- actual split 2 - * ... ... - */ -@Description(name = "get_llap_splits", value = "_FUNC_(string,int) - " - + "Returns an array of length int serialized splits for the referenced tables string." - + " Passing length 0 returns only schema data for the compiled query. " - + "The order of splits is: schema-split, plan-split, 0, 1, 2...where 0, 1, 2...are the actual splits " - + "This UDTF is for internal use by LlapBaseInputFormat and not to be invoked explicitly") -@UDFType(deterministic = false) -public class GenericUDTFGetSplits2 extends GenericUDTFGetSplits { - private static final Logger LOG = LoggerFactory.getLogger(GenericUDTFGetSplits2.class); - - @Override public StructObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { - LOG.debug("initializing GenericUDFGetSplits2"); - validateInput(arguments); - - List names = Arrays.asList("type", "split"); - List fieldOIs = Arrays.asList(PrimitiveObjectInspectorFactory.javaStringObjectInspector, - PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector); - StructObjectInspector outputOI = ObjectInspectorFactory.getStandardStructObjectInspector(names, fieldOIs); - - LOG.debug("done initializing GenericUDFGetSplits2"); - return outputOI; - } - - @Override public void process(Object[] arguments) throws HiveException { - try { - initArgs(arguments); - SplitResult splitResult = getSplitResult(true); - forwardOutput(splitResult); - } catch (Exception e) { - throw new HiveException(e); - } - } - - private void forwardOutput(SplitResult splitResult) throws IOException, HiveException { - for (Map.Entry entry : transformSplitResult(splitResult).entrySet()) { - Object[] os = new Object[2]; - os[0] = entry.getKey(); - InputSplit split = entry.getValue(); - bos.reset(); - split.write(dos); - os[1] = bos.toByteArray(); - forward(os); - } - } - - private Map transformSplitResult(SplitResult splitResult) { - Map splitMap = new LinkedHashMap<>(); - splitMap.put("schema-split", splitResult.schemaSplit); - if (splitResult.actualSplits != null && splitResult.actualSplits.length > 0) { - Preconditions.checkNotNull(splitResult.planSplit); - splitMap.put("plan-split", splitResult.planSplit); - for (int i = 0; i < splitResult.actualSplits.length; i++) { - splitMap.put("" + i, splitResult.actualSplits[i]); - } - } - return splitMap; - } -} diff --git a/ql/src/test/org/apache/hadoop/hive/llap/TestLlapOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/llap/TestLlapOutputFormat.java deleted file mode 100644 index 58044557002b..000000000000 --- a/ql/src/test/org/apache/hadoop/hive/llap/TestLlapOutputFormat.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.BeforeClass; -import org.junit.AfterClass; - -import java.net.Socket; - -import java.io.OutputStream; -import java.io.InputStream; -import java.io.IOException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapred.RecordWriter; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.llap.LlapBaseRecordReader.ReaderEvent; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.LlapOutputSocketInitMessage; -import org.apache.hadoop.hive.llap.io.api.LlapProxy; - -public class TestLlapOutputFormat { - - private static final Logger LOG = LoggerFactory.getLogger(TestLlapOutputFormat.class); - - private static LlapOutputFormatService service; - - @BeforeClass - public static void setUp() throws Exception { - LOG.debug("Setting up output service"); - Configuration conf = new Configuration(); - // Pick random avail port - HiveConf.setIntVar(conf, HiveConf.ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT, 0); - LlapOutputFormatService.initializeAndStart(conf, null); - service = LlapOutputFormatService.get(); - LlapProxy.setDaemon(true); - LOG.debug("Output service up"); - } - - @AfterClass - public static void tearDown() throws IOException, InterruptedException { - LOG.debug("Tearing down service"); - service.stop(); - LOG.debug("Tearing down complete"); - } - - @Test - public void testValues() throws Exception { - JobConf job = new JobConf(); - - for (int k = 0; k < 5; ++k) { - String id = "foobar" + k; - job.set(LlapOutputFormat.LLAP_OF_ID_KEY, id); - LlapOutputFormat format = new LlapOutputFormat(); - - HiveConf conf = new HiveConf(); - Socket socket = new Socket("localhost", service.getPort()); - - LOG.debug("Socket connected"); - - OutputStream socketStream = socket.getOutputStream(); - LlapOutputSocketInitMessage.newBuilder() - .setFragmentId(id).build().writeDelimitedTo(socketStream); - socketStream.flush(); - - Thread.sleep(3000); - - LOG.debug("Data written"); - - RecordWriter writer = format.getRecordWriter(null, job, null, null); - Text text = new Text(); - - LOG.debug("Have record writer"); - - for (int i = 0; i < 10; ++i) { - text.set(""+i); - writer.write(NullWritable.get(),text); - } - - writer.close(null); - - InputStream in = socket.getInputStream(); - LlapBaseRecordReader reader = new LlapBaseRecordReader( - in, null, Text.class, job, null, null); - - LOG.debug("Have record reader"); - - // Send done event, which LlapRecordReader is expecting upon end of input - reader.handleEvent(ReaderEvent.doneEvent()); - - int count = 0; - while(reader.next(NullWritable.get(), text)) { - LOG.debug(text.toString()); - count++; - } - - reader.close(); - - Assert.assertEquals(10, count); - } - } - - - @Test - public void testBadClientMessage() throws Exception { - JobConf job = new JobConf(); - String id = "foobar"; - job.set(LlapOutputFormat.LLAP_OF_ID_KEY, id); - LlapOutputFormat format = new LlapOutputFormat(); - - Socket socket = new Socket("localhost", service.getPort()); - - LOG.debug("Socket connected"); - - OutputStream socketStream = socket.getOutputStream(); - LlapOutputSocketInitMessage.newBuilder() - .setFragmentId(id).build().writeDelimitedTo(socketStream); - LlapOutputSocketInitMessage.newBuilder() - .setFragmentId(id).build().writeDelimitedTo(socketStream); - socketStream.flush(); - - Thread.sleep(3000); - - LOG.debug("Data written"); - - try { - format.getRecordWriter(null, job, null, null); - Assert.fail("Didn't throw"); - } catch (IOException ex) { - // Expected. - } - } -} diff --git a/ql/src/test/org/apache/hadoop/hive/llap/TestProactiveEviction.java b/ql/src/test/org/apache/hadoop/hive/llap/TestProactiveEviction.java index 322dbeb56f54..1f62f429df66 100644 --- a/ql/src/test/org/apache/hadoop/hive/llap/TestProactiveEviction.java +++ b/ql/src/test/org/apache/hadoop/hive/llap/TestProactiveEviction.java @@ -184,8 +184,6 @@ private PersistentEphemeralNode createZnode(String workersPath, String id) throw RegistryTypeUtils.ipcEndpoint("shuffle", new InetSocketAddress("localhost", 4001))); serviceRecord.addInternalEndpoint( RegistryTypeUtils.ipcEndpoint("llapmng", new InetSocketAddress("localhost", 4002))); - serviceRecord.addInternalEndpoint( - RegistryTypeUtils.ipcEndpoint("llapoutputformat", new InetSocketAddress("localhost", 4003))); serviceRecord.addExternalEndpoint( RegistryTypeUtils.webEndpoint("services", new URI("http://localhost:4004"))); serviceRecord.set(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS, "10"); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java index 63bc713bb5aa..c8a060f57b04 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java @@ -95,7 +95,6 @@ public void testGetSplitLocationProvider() throws IOException, URISyntaxExceptio Endpoint rpcEndpoint = RegistryTypeUtils.ipcEndpoint("llap", new InetSocketAddress(ACTIVE, 4000)); Endpoint shuffle = RegistryTypeUtils.ipcEndpoint("shuffle", new InetSocketAddress(ACTIVE, 4000)); Endpoint mng = RegistryTypeUtils.ipcEndpoint("llapmng", new InetSocketAddress(ACTIVE, 4000)); - Endpoint outputFormat = RegistryTypeUtils.ipcEndpoint("llapoutputformat", new InetSocketAddress(ACTIVE, 4000)); Endpoint services = RegistryTypeUtils.webEndpoint("services", new URI(ACTIVE + ":4000")); // Set 1 active instance @@ -103,7 +102,6 @@ public void testGetSplitLocationProvider() throws IOException, URISyntaxExceptio enabledSrv.addInternalEndpoint(rpcEndpoint); enabledSrv.addInternalEndpoint(shuffle); enabledSrv.addInternalEndpoint(mng); - enabledSrv.addInternalEndpoint(outputFormat); enabledSrv.addExternalEndpoint(services); enabledSrv.set(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS, 10); diff --git a/ql/src/test/queries/clientpositive/get_splits_0.q b/ql/src/test/queries/clientpositive/get_splits_0.q deleted file mode 100644 index e585fda78fd4..000000000000 --- a/ql/src/test/queries/clientpositive/get_splits_0.q +++ /dev/null @@ -1,3 +0,0 @@ ---! qt:dataset:src -select get_splits("SELECT * FROM src WHERE value in (SELECT value FROM src)",0); -select get_splits("SELECT key AS `key 1`, value AS `value 1` FROM src",0); diff --git a/ql/src/test/results/clientpositive/llap/get_splits_0.q.out b/ql/src/test/results/clientpositive/llap/get_splits_0.q.out deleted file mode 100644 index 48533e1f48c9..000000000000 Binary files a/ql/src/test/results/clientpositive/llap/get_splits_0.q.out and /dev/null differ diff --git a/ql/src/test/results/clientpositive/llap/show_functions.q.out b/ql/src/test/results/clientpositive/llap/show_functions.q.out index 605e953d6349..f45f6b9ec405 100644 --- a/ql/src/test/results/clientpositive/llap/show_functions.q.out +++ b/ql/src/test/results/clientpositive/llap/show_functions.q.out @@ -208,8 +208,6 @@ format_number from_unixtime from_utc_timestamp get_json_object -get_llap_splits -get_splits get_sql_schema greatest grouping @@ -850,8 +848,6 @@ format_number from_unixtime from_utc_timestamp get_json_object -get_llap_splits -get_splits get_sql_schema greatest grouping