[WIP]

bin/hive

@@ -50,10 +50,6 @@ while [ $# -gt 0 ]; do
       SERVICE=orcfiledump
       shift
       ;;
-    --llapdump)
-      SERVICE=llapdump
-      shift
-      ;;
     --replMigration)
       SERVICE=replMigration
       shift

common/src/java/org/apache/hadoop/hive/conf/HiveConf.java

@@ -130,12 +130,6 @@ public String toString() {
         public String toString() {
           return "RCfile";
         }
-    },
-    LLAP {
-      @Override
-        public String toString() {
-          return "Llap";
-        }
     };
 
     public static ResultFileFormat getInvalid() {
@@ -5357,48 +5351,6 @@ public static enum ConfVars {
     LLAP_VALIDATE_ACLS("hive.llap.validate.acls", true,
         "Whether LLAP should reject permissive ACLs in some cases (e.g. its own management\n" +
         "protocol or ZK paths), similar to how ssh refuses a key with bad access permissions."),
-    LLAP_DAEMON_OUTPUT_SERVICE_PORT("hive.llap.daemon.output.service.port", 15003,
-        "LLAP daemon output service port"),
-    LLAP_DAEMON_OUTPUT_STREAM_TIMEOUT("hive.llap.daemon.output.stream.timeout", "120s",
-        new TimeValidator(TimeUnit.SECONDS),
-        "The timeout for the client to connect to LLAP output service and start the fragment\n" +
-        "output after sending the fragment. The fragment will fail if its output is not claimed."),
-    LLAP_DAEMON_OUTPUT_SERVICE_SEND_BUFFER_SIZE("hive.llap.daemon.output.service.send.buffer.size",
-        128 * 1024, "Send buffer size to be used by LLAP daemon output service"),
-    LLAP_DAEMON_OUTPUT_SERVICE_MAX_PENDING_WRITES("hive.llap.daemon.output.service.max.pending.writes",
-        8, "Maximum number of queued writes allowed per connection when sending data\n" +
-        " via the LLAP output service to external clients."),
-    LLAP_EXTERNAL_SPLITS_TEMP_TABLE_STORAGE_FORMAT("hive.llap.external.splits.temp.table.storage.format",
-        "orc", new StringSet("default", "text", "orc"),
-        "Storage format for temp tables created using LLAP external client"),
-    LLAP_EXTERNAL_CLIENT_USE_HYBRID_CALENDAR("hive.llap.external.client.use.hybrid.calendar",
-        false,
-        "Whether to use hybrid calendar for parsing of data/timestamps."),
-
-    // ====== confs for llap-external-client cloud deployment ======
-    LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED(
-        "hive.llap.external.client.cloud.deployment.setup.enabled", false,
-        "Tells whether to enable additional RPC port, auth mechanism for llap external clients. This is meant"
-            + "for cloud based deployments. When true, it has following effects - \n"
-            + "1. Enables an extra RPC port on LLAP daemon to accept fragments from external clients. See"
-            + "hive.llap.external.client.cloud.rpc.port\n"
-            + "2. Uses external hostnames of LLAP in splits, so that clients can submit from outside of cloud. "
-            + "Env variable PUBLIC_HOSTNAME should be available on LLAP machines.\n"
-            + "3. Uses JWT based authentication for splits to be validated at LLAP. See "
-            + "hive.llap.external.client.cloud.jwt.shared.secret.provider"),
-    LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT("hive.llap.external.client.cloud.rpc.port", 30004,
-        "The LLAP daemon RPC port for external clients when llap is running in cloud environment."),
-    LLAP_EXTERNAL_CLIENT_CLOUD_OUTPUT_SERVICE_PORT("hive.llap.external.client.cloud.output.service.port", 30005,
-                "LLAP output service port when llap is running in cloud environment"),
-    LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_PROVIDER(
-        "hive.llap.external.client.cloud.jwt.shared.secret.provider",
-        "org.apache.hadoop.hive.llap.security.DefaultJwtSharedSecretProvider",
-        "Shared secret provider to be used to sign JWT"),
-    LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET("hive.llap.external.client.cloud.jwt.shared.secret",
-        "",
-        "The LLAP daemon RPC port for external clients when llap is running in cloud environment. "
-            + "Length of the secret should be >= 32 bytes"),
-    // ====== confs for llap-external-client cloud deployment ======
 
     LLAP_ENABLE_GRACE_JOIN_IN_LLAP("hive.llap.enable.grace.join.in.llap", false,
         "Override if grace join should be allowed to run in llap."),

itests/hive-unit/pom.xml

@@ -62,10 +62,6 @@
       <groupId>org.apache.hive</groupId>
       <artifactId>hive-llap-server</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.hive</groupId>
-      <artifactId>hive-llap-ext-client</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
       <artifactId>hive-llap-server</artifactId>

itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java

@@ -27,9 +27,7 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.conf.Configuration;
@@ -43,13 +41,8 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.conf.HiveConfForTest;
-import org.apache.hadoop.hive.metastore.api.LockState;
-import org.apache.hadoop.hive.metastore.api.LockType;
 import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
 import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
-import org.apache.hadoop.hive.metastore.api.ShowLocksRequest;
-import org.apache.hadoop.hive.metastore.api.ShowLocksResponse;
-import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil;
 import org.apache.hadoop.hive.metastore.txn.TxnStore;
@@ -747,106 +740,6 @@ public void testBucketedAcidInsertWithRemoveUnion() throws Exception {
     }
   }
 
-  @Test
-  public void testGetSplitsLocks() throws Exception {
-    // Need to test this with LLAP settings, which requires some additional configurations set.
-    hiveConf.setVar(ConfVars.HIVE_FETCH_TASK_CONVERSION, "more");
-    hiveConf.setVar(HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS, "localhost");
-
-    // SessionState/Driver needs to be restarted with the Tez conf settings.
-    restartSessionAndDriver(hiveConf);
-    TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
-
-    try {
-      // Request LLAP splits for a table.
-      String queryParam = "select * from " + Table.ACIDTBL;
-      runStatementOnDriver("select get_splits(\"" + queryParam + "\", 1)");
-
-      // The get_splits call should have resulted in a lock on ACIDTBL
-      ShowLocksResponse slr = txnHandler.showLocks(new ShowLocksRequest());
-      TestTxnDbUtil.checkLock(LockType.SHARED_READ, LockState.ACQUIRED,
-          "default", Table.ACIDTBL.name, null, slr.getLocks());
-      assertEquals(1, slr.getLocksSize());
-
-      // Try another table.
-      queryParam = "select * from " + Table.ACIDTBLPART;
-      runStatementOnDriver("select get_splits(\"" + queryParam + "\", 1)");
-
-      // Should now have new lock on ACIDTBLPART
-      slr = txnHandler.showLocks(new ShowLocksRequest());
-      TestTxnDbUtil.checkLock(LockType.SHARED_READ, LockState.ACQUIRED,
-          "default", Table.ACIDTBLPART.name, null, slr.getLocks());
-      assertEquals(2, slr.getLocksSize());
-
-      // There should be different txn IDs associated with each lock.
-      Set<Long> txnSet = new HashSet<Long>();
-      for (ShowLocksResponseElement lockResponseElem : slr.getLocks()) {
-        txnSet.add(lockResponseElem.getTxnid());
-      }
-      assertEquals(2, txnSet.size());
-
-      List<String> rows = runStatementOnDriver("show transactions");
-      // Header row + 2 transactions = 3 rows
-      assertEquals(3, rows.size());
-    } finally {
-      // Close the session which should free up the TxnHandler/locks held by the session.
-      // Done in the finally block to make sure we free up the locks; otherwise
-      // the cleanup in tearDown() will get stuck waiting on the lock held here on ACIDTBL.
-      restartSessionAndDriver(hiveConf);
-    }
-
-    // Lock should be freed up now.
-    ShowLocksResponse slr = txnHandler.showLocks(new ShowLocksRequest());
-    assertEquals(0, slr.getLocksSize());
-
-    List<String> rows = runStatementOnDriver("show transactions");
-    // Transactions should be committed.
-    // No transactions - just the header row
-    assertEquals(1, rows.size());
-  }
-
-  @Test
-  public void testGetSplitsLocksWithMaterializedView() throws Exception {
-    // Need to test this with LLAP settings, which requires some additional configurations set.
-    hiveConf.setVar(ConfVars.HIVE_FETCH_TASK_CONVERSION, "more");
-    hiveConf.setVar(HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS, "localhost");
-
-    // SessionState/Driver needs to be restarted with the Tez conf settings.
-    restartSessionAndDriver(hiveConf);
-    TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
-    String mvName = "mv_acidTbl";
-    try {
-      runStatementOnDriver("create materialized view " + mvName + " as select a from " + Table.ACIDTBL + " where a > 5");
-
-      // Request LLAP splits for a table.
-      String queryParam = "select a from " + Table.ACIDTBL + " where a > 5";
-      runStatementOnDriver("select get_splits(\"" + queryParam + "\", 1)");
-
-      // The get_splits call should have resulted in a lock on ACIDTBL and materialized view mv_acidTbl
-      ShowLocksResponse slr = txnHandler.showLocks(new ShowLocksRequest());
-      TestTxnDbUtil.checkLock(LockType.SHARED_READ, LockState.ACQUIRED,
-              "default", Table.ACIDTBL.name, null, slr.getLocks());
-      TestTxnDbUtil.checkLock(LockType.SHARED_READ, LockState.ACQUIRED,
-              "default", mvName, null, slr.getLocks());
-      assertEquals(2, slr.getLocksSize());
-    } finally {
-      // Close the session which should free up the TxnHandler/locks held by the session.
-      // Done in the finally block to make sure we free up the locks; otherwise
-      // the cleanup in tearDown() will get stuck waiting on the lock held here on ACIDTBL.
-      restartSessionAndDriver(hiveConf);
-      runStatementOnDriver("drop materialized view if exists " + mvName);
-    }
-
-    // Lock should be freed up now.
-    ShowLocksResponse slr = txnHandler.showLocks(new ShowLocksRequest());
-    assertEquals(0, slr.getLocksSize());
-
-    List<String> rows = runStatementOnDriver("show transactions");
-    // Transactions should be committed.
-    // No transactions - just the header row
-    assertEquals(1, rows.size());
-  }
-
   /**
    * HIVE-20699
    *

itests/hive-unit/src/test/java/org/apache/hive/jdbc/AbstractJdbcTriggersTest.java

@@ -41,7 +41,6 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.LlapBaseInputFormat;
 import org.apache.hadoop.hive.metastore.api.WMTrigger;
 import org.apache.hadoop.hive.ql.wm.Trigger;
 import org.apache.hive.jdbc.miniHS2.MiniHS2;
@@ -99,7 +98,6 @@ public void setUp() throws Exception {
 
   @After
   public void tearDown() throws Exception {
-    LlapBaseInputFormat.closeAll();
     hs2Conn.close();
   }