diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 6268efa4e50a..36b400ce3ce9 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -5381,34 +5381,6 @@ public static enum ConfVars { LLAP_EXTERNAL_SPLITS_TEMP_TABLE_STORAGE_FORMAT("hive.llap.external.splits.temp.table.storage.format", "orc", new StringSet("default", "text", "orc"), "Storage format for temp tables created using LLAP external client"), - LLAP_EXTERNAL_CLIENT_USE_HYBRID_CALENDAR("hive.llap.external.client.use.hybrid.calendar", - false, - "Whether to use hybrid calendar for parsing of data/timestamps."), - - // ====== confs for llap-external-client cloud deployment ====== - LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED( - "hive.llap.external.client.cloud.deployment.setup.enabled", false, - "Tells whether to enable additional RPC port, auth mechanism for llap external clients. This is meant" - + "for cloud based deployments. When true, it has following effects - \n" - + "1. Enables an extra RPC port on LLAP daemon to accept fragments from external clients. See" - + "hive.llap.external.client.cloud.rpc.port\n" - + "2. Uses external hostnames of LLAP in splits, so that clients can submit from outside of cloud. " - + "Env variable PUBLIC_HOSTNAME should be available on LLAP machines.\n" - + "3. Uses JWT based authentication for splits to be validated at LLAP. See " - + "hive.llap.external.client.cloud.jwt.shared.secret.provider"), - LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT("hive.llap.external.client.cloud.rpc.port", 30004, - "The LLAP daemon RPC port for external clients when llap is running in cloud environment."), - LLAP_EXTERNAL_CLIENT_CLOUD_OUTPUT_SERVICE_PORT("hive.llap.external.client.cloud.output.service.port", 30005, - "LLAP output service port when llap is running in cloud environment"), - LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_PROVIDER( - "hive.llap.external.client.cloud.jwt.shared.secret.provider", - "org.apache.hadoop.hive.llap.security.DefaultJwtSharedSecretProvider", - "Shared secret provider to be used to sign JWT"), - LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET("hive.llap.external.client.cloud.jwt.shared.secret", - "", - "The LLAP daemon RPC port for external clients when llap is running in cloud environment. " - + "Length of the secret should be >= 32 bytes"), - // ====== confs for llap-external-client cloud deployment ====== LLAP_ENABLE_GRACE_JOIN_IN_LLAP("hive.llap.enable.grace.join.in.llap", false, "Override if grace join should be allowed to run in llap."), diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java index 9ee65a1c314c..d0afa3e9a9f9 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java @@ -58,7 +58,7 @@ public void testWritable() throws Exception { byte[] tokenBytes = new byte[] { 1 }; LlapInputSplit split1 = new LlapInputSplit(splitNum, planBytes, fragmentBytes, null, - locations, llapDaemonInfos, schema, "hive", tokenBytes, "some-dummy-jwt"); + locations, llapDaemonInfos, schema, "hive", tokenBytes); ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(); DataOutputStream dataOut = new DataOutputStream(byteOutStream); split1.write(dataOut); @@ -89,7 +89,6 @@ static void checkLlapSplits(LlapInputSplit split1, LlapInputSplit split2) throws assertArrayEquals(split1.getLocations(), split2.getLocations()); assertEquals(split1.getSchema().toString(), split2.getSchema().toString()); assertEquals(split1.getLlapUser(), split2.getLlapUser()); - assertEquals(split1.getJwt(), split2.getJwt()); assertArrayEquals(split1.getLlapDaemonInfos(), split2.getLlapDaemonInfos()); } diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java b/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java index 619346fd9684..f52618c4e211 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java @@ -37,10 +37,6 @@ public class LlapInputSplit implements InputSplitWithLocationInfo { private String llapUser; private byte[] fragmentBytesSignature; private byte[] tokenBytes; - //only needed in cloud deployments for llap server to validate request from external llap clients. - //HS2 generates a JWT and populates this field while get_splits() call, this jwt gets validated at LLAP server - //when LlapInputSplit is submitted. - private String jwt; public LlapInputSplit() { } @@ -48,7 +44,7 @@ public LlapInputSplit() { public LlapInputSplit(int splitNum, byte[] planBytes, byte[] fragmentBytes, byte[] fragmentBytesSignature, SplitLocationInfo[] locations, LlapDaemonInfo[] llapDaemonInfos, Schema schema, - String llapUser, byte[] tokenBytes, String jwt) { + String llapUser, byte[] tokenBytes) { this.planBytes = planBytes; this.fragmentBytes = fragmentBytes; this.fragmentBytesSignature = fragmentBytesSignature; @@ -58,7 +54,6 @@ public LlapInputSplit(int splitNum, byte[] planBytes, byte[] fragmentBytes, this.splitNum = splitNum; this.llapUser = llapUser; this.tokenBytes = tokenBytes; - this.jwt = jwt; } public Schema getSchema() { @@ -107,10 +102,6 @@ public void setSchema(Schema schema) { this.schema = schema; } - public String getJwt() { - return jwt; - } - @Override public void write(DataOutput out) throws IOException { out.writeInt(splitNum); @@ -145,9 +136,8 @@ public void write(DataOutput out) throws IOException { out.writeInt(0); } - if (jwt != null) { - out.writeUTF(jwt); - } + // Retain the retired JWT field in the wire format for older readers. + out.writeUTF(""); } @Override @@ -187,7 +177,9 @@ public void readFields(DataInput in) throws IOException { tokenBytes = new byte[length]; in.readFully(tokenBytes); } - jwt = in.readUTF(); + + // Discard the retired JWT field kept for wire compatibility. + in.readUTF(); } @Override diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java index 75c00970e8de..aff0ecbcbd95 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java @@ -19,9 +19,6 @@ package org.apache.hadoop.hive.llap.registry; -import com.google.common.base.Preconditions; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.llap.LlapUtil; import org.apache.hadoop.hive.registry.ServiceInstance; import org.apache.hadoop.yarn.api.records.Resource; @@ -55,25 +52,6 @@ public interface LlapServiceInstance extends ServiceInstance { */ public int getOutputFormatPort(); - /** - * External host, usually needed in cloud envs where we cannot access internal host from outside - * - * @return - */ - String getExternalHostname(); - - /** - * RPC endpoint for external clients - tcp traffic on this port should be opened on cloud. - * - * @return - */ - int getExternalClientsRpcPort(); - - - default void ensureCloudEnv(Configuration conf) { - Preconditions.checkState(LlapUtil.isCloudDeployment(conf), "Only supported in cloud based deployments"); - } - /** * Memory and Executors available for the LLAP tasks * diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java index 58948fa68d65..48e44b8741df 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java @@ -56,16 +56,6 @@ public int getShufflePort() { throw new UnsupportedOperationException(); } - @Override - public String getExternalHostname() { - throw new UnsupportedOperationException(); - } - - @Override - public int getExternalClientsRpcPort() { - throw new UnsupportedOperationException(); - } - @Override public String getServicesAddress() { throw new UnsupportedOperationException(); diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java index af65301a2198..75c3ad9ea594 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java @@ -40,11 +40,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.llap.LlapUtil; import org.apache.hadoop.hive.llap.registry.LlapServiceInstance; import org.apache.hadoop.hive.llap.registry.LlapServiceInstanceSet; import org.apache.hadoop.hive.llap.registry.ServiceRegistry; -import org.apache.hadoop.hive.registry.ServiceInstance; import org.apache.hadoop.hive.registry.ServiceInstanceStateChangeListener; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.StringUtils; @@ -65,9 +63,7 @@ public class LlapFixedRegistryImpl implements ServiceRegistry capacityValues = new HashMap<>(2); @@ -196,15 +183,9 @@ public String register() throws IOException { } registerServiceRecord(daemonZkRecord, uniqueId); - if (LlapUtil.isCloudDeployment(conf)) { - LOG.info("Registered node. Created a znode on ZooKeeper for LLAP instance: rpc: {}, external client rpc : {} " - + "shuffle: {}, webui: {}, mgmt: {}, znodePath: {}", rpcEndpoint, externalRpcEndpoint, - getShuffleEndpoint(), getServicesEndpoint(), getMngEndpoint(), getRegistrationZnodePath()); - } else { - LOG.info("Registered node. Created a znode on ZooKeeper for LLAP instance: rpc: {}, " - + "shuffle: {}, webui: {}, mgmt: {}, znodePath: {}", rpcEndpoint, getShuffleEndpoint(), - getServicesEndpoint(), getMngEndpoint(), getRegistrationZnodePath()); - } + LOG.info("Registered node. Created a znode on ZooKeeper for LLAP instance: rpc: {}, " + + "shuffle: {}, webui: {}, mgmt: {}, znodePath: {}", rpcEndpoint, getShuffleEndpoint(), + getServicesEndpoint(), getMngEndpoint(), getRegistrationZnodePath()); return uniqueId; } @@ -242,9 +223,6 @@ public class DynamicServiceInstance private final int outputFormatPort; private final String serviceAddress; - private String externalHost; - private int externalClientsRpcPort; - private final Resource resource; public DynamicServiceInstance(ServiceRecord srv) throws IOException { @@ -267,15 +245,6 @@ public DynamicServiceInstance(ServiceRecord srv) throws IOException { this.serviceAddress = RegistryTypeUtils.getAddressField(services.addresses.get(0), AddressTypes.ADDRESS_URI); - if (LlapUtil.isCloudDeployment(conf)) { - final Endpoint externalRpc = srv.getExternalEndpoint(IPC_EXTERNAL_LLAP); - this.externalHost = RegistryTypeUtils.getAddressField(externalRpc.addresses.get(0), - AddressTypes.ADDRESS_HOSTNAME_FIELD); - this.externalClientsRpcPort = Integer.parseInt( - RegistryTypeUtils.getAddressField(externalRpc.addresses.get(0), - AddressTypes.ADDRESS_PORT_FIELD)); - } - String memStr = srv.get(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, ""); String coreStr = srv.get(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS, ""); try { @@ -296,18 +265,6 @@ public String getServicesAddress() { return serviceAddress; } - @Override - public String getExternalHostname() { - ensureCloudEnv(LlapZookeeperRegistryImpl.this.conf); - return externalHost; - } - - @Override - public int getExternalClientsRpcPort() { - ensureCloudEnv(LlapZookeeperRegistryImpl.this.conf); - return externalClientsRpcPort; - } - @Override public Resource getResource() { return resource; diff --git a/llap-common/pom.xml b/llap-common/pom.xml index 15012499451f..20689348bf95 100644 --- a/llap-common/pom.xml +++ b/llap-common/pom.xml @@ -48,18 +48,6 @@ com.google.guava guava - - io.jsonwebtoken - jjwt-api - - - io.jsonwebtoken - jjwt-impl - - - io.jsonwebtoken - jjwt-jackson - org.apache.commons commons-lang3 diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java index ce8e188f6e8d..c35840e649b5 100644 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java +++ b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java @@ -297,24 +297,4 @@ public static Credentials credentialsFromByteArray(byte[] binaryCredentials) credentials.readTokenStorageStream(dib); return credentials; } - - /** - * @return returns the value of LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED - * @param conf - */ - public static boolean isCloudDeployment(Configuration conf) { - return HiveConf.getBoolVar(conf, ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED, false); - } - - /** - * @return returns the value of PUBLIC_HOSTNAME from either environment variable or system properties - */ - public static String getPublicHostname() { - String publicHostname = System.getenv("PUBLIC_HOSTNAME"); - if (publicHostname == null) { - publicHostname = System.getProperty("PUBLIC_HOSTNAME"); - } - return publicHostname; - } - } diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/security/DefaultJwtSharedSecretProvider.java b/llap-common/src/java/org/apache/hadoop/hive/llap/security/DefaultJwtSharedSecretProvider.java deleted file mode 100644 index 1e04c2a58bfd..000000000000 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/security/DefaultJwtSharedSecretProvider.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap.security; - -import com.google.common.base.Preconditions; -import io.jsonwebtoken.security.Keys; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.StandardCharsets; -import java.security.Key; - -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET; - -/** - * Default implementation of {@link JwtSecretProvider}. - * - * 1. It first tries to get shared secret from conf {@link HiveConf.ConfVars#LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET} - * using {@link Configuration#getPassword(String)}. - * - * 2. If not found, it tries to read from env var {@link #LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR}. - * - * If secret is not found even after 1) and 2), {@link #init(Configuration)} methods throws {@link IllegalStateException}. - * - * Length of shared secret provided in 1) or 2) should be > 32 bytes. - * - * It uses the same encryption and decryption secret which can be used to sign and verify JWT. - */ -public class DefaultJwtSharedSecretProvider implements JwtSecretProvider { - - public static final String LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR = - "LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR"; - - private Key jwtEncryptionKey; - - @Override public Key getEncryptionSecret() { - return jwtEncryptionKey; - } - - @Override public Key getDecryptionSecret() { - return jwtEncryptionKey; - } - - @Override public void init(final Configuration conf) { - char[] sharedSecret; - byte[] sharedSecretBytes = null; - - // try getting secret from conf first - // if not found, get from env var - LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR - try { - sharedSecret = conf.getPassword(LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET.varname); - } catch (IOException e) { - throw new RuntimeException("Unable to get password [hive.llap.external.client.cloud.jwt.shared.secret] - " - + e.getMessage(), e); - } - if (sharedSecret != null) { - ByteBuffer bb = StandardCharsets.UTF_8.encode(CharBuffer.wrap(sharedSecret)); - sharedSecretBytes = new byte[bb.remaining()]; - bb.get(sharedSecretBytes); - } else { - String sharedSecredFromEnv = System.getenv(LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR); - if (StringUtils.isNotBlank(sharedSecredFromEnv)) { - sharedSecretBytes = sharedSecredFromEnv.getBytes(); - } - } - - Preconditions.checkState(sharedSecretBytes != null, - "With: " + LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED.varname + " = true, \n" - + "To use: org.apache.hadoop.hive.llap.security.DefaultJwtSharedSecretProvider, \n" - + "1. a non-null value of 'hive.llap.external.client.cloud.jwt.shared.secret' must be provided OR \n" - + "2. alternatively environment variable " - + "LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR can also be set. \n" - + "Length of the secret provided in 1) or 2) should be > 32 bytes."); - - this.jwtEncryptionKey = Keys.hmacShaKeyFor(sharedSecretBytes); - } - -} diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/security/JwtSecretProvider.java b/llap-common/src/java/org/apache/hadoop/hive/llap/security/JwtSecretProvider.java deleted file mode 100644 index c401f182c64b..000000000000 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/security/JwtSecretProvider.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap.security; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; - -import java.security.Key; - -/** - * JwtSecretProvider - * - * - provides encryption and decryption secrets for generating and parsing JWTs. - * - * - Hive internally uses method initAndGet() which initializes providers based on the value of config - * {@link HiveConf.ConfVars#LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_PROVIDER}. - * It expects implementations to provide default constructor and {@link #init(Configuration)} method. - */ -public interface JwtSecretProvider { - - /** - * returns secret for signing JWT. - */ - Key getEncryptionSecret(); - - /** - * returns secret for parsing JWT. - */ - Key getDecryptionSecret(); - - /** - * Initializes the provider. - * Should also contain any validations that we want to put on secret, helps us to fail fast. - * @param conf configuration - */ - void init(Configuration conf); - - /** - * Hive internally uses this method to obtain instance of {@link JwtSecretProvider} - * - * @param conf configuration - * @return implementation of {@link HiveConf.ConfVars#LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_PROVIDER} - */ - static JwtSecretProvider initAndGet(Configuration conf) { - final String providerClass = - HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_PROVIDER); - JwtSecretProvider provider; - try { - provider = (JwtSecretProvider) Class.forName(providerClass).newInstance(); - provider.init(conf); - } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { - throw new RuntimeException("Unable to instantiate provider: " + providerClass, e); - } - return provider; - } -} diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapExtClientJwtHelper.java b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapExtClientJwtHelper.java deleted file mode 100644 index 55500546a9df..000000000000 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapExtClientJwtHelper.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.llap.security; - -import io.jsonwebtoken.Claims; -import io.jsonwebtoken.Jws; -import io.jsonwebtoken.Jwts; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.records.ApplicationId; - -import java.util.Date; -import java.util.UUID; - -/** - * Contains helper methods for generating and verifying JWTs for external llap clients. - * Initializes and uses {@link JwtSecretProvider} to obtain encryption and decryption secret. - */ -public class LlapExtClientJwtHelper { - - public static final String LLAP_JWT_SUBJECT = "llap"; - public static final String LLAP_EXT_CLIENT_APP_ID = "llap_ext_client_app_id"; - private final JwtSecretProvider jwtSecretProvider; - - public LlapExtClientJwtHelper(Configuration conf) { - this.jwtSecretProvider = JwtSecretProvider.initAndGet(conf); - } - - /** - * @param extClientAppId application Id - application Id injected by get_splits - * @return JWT signed with {@link JwtSecretProvider#getEncryptionSecret()}. - * As of now this JWT contains extClientAppId in claims. - */ - public String buildJwtForLlap(ApplicationId extClientAppId) { - return Jwts.builder() - .setSubject(LLAP_JWT_SUBJECT) - .setIssuedAt(new Date()) - .setId(UUID.randomUUID().toString()) - .claim(LLAP_EXT_CLIENT_APP_ID, extClientAppId.toString()) - .signWith(jwtSecretProvider.getEncryptionSecret()) - .compact(); - } - - /** - * - * @param jwt signed JWT String - * @return claims present in JWT, this method parses jwt using {@link JwtSecretProvider#getDecryptionSecret()} - */ - public Jws parseClaims(String jwt) { - return Jwts.parser() - .setSigningKey(jwtSecretProvider.getDecryptionSecret()) - .parseClaimsJws(jwt); - } - -} diff --git a/llap-common/src/protobuf/LlapDaemonProtocol.proto b/llap-common/src/protobuf/LlapDaemonProtocol.proto index 8b15f4392eb5..4359fbc5b70b 100644 --- a/llap-common/src/protobuf/LlapDaemonProtocol.proto +++ b/llap-common/src/protobuf/LlapDaemonProtocol.proto @@ -132,8 +132,11 @@ message SubmitWorkRequestProto { optional bytes initial_event_signature = 11; optional bool is_guaranteed = 12 [default = false]; - optional string jwt = 13; - optional bool is_external_client_request = 14 [default = false]; + + // HIVE-28932: We used 13, 14, jwt, is_external_client_request to let external clients directly access LLAP daemons + // in a cloud environment. + reserved 13, 14; + reserved "jwt", "is_external_client_request"; } message RegisterDagRequestProto { diff --git a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java index 7ab8032889db..5772e391d2b1 100644 --- a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java +++ b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java @@ -50,7 +50,6 @@ import org.apache.hadoop.hive.llap.ext.LlapDaemonInfo; import org.apache.hadoop.hive.llap.ext.LlapTaskUmbilicalExternalClient; import org.apache.hadoop.hive.llap.ext.LlapTaskUmbilicalExternalClient.LlapTaskUmbilicalExternalResponder; -import org.apache.hadoop.hive.llap.registry.LlapServiceInstance; import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier; import org.apache.hadoop.hive.llap.tez.Converters; import org.apache.hadoop.io.BytesWritable; @@ -416,8 +415,6 @@ private SubmitWorkRequestProto constructSubmitWorkRequestProto(SubmitWorkInfo su if (fragmentBytesSignature != null) { builder.setInitialEventSignature(ByteString.copyFrom(fragmentBytesSignature)); } - builder.setJwt(llapInputSplit.getJwt()); - builder.setIsExternalClientRequest(true); return builder.build(); } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java index cc87e79dbf4d..f4819c586f86 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java @@ -68,9 +68,6 @@ public Void call() throws Exception { io.netty.handler.codec.http.HttpObjectAggregator.class, // netty-all com.google.flatbuffers.Table.class, //flatbuffers com.carrotsearch.hppc.ByteArrayDeque.class, //hppc - io.jsonwebtoken.security.Keys.class, //jjwt-api - io.jsonwebtoken.impl.DefaultJws.class, //jjwt-impl - io.jsonwebtoken.io.JacksonSerializer.class, //jjwt-jackson }; for (Class c : dependencies) { diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java index 2c7b729f6986..0dadbf982b01 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java @@ -19,14 +19,11 @@ package org.apache.hadoop.hive.llap.daemon.impl; -import org.apache.hadoop.hive.llap.LlapUtil; import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol.BooleanArray; import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol.TezAttemptArray; import java.util.ArrayList; import java.util.List; -import java.util.HashSet; -import java.util.Set; import javax.net.SocketFactory; @@ -198,7 +195,7 @@ public void serviceStop() { } } - public AMNodeInfo registerTask(boolean externalClientRequest, String amLocation, int port, String umbilicalUser, + public AMNodeInfo registerTask(String amLocation, int port, String umbilicalUser, Token jobToken, QueryIdentifier queryIdentifier, TezTaskAttemptID attemptId, boolean isGuaranteed) { if (LOG.isTraceEnabled()) { @@ -220,7 +217,6 @@ public AMNodeInfo registerTask(boolean externalClientRequest, String amLocation, if (amNodeInfo == null) { amNodeInfo = new AMNodeInfo(amNodeId, umbilicalUser, jobToken, queryIdentifier, retryPolicy, retryTimeout, socketFactory, conf); - amNodeInfo.setIsExternalClientRequest(externalClientRequest); amNodeInfoPerQuery.put(amNodeId, amNodeInfo); // Add to the queue only the first time this is registered, and on // subsequent instances when it's taken off the queue. @@ -413,15 +409,8 @@ protected Void callInternal() { BooleanArray guaranteed = new BooleanArray(); guaranteed.set(tasks.guaranteed.toArray(new BooleanWritable[tasks.guaranteed.size()])); - if (LlapUtil.isCloudDeployment(conf) && amNodeInfo.isExternalClientRequest()) { - String hostname = amNodeInfo.amNodeId.getHostname(); - int externalClientCloudRpcPort = amNodeInfo.amNodeId.getPort(); - amNodeInfo.getUmbilical().nodeHeartbeat(new Text(hostname), - new Text(daemonId.getUniqueNodeIdInCluster()), externalClientCloudRpcPort, aw, guaranteed); - } else { - amNodeInfo.getUmbilical().nodeHeartbeat(new Text(nodeId.getHostname()), - new Text(daemonId.getUniqueNodeIdInCluster()), nodeId.getPort(), aw, guaranteed); - } + amNodeInfo.getUmbilical().nodeHeartbeat(new Text(nodeId.getHostname()), + new Text(daemonId.getUniqueNodeIdInCluster()), nodeId.getPort(), aw, guaranteed); } catch (IOException e) { QueryIdentifier currentQueryIdentifier = amNodeInfo.getQueryIdentifier(); amNodeInfo.setAmFailed(true); @@ -480,7 +469,6 @@ protected class AMNodeInfo implements Delayed { private LlapTaskUmbilicalProtocol umbilical; private long nextHeartbeatTime; private final AtomicBoolean isDone = new AtomicBoolean(false); - private final AtomicBoolean isExternalClientRequest = new AtomicBoolean(false); public AMNodeInfo(LlapNodeId amNodeId, String umbilicalUser, @@ -552,14 +540,6 @@ boolean isDone() { return isDone.get(); } - void setIsExternalClientRequest(boolean val) { - isExternalClientRequest.set(val); - } - - boolean isExternalClientRequest() { - return isExternalClientRequest.get(); - } - /** * @return A snapshot of the tasks running at this daemon from this AM. * Doesn't have to be consistent between multiple tasks; whether some task makes it into diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java index d9be6c0b0975..fb25bc5c8c3d 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java @@ -40,10 +40,6 @@ import com.google.common.cache.RemovalListener; import com.google.common.cache.RemovalNotification; -import io.jsonwebtoken.Claims; -import io.jsonwebtoken.Jws; -import io.jsonwebtoken.JwtException; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.llap.LlapUgiManager; import org.apache.hadoop.hive.conf.HiveConf; @@ -83,7 +79,6 @@ import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SetCapacityResponseProto; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.VertexOrBinary; import org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorMetrics; -import org.apache.hadoop.hive.llap.security.LlapExtClientJwtHelper; import org.apache.hadoop.hive.llap.security.LlapSignerImpl; import org.apache.hadoop.hive.llap.tez.Converters; import org.apache.hadoop.hive.llap.tezplugins.LlapTezUtils; @@ -241,8 +236,6 @@ public SubmitWorkResponseProto submitWork(SubmitWorkRequestProto request) throws QueryIdentifierProto qIdProto = vertex.getQueryIdentifier(); - verifyJwtForExternalClient(request, qIdProto.getApplicationIdString(), fragmentIdString); - LOG.info("Queueing container for execution: fragemendId={}, {}", fragmentIdString, stringifySubmitRequest(request, vertex)); @@ -351,39 +344,6 @@ public SubmitWorkResponseProto submitWork(SubmitWorkRequestProto request) throws .build(); } - // if request is coming from llap external client, verify the JWT - // as of now, JWT contains applicationId - private void verifyJwtForExternalClient(SubmitWorkRequestProto request, String extClientAppIdFromSplit, - String fragmentIdString) { - LOG.info("Checking if request[{}] is from llap external client in a cloud based deployment", - extClientAppIdFromSplit); - if (request.getIsExternalClientRequest() && LlapUtil.isCloudDeployment(getConfig())) { - LOG.info("Llap external client request - {}, verifying JWT", extClientAppIdFromSplit); - Preconditions.checkState(request.hasJwt(), "JWT not found in request, fragmentId: " + fragmentIdString); - - LlapExtClientJwtHelper llapExtClientJwtHelper = new LlapExtClientJwtHelper(getConfig()); - Jws claimsJws; - try { - claimsJws = llapExtClientJwtHelper.parseClaims(request.getJwt()); - } catch (JwtException e) { - LOG.error("Cannot verify JWT provided with the request, fragmentId: {}, {}", fragmentIdString, e); - throw e; - } - - String extClientAppIdFromJwt = (String) claimsJws.getBody().get(LlapExtClientJwtHelper.LLAP_EXT_CLIENT_APP_ID); - - // this should never happen ideally. - // extClientAppId is injected in JWT and fragment request by initial get_splits() call. - // so both of these - extClientAppIdFromJwt and extClientAppIdFromSplit should be equal eventually if the signed JWT is valid for this request. - // In get_splits, this extClientAppId is obtained via LlapCoordinator#createExtClientAppId which generates a - // application Id to be used by external clients. - Preconditions.checkState(extClientAppIdFromJwt.equals(extClientAppIdFromSplit), - String.format("applicationId[%s] in request does not match to applicationId[%s] in JWT", - extClientAppIdFromSplit, extClientAppIdFromJwt)); - LOG.info("Llap external client request - {}, JWT verification successful", extClientAppIdFromSplit); - } - } - private SignableVertexSpec extractVertexSpec(SubmitWorkRequestProto request, LlapTokenInfo tokenInfo) throws InvalidProtocolBufferException, IOException { VertexOrBinary vob = request.getWorkSpec(); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index 4d3a5d06e6c0..615e82cf9848 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -76,7 +76,6 @@ import org.apache.hadoop.hive.llap.metrics.LlapMetricsSystem; import org.apache.hadoop.hive.llap.metrics.MetricsUtils; import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService; -import org.apache.hadoop.hive.llap.security.LlapExtClientJwtHelper; import org.apache.hadoop.hive.llap.security.SecretManager; import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler; import org.apache.hadoop.hive.ql.ServiceContext; @@ -141,7 +140,6 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemoryBytes, boolean ioEnabled, boolean isDirectCache, long ioMemoryBytes, String[] localDirs, int srvPort, - boolean externalClientCloudSetupEnabled, int externalClientsRpcPort, int mngPort, int shufflePort, int webPort, String appName) { super("LlapDaemon"); @@ -150,11 +148,6 @@ public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemor Preconditions.checkArgument(numExecutors > 0); Preconditions.checkArgument(srvPort == 0 || (srvPort > 1024 && srvPort < 65536), "Server RPC Port must be between 1025 and 65535, or 0 automatic selection"); - if (externalClientCloudSetupEnabled) { - Preconditions.checkArgument( - externalClientsRpcPort == 0 || (externalClientsRpcPort > 1024 && externalClientsRpcPort < 65536), - "Server RPC port for external clients must be between 1025 and 65535, or 0 automatic selection"); - } Preconditions.checkArgument(mngPort == 0 || (mngPort > 1024 && mngPort < 65536), "Management RPC Port must be between 1025 and 65535, or 0 automatic selection"); @@ -241,8 +234,6 @@ public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemor ", llapIoEnabled=" + ioEnabled + ", llapIoCacheIsDirect=" + isDirectCache + ", rpcListenerPort=" + srvPort + - ", externalClientCloudSetupEnabled=" + externalClientCloudSetupEnabled + - ", rpcListenerPortForExternalClients=" + externalClientsRpcPort + ", mngListenerPort=" + mngPort + ", webPort=" + webPort + ", outputFormatSvcPort=" + outputFormatServicePort + @@ -340,7 +331,7 @@ public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemor this.secretManager = sm; this.server = new LlapProtocolServerImpl(secretManager, numHandlers, this, srvAddress, mngAddress, srvPort, - externalClientsRpcPort, mngPort, daemonId, metrics).withTokenManager(this.llapTokenManager); + mngPort, daemonId, metrics).withTokenManager(this.llapTokenManager); LlapUgiManager llapUgiManager = LlapUgiManager.getInstance(daemonConf); @@ -502,15 +493,6 @@ public void serviceStart() throws Exception { getConfig().setInt(ConfVars.LLAP_DAEMON_WEB_PORT.varname, webServices.getPort()); } getConfig().setInt(ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT.varname, LlapOutputFormatService.get().getPort()); - if (LlapUtil.isCloudDeployment(getConfig())) { - - // this invokes JWT secret provider and tries to get shared secret. - // meant to validate shared secret as well. - new LlapExtClientJwtHelper(getConfig()); - - getConfig().setInt(ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT.varname, - server.getExternalClientsRpcServerBindAddress().getPort()); - } // Ensure this is set in the config so that the AM can read it. getConfig() @@ -624,8 +606,6 @@ public static void main(String[] args) throws Exception { String[] localDirs = (localDirList == null || localDirList.isEmpty()) ? new String[0] : StringUtils.getTrimmedStrings(localDirList); int rpcPort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_PORT); - int externalClientCloudRpcPort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT); - boolean externalClientCloudSetupEnabled = LlapUtil.isCloudDeployment(daemonConf); int mngPort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_MANAGEMENT_RPC_PORT); int shufflePort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_YARN_SHUFFLE_PORT); int webPort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_WEB_PORT); @@ -643,8 +623,8 @@ public static void main(String[] args) throws Exception { LlapDaemon.initializeLogging(daemonConf); llapDaemon = new LlapDaemon(daemonConf, numExecutors, executorMemoryBytes, isLlapIo, isDirectCache, - ioMemoryBytes, localDirs, rpcPort, externalClientCloudSetupEnabled, - externalClientCloudRpcPort, mngPort, shufflePort, webPort, appName); + ioMemoryBytes, localDirs, rpcPort, + mngPort, shufflePort, webPort, appName); LOG.info("Adding shutdown hook for LlapDaemon"); ShutdownHookManager.addShutdownHook(new CompositeServiceShutdownHook(llapDaemon), 1); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java index b1e8db5287d8..5bd32069c0ff 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java @@ -83,8 +83,10 @@ private enum TokenRequiresSigning { private final int numHandlers; private final ContainerRunner containerRunner; - private final int srvPort, mngPort, externalClientsRpcPort; - private RPC.Server server, mngServer, externalClientsRpcServer; + private final int srvPort; + private final int mngPort; + private RPC.Server server; + private RPC.Server mngServer; private final AtomicReference srvAddress, mngAddress; private final SecretManager secretManager; private String clusterUser = null; @@ -95,7 +97,7 @@ private enum TokenRequiresSigning { public LlapProtocolServerImpl(SecretManager secretManager, int numHandlers, ContainerRunner containerRunner, AtomicReference srvAddress, - AtomicReference mngAddress, int srvPort, int externalClientsRpcPort, + AtomicReference mngAddress, int srvPort, int mngPort, DaemonId daemonId, LlapDaemonExecutorMetrics executorMetrics) { super("LlapDaemonProtocolServerImpl"); @@ -105,7 +107,6 @@ public LlapProtocolServerImpl(SecretManager secretManager, int numHandlers, this.srvAddress = srvAddress; this.srvPort = srvPort; this.mngAddress = mngAddress; - this.externalClientsRpcPort = externalClientsRpcPort; this.mngPort = mngPort; this.executorMetrics = executorMetrics; LOG.info("Creating: " + LlapProtocolServerImpl.class.getSimpleName() + @@ -241,15 +242,6 @@ private void startProtocolServers( server = LlapUtil.startProtocolServer(srvPort, numHandlers, srvAddress, conf, daemonImpl, LlapProtocolBlockingPB.class, secretManager, pp, ConfVars.LLAP_SECURITY_ACL, ConfVars.LLAP_SECURITY_ACL_DENY); - // for cloud deployments, start a separate RPC server on the port - // which we can open to accept requests from external clients. - if (LlapUtil.isCloudDeployment(conf)) { - externalClientsRpcServer = LlapUtil.startProtocolServer(externalClientsRpcPort, numHandlers, null, conf, daemonImpl, - LlapProtocolBlockingPB.class, secretManager, pp, ConfVars.LLAP_SECURITY_ACL, - ConfVars.LLAP_SECURITY_ACL_DENY); - - LOG.info("Started externalClientsRpcServer for cloud based deployments : {}, {}", externalClientsRpcServer.getListenerAddress(), externalClientsRpcServer); - } mngServer = LlapUtil.startProtocolServer(mngPort, 2, mngAddress, conf, managementImpl, LlapManagementProtocolPB.class, secretManager, pp, ConfVars.LLAP_MANAGEMENT_ACL, ConfVars.LLAP_MANAGEMENT_ACL_DENY); @@ -261,9 +253,6 @@ public void serviceStop() { if (server != null) { server.stop(); } - if (externalClientsRpcServer != null) { - externalClientsRpcServer.stop(); - } if (mngServer != null) { mngServer.stop(); } @@ -279,11 +268,6 @@ InetSocketAddress getManagementBindAddress() { return mngAddress.get(); } - @InterfaceAudience.Private - InetSocketAddress getExternalClientsRpcServerBindAddress() { - return externalClientsRpcServer.getListenerAddress(); - } - @Override public GetTokenResponseProto getDelegationToken(RpcController controller, GetTokenRequestProto request) throws ServiceException { diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java index 9f6ff74ecc9c..eddf2834eced 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java @@ -160,7 +160,7 @@ public TaskRunnerCallable(SubmitWorkRequestProto request, QueryFragmentInfo frag this.amReporter = amReporter; // Register with the AMReporter when the callable is setup. Unregister once it starts running. if (amReporter != null && jobToken != null) { - this.amNodeInfo = amReporter.registerTask(request.getIsExternalClientRequest(), request.getAmHost(), request.getAmPort(), + this.amNodeInfo = amReporter.registerTask(request.getAmHost(), request.getAmPort(), vertex.getTokenIdentifier(), jobToken, fragmentInfo.getQueryInfo().getQueryIdentifier(), attemptId, isGuaranteed); } else { diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/LlapDaemonExtension.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/LlapDaemonExtension.java index 0b6ddb25daee..0d1afc569b9e 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/LlapDaemonExtension.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/LlapDaemonExtension.java @@ -64,7 +64,7 @@ public void beforeEach(ExtensionContext context) throws Exception { HiveConf.setVar(conf, HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS, "llap"); LlapDaemonInfo.initialize(appName, conf); daemon = - new LlapDaemon(conf, 1, LlapDaemon.getTotalHeapSize(), false, false, -1, new String[1], 0, false, 0, 0, 0, 0, + new LlapDaemon(conf, 1, LlapDaemon.getTotalHeapSize(), false, false, -1, new String[1], 0, 0, 0, 0, appName); daemon.init(conf); daemon.start(); diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java index f8cd79d6826f..47cadddf5cea 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java @@ -24,7 +24,6 @@ import java.io.File; import java.io.IOException; -import org.apache.hadoop.hive.llap.LlapUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -37,8 +36,6 @@ import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.Service; -import org.apache.hadoop.util.Shell; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hive.testutils.MiniZooKeeperCluster; import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; @@ -142,7 +139,6 @@ private MiniLlapCluster(String clusterName, @Nullable MiniZooKeeperCluster miniZ @Override public void serviceInit(Configuration conf) throws IOException, InterruptedException { int rpcPort = 0; - int externalClientCloudRpcPort = 0; int mngPort = 0; int shufflePort = 0; int webPort = 0; @@ -151,7 +147,6 @@ public void serviceInit(Configuration conf) throws IOException, InterruptedExcep LOG.info("MiniLlap configured to use ports from conf: {}", usePortsFromConf); if (usePortsFromConf) { rpcPort = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_DAEMON_RPC_PORT); - externalClientCloudRpcPort = HiveConf.getIntVar(conf, ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT); mngPort = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_MANAGEMENT_RPC_PORT); shufflePort = conf.getInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, ShuffleHandler.DEFAULT_SHUFFLE_PORT); webPort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_WEB_PORT); @@ -174,12 +169,10 @@ public void serviceInit(Configuration conf) throws IOException, InterruptedExcep clusterSpecificConfiguration.set(ConfVars.HIVE_ZOOKEEPER_QUORUM.varname, "localhost"); clusterSpecificConfiguration.setInt(ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT.varname, miniZooKeeperCluster.getClientPort()); - boolean externalClientCloudSetupEnabled = LlapUtil.isCloudDeployment(conf); - LOG.info("Initializing {} llap instances for MiniLlapCluster with name={}", numInstances, clusterNameTrimmed); for (int i = 0 ;i < numInstances ; i++) { llapDaemons[i] = new LlapDaemon(conf, numExecutorsPerService, execBytesPerService, llapIoEnabled, - ioIsDirect, ioBytesPerService, localDirs, rpcPort, externalClientCloudSetupEnabled, externalClientCloudRpcPort, + ioIsDirect, ioBytesPerService, localDirs, rpcPort, mngPort, shufflePort, webPort, clusterNameTrimmed); llapDaemons[i].init(new Configuration(conf)); } diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java index 44351b9d419b..43fdfe195d37 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java @@ -106,7 +106,7 @@ public void testEnforceProperNumberOfIOThreads() throws IOException { HiveConf.setIntVar(hiveConf, HiveConf.ConfVars.LLAP_IO_THREADPOOL_SIZE, 3); daemon = new LlapDaemon(hiveConf, 4, LlapDaemon.getTotalHeapSize(), true, false, - -1, new String[1], 0, false, 0,0, 0, defaultWebPort, "TestLlapDaemon"); + -1, new String[1], 0,0, 0, defaultWebPort, "TestLlapDaemon"); } @Test @@ -120,7 +120,7 @@ public void testLocalDirCleaner() throws IOException, InterruptedException { createFile(localDirs[0] + "/file3"); daemon = new LlapDaemon(hiveConf, 1, LlapDaemon.getTotalHeapSize(), false, false, - -1, localDirs, 0, false, 0,0, 0, defaultWebPort, "TestLlapDaemon"); + -1, localDirs, 0, 0, 0, defaultWebPort, "TestLlapDaemon"); daemon.init(hiveConf); assertFileExists(localDirs[0] + "/hive/appcache/file1", true); @@ -155,7 +155,7 @@ public void testUpdateRegistration() throws IOException { int enabledQueue = 2; daemon = new LlapDaemon(hiveConf, 1, LlapDaemon.getTotalHeapSize(), false, false, - -1, new String[1], 0, false, 0,0, 0, defaultWebPort, "TestLlapDaemon"); + -1, new String[1], 0,0, 0, defaultWebPort, "TestLlapDaemon"); trySetMock(daemon, LlapRegistryService.class, mockRegistry); diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java index a3802ecda44b..8fefb3bc7243 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java @@ -60,7 +60,7 @@ public void testSimpleCall() throws ServiceException, IOException { LlapProtocolServerImpl server = new LlapProtocolServerImpl(null, numHandlers, containerRunnerMock, new AtomicReference(), new AtomicReference(), - 0, 0, 0, null, null); + 0, 0, null, null); when(containerRunnerMock.submitWork(any(SubmitWorkRequestProto.class))).thenReturn( SubmitWorkResponseProto .newBuilder() @@ -95,7 +95,7 @@ public void testGetDaemonMetrics() throws ServiceException, IOException { LlapProtocolServerImpl server = new LlapProtocolServerImpl(null, numHandlers, null, new AtomicReference(), new AtomicReference(), - 0, 0, 0, null, executorMetrics); + 0, 0, null, executorMetrics); executorMetrics.addMetricsFallOffFailedTimeLost(10); executorMetrics.addMetricsFallOffKilledTimeLost(11); executorMetrics.addMetricsFallOffSuccessTimeLost(12); @@ -166,7 +166,7 @@ public void testSetCapacity() throws ServiceException, IOException { LlapProtocolServerImpl server = new LlapProtocolServerImpl(null, numHandlers, containerRunnerMock, new AtomicReference(), new AtomicReference(), - 0, 0, 0, null, executorMetrics); + 0, 0, null, executorMetrics); try { server.init(new Configuration()); diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/comparator/TestAMReporter.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/comparator/TestAMReporter.java index 5070dfc62a1b..56b3e8b69399 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/comparator/TestAMReporter.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/comparator/TestAMReporter.java @@ -72,9 +72,9 @@ public void testMultipleAM() throws InterruptedException { String am2Location = "am2"; String umbilicalUser = "user"; QueryIdentifier queryId = new QueryIdentifier("app", 0); - amReporter.registerTask(false,am1Location, am1Port, umbilicalUser, null, queryId, + amReporter.registerTask(am1Location, am1Port, umbilicalUser, null, queryId, mock(TezTaskAttemptID.class), false); - amReporter.registerTask(false,am2Location, am2Port, umbilicalUser, null, queryId, + amReporter.registerTask(am2Location, am2Port, umbilicalUser, null, queryId, mock(TezTaskAttemptID.class), false); Thread.currentThread().sleep(2000); diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestLlapOrcCacheLoader.java b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestLlapOrcCacheLoader.java index fa0e59b04a2c..e98afaa776a8 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestLlapOrcCacheLoader.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestLlapOrcCacheLoader.java @@ -18,7 +18,6 @@ */ package org.apache.hadoop.hive.llap.io.encoded; -import io.jsonwebtoken.lang.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -37,6 +36,7 @@ import org.apache.hive.common.util.FixedSizedObjectPool; import org.junit.Before; import org.junit.Test; +import org.junit.jupiter.api.Assertions; import java.io.IOException; @@ -89,7 +89,7 @@ public void testLoadFooter() throws IOException { loader.loadFileFooter(); } MetadataCache.LlapBufferOrBuffers metadata = metaCache.getFileMetadata(key); - Assert.notNull(metadata); + Assertions.assertNotNull(metadata); } @@ -108,7 +108,7 @@ public void testLoadUncompressedRanges() throws IOException { cache.getFileData(key, range, 0, mockDiskRangeListFactory, null, gotAllData); - Assert.isTrue(gotAllData.value); + Assertions.assertTrue(gotAllData.value); } } @@ -126,7 +126,7 @@ public void testLoadValidRanges() throws IOException { cache.getFileData(key, range, 0, mockDiskRangeListFactory, null, gotAllData); - Assert.isTrue(gotAllData.value); + Assertions.assertTrue(gotAllData.value); } @Test @@ -143,7 +143,7 @@ public void testLoadAlreadyLoadedRange() throws IOException { DataCache.BooleanRef gotAllData = new DataCache.BooleanRef(); cache.getFileData(key, range, 0, mockDiskRangeListFactory, null, gotAllData); - Assert.isTrue(gotAllData.value); + Assertions.assertTrue(gotAllData.value); DiskRangeList range2 = new DiskRangeList(ORC_PADDING,14); try(LlapOrcCacheLoader loader = new LlapOrcCacheLoader(path, key, conf, mockDataCache, metaCache, @@ -154,7 +154,7 @@ public void testLoadAlreadyLoadedRange() throws IOException { gotAllData.value = false; cache.getFileData(key, range, 0, mockDiskRangeListFactory, null, gotAllData); - Assert.isTrue(gotAllData.value); + Assertions.assertTrue(gotAllData.value); } @Test @@ -171,7 +171,7 @@ public void testLoadBadlyEstimatedRanges() throws IOException { cache.getFileData(key, range, 0, mockDiskRangeListFactory, null, gotAllData); - Assert.isTrue(!gotAllData.value); + Assertions.assertFalse(gotAllData.value); } diff --git a/pom.xml b/pom.xml index 9f6ff46225b0..c73d13d393e0 100644 --- a/pom.xml +++ b/pom.xml @@ -228,7 +228,6 @@ 5.7.1 3.0.0 2.9.0 - 0.10.5 1.2 2.0.1 2.9.0 @@ -446,21 +445,6 @@ truffle-runtime ${graalvm.version} - - io.jsonwebtoken - jjwt-api - ${jjwt.version} - - - io.jsonwebtoken - jjwt-impl - ${jjwt.version} - - - io.jsonwebtoken - jjwt-jackson - ${jjwt.version} - io.netty netty-all diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java index d135dc2f9fcc..df856d804845 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java @@ -47,7 +47,6 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.FieldDesc; import org.apache.hadoop.hive.llap.LlapInputSplit; -import org.apache.hadoop.hive.llap.LlapUtil; import org.apache.hadoop.hive.llap.NotTezEventHelper; import org.apache.hadoop.hive.llap.Schema; import org.apache.hadoop.hive.llap.SubmitWorkInfo; @@ -58,7 +57,6 @@ import org.apache.hadoop.hive.llap.ext.LlapDaemonInfo; import org.apache.hadoop.hive.llap.registry.LlapServiceInstance; import org.apache.hadoop.hive.llap.registry.LlapServiceInstanceSet; -import org.apache.hadoop.hive.llap.security.LlapExtClientJwtHelper; import org.apache.hadoop.hive.llap.security.LlapSigner; import org.apache.hadoop.hive.llap.security.LlapSigner.Signable; import org.apache.hadoop.hive.llap.security.LlapSigner.SignedMessage; @@ -422,7 +420,7 @@ private SplitResult getSplits(JobConf job, TezWork work, Schema schema, Applicat SplitResult splitResult = new SplitResult(); splitResult.schemaSplit = new LlapInputSplit( 0, new byte[0], new byte[0], new byte[0], - new SplitLocationInfo[0], new LlapDaemonInfo[0], schema, "", new byte[0], ""); + new SplitLocationInfo[0], new LlapDaemonInfo[0], schema, "", new byte[0]); if (schemaSplitOnly) { // schema only return splitResult; @@ -542,7 +540,7 @@ private SplitResult getSplits(JobConf job, TezWork work, Schema schema, Applicat if (generateLightWeightSplits) { splitResult.planSplit = new LlapInputSplit( 0, submitWorkBytes, new byte[0], new byte[0], - new SplitLocationInfo[0], new LlapDaemonInfo[0], new Schema(), "", new byte[0], ""); + new SplitLocationInfo[0], new LlapDaemonInfo[0], new Schema(), "", new byte[0]); } } @@ -555,22 +553,12 @@ private SplitResult getSplits(JobConf job, TezWork work, Schema schema, Applicat // 5. populate info about llap daemons(to help client submit request and read data) LlapDaemonInfo[] llapDaemonInfos = populateLlapDaemonInfos(job, locations); - // 6. Generate JWT for external clients if it's a cloud deployment - // we inject extClientAppId in JWT which is same as what fragment contains. - // extClientAppId in JWT and in fragment are compared on LLAP when a fragment is submitted. - // see method ContainerRunnerImpl#verifyJwtForExternalClient - String jwt = ""; - if (LlapUtil.isCloudDeployment(job)) { - LlapExtClientJwtHelper llapExtClientJwtHelper = new LlapExtClientJwtHelper(job); - jwt = llapExtClientJwtHelper.buildJwtForLlap(extClientAppId); - } - if (generateLightWeightSplits) { result[i] = new LlapInputSplit(i, emptySubmitWorkBytes, eventBytes.message, - eventBytes.signature, locations, llapDaemonInfos, emptySchema, llapUser, tokenBytes, jwt); + eventBytes.signature, locations, llapDaemonInfos, emptySchema, llapUser, tokenBytes); } else { result[i] = new LlapInputSplit(i, submitWorkBytes, eventBytes.message, - eventBytes.signature, locations, llapDaemonInfos, schema, llapUser, tokenBytes, jwt); + eventBytes.signature, locations, llapDaemonInfos, schema, llapUser, tokenBytes); } } splitResult.actualSplits = result; @@ -668,12 +656,7 @@ private LlapDaemonInfo[] populateLlapDaemonInfos(JobConf job, SplitLocationInfo[ LlapDaemonInfo[] llapDaemonInfos = new LlapDaemonInfo[llapServiceInstances.size()]; int count = 0; for (LlapServiceInstance inst : llapServiceInstances) { - LlapDaemonInfo info; - if (LlapUtil.isCloudDeployment(job)) { - info = new LlapDaemonInfo(inst.getExternalHostname(), inst.getExternalClientsRpcPort(), inst.getOutputFormatPort()); - } else { - info = new LlapDaemonInfo(inst.getHost(), inst.getRpcPort(), inst.getOutputFormatPort()); - } + LlapDaemonInfo info = new LlapDaemonInfo(inst.getHost(), inst.getRpcPort(), inst.getOutputFormatPort()); llapDaemonInfos[count++] = info; } return llapDaemonInfos;