Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 0 additions & 28 deletions common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
Original file line number Diff line number Diff line change
Expand Up @@ -5381,34 +5381,6 @@ public static enum ConfVars {
LLAP_EXTERNAL_SPLITS_TEMP_TABLE_STORAGE_FORMAT("hive.llap.external.splits.temp.table.storage.format",
"orc", new StringSet("default", "text", "orc"),
"Storage format for temp tables created using LLAP external client"),
LLAP_EXTERNAL_CLIENT_USE_HYBRID_CALENDAR("hive.llap.external.client.use.hybrid.calendar",
false,
"Whether to use hybrid calendar for parsing of data/timestamps."),

// ====== confs for llap-external-client cloud deployment ======
LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED(
"hive.llap.external.client.cloud.deployment.setup.enabled", false,
"Tells whether to enable additional RPC port, auth mechanism for llap external clients. This is meant"
+ "for cloud based deployments. When true, it has following effects - \n"
+ "1. Enables an extra RPC port on LLAP daemon to accept fragments from external clients. See"
+ "hive.llap.external.client.cloud.rpc.port\n"
+ "2. Uses external hostnames of LLAP in splits, so that clients can submit from outside of cloud. "
+ "Env variable PUBLIC_HOSTNAME should be available on LLAP machines.\n"
+ "3. Uses JWT based authentication for splits to be validated at LLAP. See "
+ "hive.llap.external.client.cloud.jwt.shared.secret.provider"),
LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT("hive.llap.external.client.cloud.rpc.port", 30004,
"The LLAP daemon RPC port for external clients when llap is running in cloud environment."),
LLAP_EXTERNAL_CLIENT_CLOUD_OUTPUT_SERVICE_PORT("hive.llap.external.client.cloud.output.service.port", 30005,
"LLAP output service port when llap is running in cloud environment"),
LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_PROVIDER(
"hive.llap.external.client.cloud.jwt.shared.secret.provider",
"org.apache.hadoop.hive.llap.security.DefaultJwtSharedSecretProvider",
"Shared secret provider to be used to sign JWT"),
LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET("hive.llap.external.client.cloud.jwt.shared.secret",
"",
"The LLAP daemon RPC port for external clients when llap is running in cloud environment. "
+ "Length of the secret should be >= 32 bytes"),
// ====== confs for llap-external-client cloud deployment ======

LLAP_ENABLE_GRACE_JOIN_IN_LLAP("hive.llap.enable.grace.join.in.llap", false,
"Override if grace join should be allowed to run in llap."),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public void testWritable() throws Exception {

byte[] tokenBytes = new byte[] { 1 };
LlapInputSplit split1 = new LlapInputSplit(splitNum, planBytes, fragmentBytes, null,
locations, llapDaemonInfos, schema, "hive", tokenBytes, "some-dummy-jwt");
locations, llapDaemonInfos, schema, "hive", tokenBytes);
ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream();
DataOutputStream dataOut = new DataOutputStream(byteOutStream);
split1.write(dataOut);
Expand Down Expand Up @@ -89,7 +89,6 @@ static void checkLlapSplits(LlapInputSplit split1, LlapInputSplit split2) throws
assertArrayEquals(split1.getLocations(), split2.getLocations());
assertEquals(split1.getSchema().toString(), split2.getSchema().toString());
assertEquals(split1.getLlapUser(), split2.getLlapUser());
assertEquals(split1.getJwt(), split2.getJwt());
assertArrayEquals(split1.getLlapDaemonInfos(), split2.getLlapDaemonInfos());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,18 +37,14 @@ public class LlapInputSplit implements InputSplitWithLocationInfo {
private String llapUser;
private byte[] fragmentBytesSignature;
private byte[] tokenBytes;
//only needed in cloud deployments for llap server to validate request from external llap clients.
//HS2 generates a JWT and populates this field while get_splits() call, this jwt gets validated at LLAP server
//when LlapInputSplit is submitted.
private String jwt;

public LlapInputSplit() {
}

public LlapInputSplit(int splitNum, byte[] planBytes, byte[] fragmentBytes,
byte[] fragmentBytesSignature, SplitLocationInfo[] locations,
LlapDaemonInfo[] llapDaemonInfos, Schema schema,
String llapUser, byte[] tokenBytes, String jwt) {
String llapUser, byte[] tokenBytes) {
this.planBytes = planBytes;
this.fragmentBytes = fragmentBytes;
this.fragmentBytesSignature = fragmentBytesSignature;
Expand All @@ -58,7 +54,6 @@ public LlapInputSplit(int splitNum, byte[] planBytes, byte[] fragmentBytes,
this.splitNum = splitNum;
this.llapUser = llapUser;
this.tokenBytes = tokenBytes;
this.jwt = jwt;
}

public Schema getSchema() {
Expand Down Expand Up @@ -107,10 +102,6 @@ public void setSchema(Schema schema) {
this.schema = schema;
}

public String getJwt() {
return jwt;
}

@Override
public void write(DataOutput out) throws IOException {
out.writeInt(splitNum);
Expand Down Expand Up @@ -145,9 +136,8 @@ public void write(DataOutput out) throws IOException {
out.writeInt(0);
}

if (jwt != null) {
out.writeUTF(jwt);
}
// Retain the retired JWT field in the wire format for older readers.
out.writeUTF("");
}

@Override
Expand Down Expand Up @@ -187,7 +177,9 @@ public void readFields(DataInput in) throws IOException {
tokenBytes = new byte[length];
in.readFully(tokenBytes);
}
jwt = in.readUTF();

// Discard the retired JWT field kept for wire compatibility.
in.readUTF();
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@
package org.apache.hadoop.hive.llap.registry;


import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.llap.LlapUtil;
import org.apache.hadoop.hive.registry.ServiceInstance;

import org.apache.hadoop.yarn.api.records.Resource;
Expand Down Expand Up @@ -55,25 +52,6 @@ public interface LlapServiceInstance extends ServiceInstance {
*/
public int getOutputFormatPort();

/**
* External host, usually needed in cloud envs where we cannot access internal host from outside
*
* @return
*/
String getExternalHostname();

/**
* RPC endpoint for external clients - tcp traffic on this port should be opened on cloud.
*
* @return
*/
int getExternalClientsRpcPort();


default void ensureCloudEnv(Configuration conf) {
Preconditions.checkState(LlapUtil.isCloudDeployment(conf), "Only supported in cloud based deployments");
}

/**
* Memory and Executors available for the LLAP tasks
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,16 +56,6 @@ public int getShufflePort() {
throw new UnsupportedOperationException();
}

@Override
public String getExternalHostname() {
throw new UnsupportedOperationException();
}

@Override
public int getExternalClientsRpcPort() {
throw new UnsupportedOperationException();
}

@Override
public String getServicesAddress() {
throw new UnsupportedOperationException();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,9 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.LlapUtil;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstance;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstanceSet;
import org.apache.hadoop.hive.llap.registry.ServiceRegistry;
import org.apache.hadoop.hive.registry.ServiceInstance;
import org.apache.hadoop.hive.registry.ServiceInstanceStateChangeListener;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.StringUtils;
Expand All @@ -65,9 +63,7 @@ public class LlapFixedRegistryImpl implements ServiceRegistry<LlapServiceInstanc
private final int shuffle;
private final int mngPort;
private final int webPort;
private Configuration conf;
private final int outputFormatPort;
private final int externalClientsRpcPort;
private final String webScheme;
private final String[] hosts;
private final int memory;
Expand All @@ -83,10 +79,8 @@ public LlapFixedRegistryImpl(String hosts, Configuration conf) {
this.resolveHosts = conf.getBoolean(FIXED_REGISTRY_RESOLVE_HOST_NAMES, true);
this.mngPort = HiveConf.getIntVar(conf, ConfVars.LLAP_MANAGEMENT_RPC_PORT);
this.outputFormatPort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT);
this.externalClientsRpcPort = HiveConf.getIntVar(conf, ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT);

this.webPort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_WEB_PORT);
this.conf = conf;
boolean isSsl = HiveConf.getBoolVar(conf, ConfVars.LLAP_DAEMON_WEB_SSL);
this.webScheme = isSsl ? "https" : "http";

Expand Down Expand Up @@ -199,18 +193,6 @@ public int getOutputFormatPort() {
return LlapFixedRegistryImpl.this.outputFormatPort;
}

@Override
public String getExternalHostname() {
ensureCloudEnv(LlapFixedRegistryImpl.this.conf);
return LlapUtil.getPublicHostname();
}

@Override
public int getExternalClientsRpcPort() {
ensureCloudEnv(LlapFixedRegistryImpl.this.conf);
return LlapFixedRegistryImpl.this.externalClientsRpcPort;
}

@Override
public String getServicesAddress() {
return serviceAddress;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.LlapUtil;
import org.apache.hadoop.hive.llap.io.api.LlapProxy;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstance;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstanceSet;
Expand Down Expand Up @@ -73,7 +72,6 @@ public class LlapZookeeperRegistryImpl
private static final String IPC_SHUFFLE = "shuffle";
private static final String IPC_LLAP = "llap";
private static final String IPC_OUTPUTFORMAT = "llapoutputformat";
private static final String IPC_EXTERNAL_LLAP = "externalllap";
private final static String NAMESPACE_PREFIX = "llap-";
private static final String SLOT_PREFIX = "slot-";
private static final String SASL_LOGIN_CONTEXT_NAME = "LlapZooKeeperClient";
Expand Down Expand Up @@ -137,12 +135,6 @@ public Endpoint getOutputFormatEndpoint() {
HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT)));
}

private Endpoint getExternalRpcEndpoint() {
int port = HiveConf.getIntVar(conf, ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT);
String host = LlapUtil.getPublicHostname();
return RegistryTypeUtils.ipcEndpoint(IPC_EXTERNAL_LLAP, new InetSocketAddress(host, port));
}

@Override
public String register() throws IOException {
daemonZkRecord = new ServiceRecord();
Expand All @@ -152,11 +144,6 @@ public String register() throws IOException {
daemonZkRecord.addInternalEndpoint(getShuffleEndpoint());
daemonZkRecord.addExternalEndpoint(getServicesEndpoint());
daemonZkRecord.addInternalEndpoint(getOutputFormatEndpoint());
Endpoint externalRpcEndpoint = null;
if (LlapUtil.isCloudDeployment(conf)) {
externalRpcEndpoint = getExternalRpcEndpoint();
daemonZkRecord.addExternalEndpoint(externalRpcEndpoint);
}

populateConfigValues(this.conf);
Map<String, String> capacityValues = new HashMap<>(2);
Expand Down Expand Up @@ -196,15 +183,9 @@ public String register() throws IOException {
}

registerServiceRecord(daemonZkRecord, uniqueId);
if (LlapUtil.isCloudDeployment(conf)) {
LOG.info("Registered node. Created a znode on ZooKeeper for LLAP instance: rpc: {}, external client rpc : {} "
+ "shuffle: {}, webui: {}, mgmt: {}, znodePath: {}", rpcEndpoint, externalRpcEndpoint,
getShuffleEndpoint(), getServicesEndpoint(), getMngEndpoint(), getRegistrationZnodePath());
} else {
LOG.info("Registered node. Created a znode on ZooKeeper for LLAP instance: rpc: {}, "
+ "shuffle: {}, webui: {}, mgmt: {}, znodePath: {}", rpcEndpoint, getShuffleEndpoint(),
getServicesEndpoint(), getMngEndpoint(), getRegistrationZnodePath());
}
LOG.info("Registered node. Created a znode on ZooKeeper for LLAP instance: rpc: {}, "
+ "shuffle: {}, webui: {}, mgmt: {}, znodePath: {}", rpcEndpoint, getShuffleEndpoint(),
getServicesEndpoint(), getMngEndpoint(), getRegistrationZnodePath());

return uniqueId;
}
Expand Down Expand Up @@ -242,9 +223,6 @@ public class DynamicServiceInstance
private final int outputFormatPort;
private final String serviceAddress;

private String externalHost;
private int externalClientsRpcPort;

private final Resource resource;

public DynamicServiceInstance(ServiceRecord srv) throws IOException {
Expand All @@ -267,15 +245,6 @@ public DynamicServiceInstance(ServiceRecord srv) throws IOException {
this.serviceAddress =
RegistryTypeUtils.getAddressField(services.addresses.get(0), AddressTypes.ADDRESS_URI);

if (LlapUtil.isCloudDeployment(conf)) {
final Endpoint externalRpc = srv.getExternalEndpoint(IPC_EXTERNAL_LLAP);
this.externalHost = RegistryTypeUtils.getAddressField(externalRpc.addresses.get(0),
AddressTypes.ADDRESS_HOSTNAME_FIELD);
this.externalClientsRpcPort = Integer.parseInt(
RegistryTypeUtils.getAddressField(externalRpc.addresses.get(0),
AddressTypes.ADDRESS_PORT_FIELD));
}

String memStr = srv.get(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, "");
String coreStr = srv.get(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS, "");
try {
Expand All @@ -296,18 +265,6 @@ public String getServicesAddress() {
return serviceAddress;
}

@Override
public String getExternalHostname() {
ensureCloudEnv(LlapZookeeperRegistryImpl.this.conf);
return externalHost;
}

@Override
public int getExternalClientsRpcPort() {
ensureCloudEnv(LlapZookeeperRegistryImpl.this.conf);
return externalClientsRpcPort;
}

@Override
public Resource getResource() {
return resource;
Expand Down
12 changes: 0 additions & 12 deletions llap-common/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -48,18 +48,6 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt-api</artifactId>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt-impl</artifactId>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt-jackson</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
Expand Down
20 changes: 0 additions & 20 deletions llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -297,24 +297,4 @@ public static Credentials credentialsFromByteArray(byte[] binaryCredentials)
credentials.readTokenStorageStream(dib);
return credentials;
}

/**
* @return returns the value of LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED
* @param conf
*/
public static boolean isCloudDeployment(Configuration conf) {
return HiveConf.getBoolVar(conf, ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED, false);
}

/**
* @return returns the value of PUBLIC_HOSTNAME from either environment variable or system properties
*/
public static String getPublicHostname() {
String publicHostname = System.getenv("PUBLIC_HOSTNAME");
if (publicHostname == null) {
publicHostname = System.getProperty("PUBLIC_HOSTNAME");
}
return publicHostname;
}

}
Loading
Loading