Merge pull request #53 from shiva-cirus/feature/PLUGIN-420

rmstar · web-flow · commit 92d33cbc66e8 · 2020-12-21T13:38:32.000-08:00
Plugin change to support Salesforce Security Token.
diff --git a/docs/Salesforce-batchsink.md b/docs/Salesforce-batchsink.md
@@ -17,6 +17,9 @@ Configuration
 
 **Password:** Salesforce password.
 
+**Security Token:** Salesforce security Token. If the password does not contain the security token the plugin 
+will append the token before authenticating with salesforce.
+
 **Consumer Key:** Application Consumer Key. This is also known as the OAuth client id.
 A Salesforce connected application must be created in order to get a consumer key.
 
diff --git a/docs/Salesforce-batchsource.md b/docs/Salesforce-batchsource.md
@@ -18,6 +18,9 @@ Configuration
 
 **Password:** Salesforce password.
 
+**Security Token:** Salesforce security Token. If the password does not contain the security token the plugin 
+will append the token before authenticating with salesforce.
+
 **Consumer Key:** Application Consumer Key. This is also known as the OAuth client id.
 A Salesforce connected application must be created in order to get a consumer key.
 
diff --git a/docs/Salesforce-streamingsource.md b/docs/Salesforce-streamingsource.md
@@ -25,6 +25,9 @@ Configuration
 
 **Password:** Salesforce password.
 
+**Security Token:** Salesforce security Token. If the password does not contain the security token the plugin 
+will append the token before authenticating with salesforce.
+
 **Consumer Key:** Application Consumer Key. This is also known as the OAuth client id.
 A Salesforce connected application must be created in order to get a consumer key.
 
diff --git a/docs/SalesforceMultiObjects-batchsource.md b/docs/SalesforceMultiObjects-batchsource.md
@@ -18,6 +18,9 @@ Configuration
 
 **Password:** Salesforce password.
 
+**Security Token:** Salesforce security Token. If the password does not contain the security token the plugin 
+will append the token before authenticating with salesforce.
+
 **Consumer Key:** Application Consumer Key. This is also known as the OAuth client id.
 A Salesforce connected application must be created in order to get a consumer key.
 
diff --git a/pom.xml b/pom.xml
@@ -30,13 +30,13 @@
     <surefire.redirectTestOutputToFile>true</surefire.redirectTestOutputToFile>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <!-- version properties -->
-    <cdap.version>6.2.0-SNAPSHOT</cdap.version>
+    <cdap.version>6.2.0</cdap.version>
     <junit.version>4.12</junit.version>
     <commons.io.version>2.6</commons.io.version>
     <commons.lang.version>2.6</commons.lang.version>
     <hadoop.version>2.3.0</hadoop.version>
     <spark2.version>2.1.3</spark2.version>
-    <hydrator.version>2.4.0-SNAPSHOT</hydrator.version>
+    <hydrator.version>2.4.0</hydrator.version>
     <commons.version>3.8.1</commons.version>
     <salesforce.api.version>45.0.0</salesforce.api.version>
     <cometd.java.client.version>4.0.0</cometd.java.client.version>
diff --git a/src/main/java/io/cdap/plugin/salesforce/SalesforceConstants.java b/src/main/java/io/cdap/plugin/salesforce/SalesforceConstants.java
@@ -27,6 +27,7 @@ public class SalesforceConstants {
   public static final String PROPERTY_CONSUMER_SECRET = "consumerSecret";
   public static final String PROPERTY_USERNAME = "username";
   public static final String PROPERTY_PASSWORD = "password";
+  public static final String PROPERTY_SECURITY_TOKEN = "securityToken";
   public static final String PROPERTY_LOGIN_URL = "loginUrl";
 
   public static final String CONFIG_CONSUMER_KEY = "mapred.salesforce.consumer.key";
diff --git a/src/main/java/io/cdap/plugin/salesforce/plugin/BaseSalesforceConfig.java b/src/main/java/io/cdap/plugin/salesforce/plugin/BaseSalesforceConfig.java
@@ -25,6 +25,8 @@
 import io.cdap.plugin.salesforce.SalesforceConstants;
 import io.cdap.plugin.salesforce.authenticator.AuthenticatorCredentials;
 
+import javax.annotation.Nullable;
+
 /**
  * Base configuration for Salesforce Streaming and Batch plugins
  */
@@ -50,19 +52,27 @@ public class BaseSalesforceConfig extends ReferencePluginConfig {
   @Macro
   private String password;
 
+  @Name(SalesforceConstants.PROPERTY_SECURITY_TOKEN)
+  @Description("Salesforce security token")
+  @Nullable
+  @Macro
+  private String securityToken;
+
   @Name(SalesforceConstants.PROPERTY_LOGIN_URL)
   @Description("Endpoint to authenticate to")
   @Macro
   private String loginUrl;
 
   public BaseSalesforceConfig(String referenceName, String consumerKey, String consumerSecret,
-                              String username, String password, String loginUrl) {
+                              String username, String password, String loginUrl,
+                              @Nullable String securityToken) {
     super(referenceName);
     this.consumerKey = consumerKey;
     this.consumerSecret = consumerSecret;
     this.username = username;
     this.password = password;
     this.loginUrl = loginUrl;
+    this.securityToken = securityToken;
   }
 
   public String getConsumerKey() {
@@ -78,7 +88,7 @@ public String getUsername() {
   }
 
   public String getPassword() {
-    return password;
+    return constructPasswordWithToken(password, securityToken);
   }
 
   public String getLoginUrl() {
@@ -96,7 +106,7 @@ public void validate(FailureCollector collector) {
   }
 
   public AuthenticatorCredentials getAuthenticatorCredentials() {
-    return SalesforceConnectionUtil.getAuthenticatorCredentials(username, password,
+    return SalesforceConnectionUtil.getAuthenticatorCredentials(username, getPassword(),
                                                                 consumerKey, consumerSecret, loginUrl);
   }
 
@@ -111,7 +121,8 @@ public boolean canAttemptToEstablishConnection() {
       || containsMacro(SalesforceConstants.PROPERTY_CONSUMER_SECRET)
       || containsMacro(SalesforceConstants.PROPERTY_USERNAME)
       || containsMacro(SalesforceConstants.PROPERTY_PASSWORD)
-      || containsMacro(SalesforceConstants.PROPERTY_LOGIN_URL));
+      || containsMacro(SalesforceConstants.PROPERTY_LOGIN_URL)
+      || containsMacro(SalesforceConstants.PROPERTY_SECURITY_TOKEN));
   }
 
   private void validateConnection() {
@@ -125,4 +136,12 @@ private void validateConnection() {
       throw new RuntimeException("There was issue communicating with Salesforce. " + e.getMessage(), e);
     }
   }
+
+  private String constructPasswordWithToken(String password, @Nullable String securityToken) {
+    if (securityToken != null && !securityToken.isEmpty() && !password.endsWith(securityToken)) {
+      return password + securityToken;
+    } else {
+      return password;
+    }
+  }
 }
diff --git a/src/main/java/io/cdap/plugin/salesforce/plugin/sink/batch/SalesforceSinkConfig.java b/src/main/java/io/cdap/plugin/salesforce/plugin/sink/batch/SalesforceSinkConfig.java
@@ -107,8 +107,8 @@ public SalesforceSinkConfig(String referenceName, String clientId,
                               String password, String loginUrl, String sObject,
                               String operation, String externalIdField,
                               String maxBytesPerBatch, String maxRecordsPerBatch,
-                              String errorHandling) {
-    super(referenceName, clientId, clientSecret, username, password, loginUrl);
+                              String errorHandling, @Nullable String securityToken) {
+    super(referenceName, clientId, clientSecret, username, password, loginUrl, securityToken);
     this.sObject = sObject;
     this.operation = operation;
     this.externalIdField = externalIdField;
diff --git a/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceBaseSourceConfig.java b/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceBaseSourceConfig.java
@@ -84,8 +84,9 @@ protected SalesforceBaseSourceConfig(String referenceName,
                                        @Nullable String datetimeAfter,
                                        @Nullable String datetimeBefore,
                                        @Nullable String duration,
-                                       @Nullable String offset) {
-    super(referenceName, consumerKey, consumerSecret, username, password, loginUrl);
+                                       @Nullable String offset,
+                                       @Nullable String securityToken) {
+    super(referenceName, consumerKey, consumerSecret, username, password, loginUrl, securityToken);
     this.datetimeAfter = datetimeAfter;
     this.datetimeBefore = datetimeBefore;
     this.duration = duration;
diff --git a/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceBatchMultiSource.java b/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceBatchMultiSource.java
@@ -83,12 +83,15 @@ public void prepareRun(BatchSourceContext context) throws ConnectionException {
     String sObjectNameField = config.getSObjectNameField();
     context.setInput(Input.of(config.referenceName, new SalesforceInputFormatProvider(
       config, queries, getSchemaWithNameField(sObjectNameField, schemas), sObjectNameField)));
-
+    /* TODO PLUGIN-510
+     *  As part of [CDAP-16290], recordLineage function was introduced with out implementation.
+     *  To avoid compilation errors the code block is commented for future fix.
     Schema schema = context.getInputSchema();
     if (schema != null && schema.getFields() != null) {
       recordLineage(context, config.referenceName, schema,
                     "Read", "Read from Salesforce MultiObjects.");
     }
+     */
   }
 
   @Override
diff --git a/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceMultiSourceConfig.java b/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceMultiSourceConfig.java
@@ -80,9 +80,10 @@ public SalesforceMultiSourceConfig(String referenceName,
                               @Nullable String offset,
                               @Nullable String whiteList,
                               @Nullable String blackList,
-                              @Nullable String sObjectNameField) {
+                              @Nullable String sObjectNameField,
+                              @Nullable String securityToken) {
     super(referenceName, consumerKey, consumerSecret, username, password, loginUrl,
-          datetimeAfter, datetimeBefore, duration, offset);
+          datetimeAfter, datetimeBefore, duration, offset, securityToken);
     this.whiteList = whiteList;
     this.blackList = blackList;
     this.sObjectNameField = sObjectNameField;
diff --git a/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceSourceConfig.java b/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceSourceConfig.java
@@ -75,9 +75,10 @@ public class SalesforceSourceConfig extends SalesforceBaseSourceConfig {
                                 @Nullable String datetimeBefore,
                                 @Nullable String duration,
                                 @Nullable String offset,
-                                @Nullable String schema) {
+                                @Nullable String schema,
+                                @Nullable String securityToken) {
     super(referenceName, consumerKey, consumerSecret, username, password, loginUrl,
-          datetimeAfter, datetimeBefore, duration, offset);
+          datetimeAfter, datetimeBefore, duration, offset, securityToken);
     this.query = query;
     this.sObjectName = sObjectName;
     this.schema = schema;
diff --git a/src/main/java/io/cdap/plugin/salesforce/plugin/source/streaming/SalesforceStreamingSourceConfig.java b/src/main/java/io/cdap/plugin/salesforce/plugin/source/streaming/SalesforceStreamingSourceConfig.java
@@ -107,8 +107,8 @@ public class SalesforceStreamingSourceConfig extends BaseSalesforceConfig implem
 
   public SalesforceStreamingSourceConfig(String referenceName, String consumerKey, String consumerSecret,
                                          String username, String password, String loginUrl,
-                                         String pushTopicName, String sObjectName) {
-    super(referenceName, consumerKey, consumerSecret, username, password, loginUrl);
+                                         String pushTopicName, String sObjectName, @Nullable String securityToken) {
+    super(referenceName, consumerKey, consumerSecret, username, password, loginUrl, securityToken);
     this.pushTopicName = pushTopicName;
     this.sObjectName = sObjectName;
   }
diff --git a/src/test/java/io/cdap/plugin/salesforce/etl/BaseSalesforceBatchSinkETLTest.java b/src/test/java/io/cdap/plugin/salesforce/etl/BaseSalesforceBatchSinkETLTest.java
@@ -228,6 +228,7 @@ protected SalesforceSinkConfig getDefaultConfig(String sObject) {
                                     BaseSalesforceETLTest.CONSUMER_KEY, BaseSalesforceETLTest.CONSUMER_SECRET,
                                     BaseSalesforceETLTest.USERNAME, BaseSalesforceETLTest.PASSWORD,
                                     BaseSalesforceETLTest.LOGIN_URL, sObject, "Insert", null,
-                                    "1000000", "10000", "Stop on Error");
+                                    "1000000", "10000", "Stop on Error",
+                                    BaseSalesforceETLTest.SECURITY_TOKEN);
   }
 }
diff --git a/src/test/java/io/cdap/plugin/salesforce/etl/BaseSalesforceETLTest.java b/src/test/java/io/cdap/plugin/salesforce/etl/BaseSalesforceETLTest.java
@@ -56,7 +56,7 @@
  * mvn clean test
  * -Dsalesforce.test.consumerKey= -Dsalesforce.test.consumerSecret=
  * -Dsalesforce.test.username= -Dsalesforce.test.password=
- *
+ * -Dsalesforce.test.securityToken=
  */
 public abstract class BaseSalesforceETLTest extends HydratorTestBase {
 
@@ -68,6 +68,9 @@ public abstract class BaseSalesforceETLTest extends HydratorTestBase {
   protected static final String PASSWORD = System.getProperty("salesforce.test.password");
   protected static final String LOGIN_URL = System.getProperty("salesforce.test.loginUrl",
                                                              "https://login.salesforce.com/services/oauth2/token");
+  protected static final String SECURITY_TOKEN = System.getProperty("salesforce.test.securityToken",
+                                                               "");
+
   public static final int SOAP_RECORDS_LIMIT = 200;
 
   @Rule
@@ -152,7 +155,8 @@ protected ImmutableMap.Builder<String, String> getBaseProperties(String referenc
       .put(SalesforceConstants.PROPERTY_CONSUMER_SECRET, CONSUMER_SECRET)
       .put(SalesforceConstants.PROPERTY_USERNAME, USERNAME)
       .put(SalesforceConstants.PROPERTY_PASSWORD, PASSWORD)
-      .put(SalesforceConstants.PROPERTY_LOGIN_URL, LOGIN_URL);
+      .put(SalesforceConstants.PROPERTY_LOGIN_URL, LOGIN_URL)
+      .put(SalesforceConstants.PROPERTY_SECURITY_TOKEN, SECURITY_TOKEN);
   }
 
   private void clearSObjects() {
diff --git a/src/test/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceSourceConfigBuilder.java b/src/test/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceSourceConfigBuilder.java
@@ -33,6 +33,7 @@ public class SalesforceSourceConfigBuilder {
   private String duration;
   private String offset;
   private String schema;
+  private String securityToken;
 
   public SalesforceSourceConfigBuilder setReferenceName(String referenceName) {
     this.referenceName = referenceName;
@@ -99,8 +100,15 @@ public SalesforceSourceConfigBuilder setSchema(String schema) {
     return this;
   }
 
+  public SalesforceSourceConfigBuilder setSecurityToken(String securityToken) {
+    this.securityToken = securityToken;
+    return this;
+  }
+
+
   public SalesforceSourceConfig build() {
     return new SalesforceSourceConfig(referenceName, consumerKey, consumerSecret, username, password, loginUrl,
-                                      query, sObjectName, datetimeAfter, datetimeBefore, duration, offset, schema);
+                                      query, sObjectName, datetimeAfter, datetimeBefore, duration, offset, schema,
+                                      securityToken);
   }
 }
diff --git a/widgets/Salesforce-batchsink.json b/widgets/Salesforce-batchsink.json
@@ -26,6 +26,11 @@
           "label": "Password",
           "name": "password"
         },
+        {
+          "widget-type": "password",
+          "label": "Security Token",
+          "name": "securityToken"
+        },
         {
           "widget-type": "textbox",
           "label": "Consumer Key",
diff --git a/widgets/Salesforce-batchsource.json b/widgets/Salesforce-batchsource.json
@@ -26,6 +26,11 @@
           "label": "Password",
           "name": "password"
         },
+        {
+          "widget-type": "password",
+          "label": "Security Token",
+          "name": "securityToken"
+        },
         {
           "widget-type": "textbox",
           "label": "Consumer Key",
diff --git a/widgets/Salesforce-streamingsource.json b/widgets/Salesforce-streamingsource.json
@@ -26,6 +26,11 @@
           "label": "Password",
           "name": "password"
         },
+        {
+          "widget-type": "password",
+          "label": "Security Token",
+          "name": "securityToken"
+        },
         {
           "widget-type": "textbox",
           "label": "Consumer Key",
diff --git a/widgets/SalesforceMultiObjects-batchsource.json b/widgets/SalesforceMultiObjects-batchsource.json
@@ -26,6 +26,11 @@
           "label": "Password",
           "name": "password"
         },
+        {
+          "widget-type": "password",
+          "label": "Security Token",
+          "name": "securityToken"
+        },
         {
           "widget-type": "textbox",
           "label": "Consumer Key",
