From fd593c43b5365897ed1019737a96b1d2e5af8a71 Mon Sep 17 00:00:00 2001 From: yjhawar Date: Wed, 5 Apr 2023 10:29:37 +0530 Subject: [PATCH 01/14] Added e2e tests with snapshot and cdc --- pom.xml | 123 ++++++++++++++ src/e2e-test/features/Pipeline.feature | 78 +++++++++ .../actions/ReplicationActions.java | 154 ++++++++++++++++++ .../io.cdap.plugin/actions/package-info.java | 17 ++ .../io.cdap.plugin/hooks/TestSetUpHooks.java | 86 ++++++++++ .../locators/ReplicationLocators.java | 49 ++++++ .../io.cdap.plugin/locators/package-info.java | 17 ++ .../stepsdesign/StepDefinition.java | 99 +++++++++++ .../java/io.cdap.plugin/tests/TestRunner.java | 35 ++++ .../io.cdap.plugin/tests/package-info.java | 20 +++ .../io.cdap.plugin/utils/OracleClient.java | 132 +++++++++++++++ .../utils/ValidationHelper.java | 149 +++++++++++++++++ .../resources/errorMessage.properties | 1 + .../pluginDataCyAttributes.properties | 6 + .../resources/pluginParameters.properties | 42 +++++ 15 files changed, 1008 insertions(+) create mode 100644 src/e2e-test/features/Pipeline.feature create mode 100644 src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java create mode 100644 src/e2e-test/java/io.cdap.plugin/actions/package-info.java create mode 100644 src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java create mode 100644 src/e2e-test/java/io.cdap.plugin/locators/ReplicationLocators.java create mode 100644 src/e2e-test/java/io.cdap.plugin/locators/package-info.java create mode 100644 src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java create mode 100644 src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java create mode 100644 src/e2e-test/java/io.cdap.plugin/tests/package-info.java create mode 100644 src/e2e-test/java/io.cdap.plugin/utils/OracleClient.java create mode 100644 src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java create mode 100644 src/e2e-test/resources/errorMessage.properties create mode 100644 src/e2e-test/resources/pluginDataCyAttributes.properties create mode 100644 src/e2e-test/resources/pluginParameters.properties diff --git a/pom.xml b/pom.xml index d81813d..97fdc74 100644 --- a/pom.xml +++ b/pom.xml @@ -385,6 +385,129 @@ + + e2e-tests + + src/e2e-test/java + TestRunner.java + + + + + src/e2e-test/resources + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.18.1 + + true + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 3.0.0-M5 + + + ${TEST_RUNNER} + + + classes + 2 + 2 + true + + + + ${GOOGLE_APPLICATION_CREDENTIALS} + + + ${SERVICE_ACCOUNT_TYPE} + + + ${SERVICE_ACCOUNT_FILE_PATH} + + + ${SERVICE_ACCOUNT_JSON} + + + + + + + integration-test + + + + + + + net.masterthought + maven-cucumber-reporting + 5.5.0 + + + + execution + verify + + generate + + + Cucumber Reports + target/cucumber-reports/advanced-reports + 1 + false + ${project.build.directory}/cucumber-reports + + **/*.json + + ${project.build.directory}/cucumber-reports + true + + + + + + + + + + + com.google.guava + guava + 31.1-jre + + + + + + + com.oracle.database.jdbc + ojdbc8 + 21.1.0.0 + test + + + io.cdap.tests.e2e + cdap-e2e-framework + 0.3.0-SNAPSHOT + test + + + ch.qos.logback + logback-classic + 1.2.8 + runtime + + + + + diff --git a/src/e2e-test/features/Pipeline.feature b/src/e2e-test/features/Pipeline.feature new file mode 100644 index 0000000..51b8224 --- /dev/null +++ b/src/e2e-test/features/Pipeline.feature @@ -0,0 +1,78 @@ +# +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +Feature: Oracle - Verify Oracle source data transfer + +# Scenario: Sanity test from Oracle to Big Query +# Given Open DataFusion Project with replication to configure pipeline +# When Enter input plugin property: "name" with value: "pipelineName" +# And Click on the Next button +# And Select Oracle as Source +# Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields +# Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields +# Then Click plugin property: "region" +# Then Click plugin property: "regionOption" +# Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields +# Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields +# Then Replace input plugin property: "sid" with value: "database" for Credentials and Authorization related fields +# Then Click on the Next button +# Then Replace input plugin property: "loadInterval" with value: "loadInterval" +# Then Click on the Next button +# Then Validate Table is available and can be selected "ABC.E2E-sanity" +# And Click on the Next button +# And Click on the Next button +# And Click on the Next button +# Then Deploy the replication pipeline +# And Run the replication Pipeline +# Then Open the logs +# And Wait till pipeline is in running state and check if no errors occurred +# Then Verify expected Oracle records in target BigQuery table +# And Capture raw logs +# Then Close the pipeline logs and stop the pipeline + +# @ORACLE_SOURCE + Scenario: To verify snapshot and cdc from Oracle to Big Query successfully + Given Open DataFusion Project with replication to configure pipeline + When Enter input plugin property: "name" with value: "pipelineName" + And Click on the Next button + And Select Oracle as Source + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Click plugin property: "region" + Then Click plugin property: "regionOption" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Replace input plugin property: "sid" with value: "database" for Credentials and Authorization related fields + Then Click on the Next button + Then Replace input plugin property: "loadInterval" with value: "loadInterval" + Then Click on the Next button + Then Validate Table is available and can be selected "ABC.E2E1" + And Click on the Next button + And Click on the Next button + And Click on the Next button + Then Deploy the replication pipeline + And Run the replication Pipeline + Then Open the logs + And Wait till pipeline is in running state and check if no errors occurred + Then Verify expected Oracle records in target BigQuery table + And Insert a record in the source table and wait for replication + Then Verify expected Oracle records in target BigQuery table + And Delete a record in the source table and wait for replication + Then Verify expected Oracle records in target BigQuery table + And Update a record in the source table and wait for replication + Then Verify expected Oracle records in target BigQuery table + And Capture raw logs + Then Close the pipeline logs and stop the pipeline \ No newline at end of file diff --git a/src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java b/src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java new file mode 100644 index 0000000..b92acb1 --- /dev/null +++ b/src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.actions; + +import io.cdap.e2e.pages.actions.CdfPipelineRunAction; +import io.cdap.e2e.pages.locators.CdfPipelineRunLocators; +import io.cdap.e2e.utils.*; +import io.cdap.plugin.locators.ReplicationLocators; +import io.cdap.plugin.utils.OracleClient; +import io.cdap.plugin.utils.ValidationHelper; +import org.apache.commons.lang.StringUtils; +import org.junit.Assert; +import stepsdesign.BeforeActions; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class ReplicationActions { + private static String parentWindow = StringUtils.EMPTY; + private static final String projectId = PluginPropertyUtils.pluginProp("projectId"); + private static final String database = PluginPropertyUtils.pluginProp("database"); + public static String tableName = PluginPropertyUtils.pluginProp("sourceTable"); + public static String schemaName = PluginPropertyUtils.pluginProp("schema"); + public static String datatypeColumnNames = PluginPropertyUtils.pluginProp("datatypeColumnNames"); + public static String datatypeValues = PluginPropertyUtils.pluginProp("datatypeValuesForInsertOperation"); + public static String deleteCondition = PluginPropertyUtils.pluginProp("deleteRowCondition"); + public static String updateCondition = PluginPropertyUtils.pluginProp("updateRowCondition"); + public static String updatedValue = PluginPropertyUtils.pluginProp("updatedRow"); + + static { + SeleniumHelper.getPropertiesLocators(ReplicationLocators.class); + } + public static void clickNextButton() throws InterruptedException { + TimeUnit time = TimeUnit.SECONDS; + time.sleep(1); + ElementHelper.clickOnElement(ReplicationLocators.next); + } + public static void clickOnOraclePlugin() { + ElementHelper.clickOnElement(ReplicationLocators.oraclePlugin); + } + + public static void selectTable(String tableName) { + WaitHelper.waitForElementToBeDisplayed(ReplicationLocators.selectTable(tableName)); + AssertionHelper.verifyElementDisplayed(ReplicationLocators.selectTable(tableName)); + ElementHelper.clickOnElement(ReplicationLocators.selectTable(tableName)); + } + + public static void deployPipeline() { + ElementHelper.clickOnElement(ReplicationLocators.deployPipeline); + } + + public static void startPipeline() { + ElementHelper.clickIfDisplayed(ReplicationLocators.start, ConstantsUtil.DEFAULT_TIMEOUT_SECONDS); + } + + public static void runThePipeline() { + startPipeline(); + WaitHelper.waitForElementToBeDisplayed(ReplicationLocators.running); + } + + public static void openAdvanceLogs() { + ReplicationLocators.logs.click(); + parentWindow = SeleniumDriver.getDriver().getWindowHandle(); + ArrayList tabs = new ArrayList(SeleniumDriver.getDriver().getWindowHandles()); + SeleniumDriver.getDriver().switchTo().window(tabs.get(tabs.indexOf(parentWindow) + 1)); + ReplicationLocators.advancedLogs.click(); + } + + public static void captureRawLog() { + //Capturing raw logs. + try { + String rawLogs = getRawLogs(); + String logsSeparatorMessage = ConstantsUtil.LOGS_SEPARATOR_MESSAGE + .replace("MESSAGE", "DEPLOYED PIPELINE RUNTIME LOGS"); + BeforeActions.scenario.write(rawLogs); + CdfPipelineRunAction.writeRawLogsToFile(BeforeActions.file, logsSeparatorMessage, rawLogs); + } catch (Exception e) { + BeforeActions.scenario.write("Exception in capturing logs : " + e); + } + } + + public static String getRawLogs() { + CdfPipelineRunAction.viewRawLogs(); + ArrayList tabs = new ArrayList(SeleniumDriver.getDriver().getWindowHandles()); + PageHelper.switchToWindow(tabs.indexOf(parentWindow) + 2); + String logs = CdfPipelineRunLocators.logsTextbox.getText(); + Assert.assertNotNull(logs); + PageHelper.closeCurrentWindow(); + return logs; + } + + public static void waitTillPipelineIsRunningAndCheckForErrors() throws InterruptedException { + //wait for datastream to startup + int defaultTimeout = Integer.parseInt(PluginPropertyUtils.pluginProp("datastream.timeout")); + TimeUnit time = TimeUnit.SECONDS; + time.sleep(defaultTimeout); + // Checking if an error message is displayed. + Assert.assertFalse(ElementHelper.isElementDisplayed(ReplicationLocators.error)); + } + + public static void closeTheLogsAndClickOnStopButton() { + //As the logs get opened in a new window in this plugin so after closing them we have to switch to parent window. + SeleniumDriver.getDriver().switchTo().window(parentWindow); + //Stopping the pipeline + ElementHelper.clickOnElement(ReplicationLocators.stop); + WaitHelper.waitForElementToBeDisplayed(ReplicationLocators.stopped); + } + public static void verifyTargetBigQueryRecordMatchesExpectedOracleRecord() + throws IOException, InterruptedException, SQLException, ClassNotFoundException { + // Checking if an error message is displayed. + Assert.assertFalse(ElementHelper.isElementDisplayed(ReplicationLocators.error)); + + List> sourceOracleRecords = OracleClient.getOracleRecordsAsMap(tableName, schemaName); + List> targetBigQueryRecords = ValidationHelper.getBigQueryRecordsAsMap(projectId, database, tableName); //+ "_v1`" + ValidationHelper.validateRecords(sourceOracleRecords, targetBigQueryRecords); + } + + public static void insertRecordAndWait() + throws IOException, InterruptedException, SQLException, ClassNotFoundException { //JCoException, + OracleClient.insertRow(tableName,schemaName, datatypeColumnNames,datatypeValues); + OracleClient.forceFlushCDC(); + ValidationHelper.waitForFlush(); + } + + public static void deleteRecordAndWait() throws SQLException, ClassNotFoundException, IOException, InterruptedException { + OracleClient.deleteRow(tableName,schemaName, deleteCondition); + OracleClient.forceFlushCDC(); + ValidationHelper.waitForFlush(); + } + + public static void updateRecordAndWait() throws SQLException, ClassNotFoundException, IOException, InterruptedException { + OracleClient.updateRow(tableName,schemaName, updateCondition, updatedValue ); + OracleClient.forceFlushCDC(); + ValidationHelper.waitForFlush(); + } +} \ No newline at end of file diff --git a/src/e2e-test/java/io.cdap.plugin/actions/package-info.java b/src/e2e-test/java/io.cdap.plugin/actions/package-info.java new file mode 100644 index 0000000..8e01adf --- /dev/null +++ b/src/e2e-test/java/io.cdap.plugin/actions/package-info.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.actions; \ No newline at end of file diff --git a/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java b/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java new file mode 100644 index 0000000..dbffc05 --- /dev/null +++ b/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java @@ -0,0 +1,86 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.hooks; +import io.cdap.e2e.utils.PluginPropertyUtils; +import io.cdap.plugin.utils.OracleClient; +import io.cucumber.java.After; +import io.cucumber.java.Before; +import stepsdesign.BeforeActions; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + + +/** + * Oracle test hooks. + */ +public class TestSetUpHooks { + public static List> sourceOracleRecords = new ArrayList<>(); + public static String tableName = PluginPropertyUtils.pluginProp("sourceTable"); + public static String schemaName = PluginPropertyUtils.pluginProp("schema"); + public static String primaryKey = PluginPropertyUtils.pluginProp("primaryKey"); + public static String datatypeColumns = PluginPropertyUtils.pluginProp("datatypeColumns"); + public static String datatypeColumnNames = PluginPropertyUtils.pluginProp("datatypeColumnNames"); + public static String row1 = PluginPropertyUtils.pluginProp("datatypeValuesRow1"); + public static String row2= PluginPropertyUtils.pluginProp("datatypeValuesRow2"); + + + @Before(order = 1, value = "@ORACLE_SOURCE") + public static void overridePropertiesFromEnvVarsIfProvided() { + String username = System.getenv("ORACLE_USERNAME"); + if (username != null && !username.isEmpty()) { + PluginPropertyUtils.addPluginProp("username", username); + } + String password = System.getenv("ORACLE_PASSWORD"); + if (password != null && !password.isEmpty()) { + PluginPropertyUtils.addPluginProp("password", password); + } + String port = System.getenv("ORACLE_PORT"); + if (port!= null && !port.isEmpty()) { + PluginPropertyUtils.addPluginProp("port", port); + } + String sapHost = System.getenv("ORACLE_HOST"); + if (sapHost != null && !sapHost.isEmpty()) { + PluginPropertyUtils.addPluginProp("host", sapHost); + } + } + + @Before(order = 2, value = "@ORACLE_SOURCE") + public static void createTable() throws SQLException, ClassNotFoundException { + OracleClient.createTable(tableName,schemaName,datatypeColumns, primaryKey); + } + + @Before(order = 3, value = "@ORACLE_SOURCE") + public static void insertRow() throws SQLException, ClassNotFoundException { + OracleClient.insertRow(tableName, schemaName, datatypeColumnNames, row1); + OracleClient.insertRow(tableName, schemaName, datatypeColumnNames, row2); + sourceOracleRecords = OracleClient.getOracleRecordsAsMap(tableName, schemaName); + } + + @Before(order = 4, value = "@ORACLE_SOURCE") + public static void getOracleRecordsAsMap() throws SQLException, ClassNotFoundException { + sourceOracleRecords = OracleClient.getOracleRecordsAsMap(tableName, schemaName); + BeforeActions.scenario.write("Expected Oracle records : " + sourceOracleRecords); + } + + @After(order = 1, value = "@ORACLE_SOURCE_TEMP") + public static void dropTables() throws SQLException, ClassNotFoundException { + OracleClient.deleteTables(schemaName, tableName); + } +} \ No newline at end of file diff --git a/src/e2e-test/java/io.cdap.plugin/locators/ReplicationLocators.java b/src/e2e-test/java/io.cdap.plugin/locators/ReplicationLocators.java new file mode 100644 index 0000000..a5690b2 --- /dev/null +++ b/src/e2e-test/java/io.cdap.plugin/locators/ReplicationLocators.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.locators; + +import io.cdap.e2e.utils.SeleniumDriver; +import org.openqa.selenium.By; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.support.FindBy; +import org.openqa.selenium.support.How; + +public class ReplicationLocators { + @FindBy(how = How.XPATH, using = "//*[contains(text(),'Next')]") + public static WebElement next; + @FindBy(how = How.XPATH, using ="//div[contains(text(),'Oracle')]") + public static WebElement oraclePlugin; + public static WebElement selectTable(String tableName) { + return SeleniumDriver.getDriver().findElement(By.xpath("//div[contains(text(),'" + tableName + "')]" + + "/preceding-sibling::div/span")); + } + @FindBy(how = How.XPATH, using = "//span[contains(text(),'Deploy Replication Job')]") + public static WebElement deployPipeline; + @FindBy(how = How.XPATH, using = "//*[contains(text(), 'Running')]") + public static WebElement running; + @FindBy(how = How.XPATH, using = "//*[contains(text(), 'Logs')]") + public static WebElement logs; + @FindBy(how = How.XPATH, using = "(//*[contains(text(), 'View')])[1]") + public static WebElement advancedLogs; + @FindBy(how = How.XPATH, using = "//*[contains(@class, 'icon-stop')]") + public static WebElement stop; + @FindBy(how = How.XPATH, using = "//div[@data-cy='log-viewer-row']//div[contains(text(),'ERROR')]") + public static WebElement error; + @FindBy(how = How.XPATH, using = "//*[contains(text(),'Stopped')]") + public static WebElement stopped; + public static By start = By.xpath("//*[contains(@class, 'icon-play ')]"); +} \ No newline at end of file diff --git a/src/e2e-test/java/io.cdap.plugin/locators/package-info.java b/src/e2e-test/java/io.cdap.plugin/locators/package-info.java new file mode 100644 index 0000000..e9988de --- /dev/null +++ b/src/e2e-test/java/io.cdap.plugin/locators/package-info.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.locators; \ No newline at end of file diff --git a/src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java b/src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java new file mode 100644 index 0000000..5b73e6a --- /dev/null +++ b/src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.stepsdesign; + +import io.cdap.e2e.utils.CdfHelper; +import io.cdap.e2e.utils.PluginPropertyUtils; +import io.cdap.e2e.utils.SeleniumDriver; +import io.cdap.plugin.actions.ReplicationActions; +import io.cucumber.java.en.And; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; + +import java.io.IOException; +import java.sql.SQLException; + +public class StepDefinition implements CdfHelper { + @Given("Open DataFusion Project with replication to configure pipeline") + public void openDataFusionProjectWithReplicationToConfigurePipeline() throws IOException, InterruptedException { + openCdf(); + SeleniumDriver.getDriver().get(SeleniumDriver.getDriver().getCurrentUrl().replace( + PluginPropertyUtils.pluginProp("cdfUrl"), PluginPropertyUtils.pluginProp("replication.url"))); + } + @And("Click on the Next button") + public void clickOnTheNextButton() throws InterruptedException { + ReplicationActions.clickNextButton(); + } + @And("Select Oracle as Source") + public void selectSourceAsOracle() { + ReplicationActions.clickOnOraclePlugin(); + } + @Then("Validate Table is available and can be selected {string}") + public void selectTable(String table) { + ReplicationActions.selectTable(table); + } + @Then("Deploy the replication pipeline") + public void deployPipeline() { + ReplicationActions.deployPipeline(); + } + + @Then("Run the replication Pipeline") + public void runPipelineInRuntime() { + ReplicationActions.runThePipeline(); + } + + @Then("Open the logs") + public void openLogsOfSltPipeline() { + ReplicationActions.openAdvanceLogs(); + } + + @Then("Wait till pipeline is in running state and check if no errors occurred") + public void waitTillSltPipelineIsInRunningState() throws InterruptedException { + ReplicationActions.waitTillPipelineIsRunningAndCheckForErrors(); + } + + @Then("Close the pipeline logs and stop the pipeline") + public void closeLogsAndStopThePipeline() { + ReplicationActions.closeTheLogsAndClickOnStopButton(); + } + + @And("Capture raw logs") + public void captureRawLogs() { + ReplicationActions.captureRawLog(); + } + + @And("Insert a record in the source table and wait for replication") + public void triggerInsertCdcEvent() throws IOException, InterruptedException, SQLException, ClassNotFoundException { + ReplicationActions.insertRecordAndWait(); //JCoException, + } + + @And("Delete a record in the source table and wait for replication") + public void triggerDeleteCdcEvent() throws IOException, InterruptedException, SQLException, ClassNotFoundException { + ReplicationActions.deleteRecordAndWait(); //JCoException, + } + + @And("Update a record in the source table and wait for replication") + public void triggerUpdateCdcEvent() throws IOException, InterruptedException, SQLException, ClassNotFoundException { + ReplicationActions.updateRecordAndWait(); //JCoException, + } + @Then("Verify expected Oracle records in target BigQuery table") + public void verifyExpectedOracleRecordsInTargetBigQueryTable() throws IOException, InterruptedException, SQLException, ClassNotFoundException { + ReplicationActions.verifyTargetBigQueryRecordMatchesExpectedOracleRecord(); + } + + +} diff --git a/src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java b/src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java new file mode 100644 index 0000000..2959b32 --- /dev/null +++ b/src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.tests; + +import io.cucumber.junit.Cucumber; +import io.cucumber.junit.CucumberOptions; +import org.junit.runner.RunWith; + +/** + * Test Runner to execute Oracle plugin test cases. + */ +@RunWith(Cucumber.class) +@CucumberOptions( + features = {"src/e2e-test/features"}, + glue = {"stepsdesign", "io.cdap.plugin.common.stepsdesign", "io.cdap.plugin.oracle.stepsdesign"}, + tags = {"@Oracle"}, + plugin = {"pretty", "html:target/cucumber-html-report/oracle", + "json:target/cucumber-reports/cucumber-oracle.json", + "junit:target/cucumber-reports/cucumber-oracle.xml"} +) +public class TestRunner { +} diff --git a/src/e2e-test/java/io.cdap.plugin/tests/package-info.java b/src/e2e-test/java/io.cdap.plugin/tests/package-info.java new file mode 100644 index 0000000..05da828 --- /dev/null +++ b/src/e2e-test/java/io.cdap.plugin/tests/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * Package contains the runners for Oracle features. + */ +package io.cdap.plugin.tests; diff --git a/src/e2e-test/java/io.cdap.plugin/utils/OracleClient.java b/src/e2e-test/java/io.cdap.plugin/utils/OracleClient.java new file mode 100644 index 0000000..b803f4d --- /dev/null +++ b/src/e2e-test/java/io.cdap.plugin/utils/OracleClient.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.utils; + + +import io.cdap.e2e.utils.PluginPropertyUtils; + +import java.sql.*; +import java.time.Instant; +import java.util.*; + +import java.util.concurrent.TimeUnit; + +/** + * Oracle client. + */ +public class OracleClient { + private static Connection getOracleConnection() throws SQLException, ClassNotFoundException { + TimeZone timezone = TimeZone.getTimeZone("UTC"); + TimeZone.setDefault(timezone); + Class.forName("oracle.jdbc.driver.OracleDriver"); + String databaseName = PluginPropertyUtils.pluginProp("database"); + String host = PluginPropertyUtils.pluginProp("host"); + String port = PluginPropertyUtils.pluginProp("port"); + String username = PluginPropertyUtils.pluginProp("username"); + String password = PluginPropertyUtils.pluginProp("password"); + + return DriverManager.getConnection("jdbc:oracle:thin:@//" +host + + ":" + port + "/" + databaseName, + username, password); + } + + + public static void createTable(String table, String schema, String datatypeColumns, String primaryKey) throws SQLException, ClassNotFoundException { + try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { + String createTableQuery = "CREATE TABLE " + schema + "." + table + datatypeColumns; + statement.executeUpdate(createTableQuery); + } + } + + public static void forceFlushCDC() throws SQLException, ClassNotFoundException{ + try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { + //Oracle doesn't immediately flush CDC events, it can automatically happen based on time/log file size or you can force it + statement.executeUpdate("ALTER SYSTEM SWITCH LOGFILE"); + } + } + + + + public static void insertRow(String table, String schema, String datatypeColumns, String datatypeValues) throws SQLException, ClassNotFoundException { + try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { + // Insert dummy data. + statement.executeUpdate("INSERT INTO " + schema + "." + table + " " + datatypeColumns + + " VALUES " + datatypeValues); + + } + } + public static void deleteRow(String table, String schema, String deleteCondition) throws SQLException, ClassNotFoundException { + try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { + // Insert dummy data. + statement.executeUpdate("DELETE FROM " + schema + "." + table + " WHERE " + deleteCondition); + } + } + public static void updateRow(String table, String schema, String updateCondition, String updatedValue) throws SQLException, ClassNotFoundException { + try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { + // Insert dummy data. + statement.executeUpdate("UPDATE " + schema + "." + table + " SET " + updatedValue + + " WHERE " + updateCondition); + } + } + + public static List> getOracleRecordsAsMap(String table, String schema)throws SQLException, ClassNotFoundException { + try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { + // Insert dummy data. + List> oracleRecords = new ArrayList<>(); + String query = "select * from " + schema + "." + table; + ResultSet result = statement.executeQuery(query); + + ResultSetMetaData rsmd = result.getMetaData(); + int numberOfColumns = rsmd.getColumnCount(); + List columns = new ArrayList<>(); + columns.add(""); + for(int colIndex = 1 ; colIndex <= numberOfColumns ; colIndex++){ + columns.add(rsmd.getColumnName(colIndex) + "#" + rsmd.getColumnType(colIndex)); + } + while (result.next()) { + Map record = new HashMap<>(); + for(int colIndex = 1 ; colIndex <= numberOfColumns ; colIndex++){ + String columnName = columns.get(colIndex).split("#")[0]; + int type = Integer.parseInt(columns.get(colIndex).split("#")[1]); + Object value; + switch (type){ + case Types.TIMESTAMP: + Instant instant = result.getTimestamp(colIndex).toInstant(); + //Rounding off as BQ supports till microseconds + value = TimeUnit.SECONDS.toMicros(instant.getEpochSecond()) + TimeUnit.NANOSECONDS.toMicros(instant.getNano()); + break; + default: + //else convert all data types toString as bq converts certain data types to string to preserve precision and scale + value = result.getString(colIndex); + } + record.put(columnName,value); + } + oracleRecords.add(record); + } + statement.close(); + return oracleRecords; + } + } + + public static void deleteTables(String table, String schema) + throws SQLException, ClassNotFoundException { + try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { + String dropTableQuery = "DROP TABLE " + schema + "." + table; + statement.execute(dropTableQuery); + } + } +} \ No newline at end of file diff --git a/src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java b/src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java new file mode 100644 index 0000000..00344d9 --- /dev/null +++ b/src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.utils; + +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.FieldValue; +import com.google.cloud.bigquery.FieldValueList; +import com.google.cloud.bigquery.TableResult; +import io.cdap.e2e.utils.BigQueryClient; +import io.cdap.e2e.utils.ConstantsUtil; +import io.cdap.e2e.utils.PluginPropertyUtils; +import net.jodah.failsafe.Failsafe; +import net.jodah.failsafe.RetryPolicy; +import org.junit.Assert; + +import java.io.IOException; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public class ValidationHelper { + + public static List> getBigQueryRecordsAsMap(String projectId, String database, String tableName) + throws IOException, InterruptedException { + String query = "SELECT * EXCEPT ( _row_id, _source_timestamp, _sort) FROM `" + projectId + + "." + database + "." + tableName + "`"; + List> bqRecords = new ArrayList<>(); + //Let us account for 90 seconds flush here.And for every flush, how long does it take to reflect the changes in bg by running queries. + //handle exceptions and retry + RetryPolicy retryPolicy = new RetryPolicy<>() + .withMaxAttempts(5) + .onRetry( e -> System.out.println("Retrying exception")) + .withDelay(Duration.ofSeconds(60)); + //.withMaxDuration(Duration.of(5, ChronoUnit.MINUTES)); + TableResult results = Failsafe.with(retryPolicy).get(() -> { + //Let changes reflect in bq making sure flush cycle is included +// TimeUnit time = TimeUnit.SECONDS; +// time.sleep(180); + return BigQueryClient.getQueryResult(query); + }); + List columns = new ArrayList<>(); + for (Field field : results.getSchema().getFields()) { + columns.add(field.getName() + "#" + field.getType()); + } + for (FieldValueList row : results.getValues()) { + Map record = new HashMap<>(); + int index = 0; + for (FieldValue fieldValue : row) { + String columnName = columns.get(index).split("#")[0]; + String dataType = columns.get(index).split("#")[1]; + Object value; + if(dataType.equalsIgnoreCase("TIMESTAMP")){ + value = fieldValue.getTimestampValue(); +// } else if (dataType.toUpperCase().equals("FLOAT")) { +// value = Float.parseFloat(fieldValue.getStringValue()); + } else { + value = fieldValue.getValue(); + value = value != null ? value.toString() : null; + } + record.put(columnName, value); + index++; + } + bqRecords.add(record); + } + return bqRecords; + } + + public static void validateRecords(List> sourceOracleRecords, + List> targetBigQueryRecords) { + + String uniqueField = PluginPropertyUtils.pluginProp("primaryKey"); + //Assert.assertEquals("Checking the size of lists ", targetBigQueryRecords.size(), sourceOracleRecords.size()); //doesn't make sense in our case, since few datatypes are dropped during assessment + + // Logic to maintain the order of both lists and validate records based on that order. + Map BqUniqueIdMap = (Map)targetBigQueryRecords.stream() + .filter(t -> t.get("_is_deleted")==null) + .collect(Collectors.toMap( + t -> t.get(uniqueField), + t -> t, + (x,y) -> { + Long xSeqNum = Long.parseLong(x.get("_sequence_num").toString()); + Long ySeqNum = Long.parseLong(y.get("_sequence_num").toString()); + if(xSeqNum > ySeqNum){ + return x; + } + return y; + })); + + + for (int record = 0; record < sourceOracleRecords.size(); record++) { + Map oracleRecord = sourceOracleRecords.get(record); + Object uniqueId = oracleRecord.get(uniqueField); + //EXP : In case data type not matched, convert to string and compare +// if( !uniqueId.getClass().equals(targetBigQueryRecords.get(0).get(uniqueField).getClass()) ){ +// uniqueId = uniqueId.toString(); +// } + Map bqRow = (Map) BqUniqueIdMap.get(uniqueId); + bqRow.remove("_is_deleted"); + bqRow.remove("_sequence_num"); + + ValidationHelper.compareBothResponses( bqRow, oracleRecord); + } + } + + public static void compareBothResponses(Map targetBigQueryRecords, + Map sourceOracleRecord) { + Assert.assertNotNull("Checking if target BigQuery Record is null", targetBigQueryRecords); + Set bigQueryKeySet = targetBigQueryRecords.keySet(); + for (String field : bigQueryKeySet) { + Object bigQueryFieldValue = targetBigQueryRecords.get(field); + Object sapFieldValue = sourceOracleRecord.get(field); +// if(!bigQueryFieldValue.getClass().equals(sapFieldValue.getClass())){ +// sapFieldValue = sapFieldValue.toString(); +// bigQueryFieldValue = bigQueryFieldValue.toString(); +// } + Assert.assertEquals(String.format("Field %s is not equal: expected %s but got %s", field, + sapFieldValue, bigQueryFieldValue), + sapFieldValue, bigQueryFieldValue); + } + } + + + + + public static void waitForFlush() throws InterruptedException { + int flushInterval = Integer.parseInt(PluginPropertyUtils.pluginProp("loadInterval")); + TimeUnit time = TimeUnit.SECONDS; + time.sleep(2*flushInterval); + } + + + +} diff --git a/src/e2e-test/resources/errorMessage.properties b/src/e2e-test/resources/errorMessage.properties new file mode 100644 index 0000000..283352d --- /dev/null +++ b/src/e2e-test/resources/errorMessage.properties @@ -0,0 +1 @@ +validationSuccessMessage=No errors found \ No newline at end of file diff --git a/src/e2e-test/resources/pluginDataCyAttributes.properties b/src/e2e-test/resources/pluginDataCyAttributes.properties new file mode 100644 index 0000000..2608ac4 --- /dev/null +++ b/src/e2e-test/resources/pluginDataCyAttributes.properties @@ -0,0 +1,6 @@ +host=host +user=user +password=password +region=select-region +regionOption=option-us-west1 + diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties new file mode 100644 index 0000000..7fa82fc --- /dev/null +++ b/src/e2e-test/resources/pluginParameters.properties @@ -0,0 +1,42 @@ + +pipelineName=e2e-ds3 +projectId= + +database= +schema= +sourceTable=E2E1 +host= +port= +username= +password= + +datastream.timeout=300 +loadInterval=30 + +replication.url=http://localhost:11011/cdap/ns/default/replication/create +cdfUrl=http://localhost:11011/pipelines/ns/default/studio + +datatypeColumns= (ID number(38), LASTNAME varchar2(100), PRIMARY KEY (ID)) +datatypeColumnNames=(ID, LASTNAME) +primaryKey=ID + +datatypeValuesRow1=(1, 'Sheldon') +datatypeValuesRow2=(2, 'Shelby') + +datatypeValuesForInsertOperation=(3, 'Simpson') + +deleteRowCondition=ID=3 + +updateRowCondition=ID=1 +updatedRow=LASTNAME='Leonard' + +datatypeColumnsForSanityTesting=('USER1', 'M','ABCDEF','ABC','ABC','ä','ä½ å¥½ï¼?è¿?','ä½ å¥½ï¼?è¿?','AAAAaoAATAAABrXAAA',1234,1234.56789,\ + 1234.56789,1234.56789,1234.56789,1234.56789,1234.56789,1234.56789,1234.5679,\ + 1234.56789,1234.5679,1234.5679,1234.56789,TIMESTAMP'2023-01-01 2:00:00',TIMESTAMP'2023-01-01 2:00:00',\ + TIMESTAMP '2023-01-01 00:00:00.000000') + +datatypeValuesForSanityTesting=('USER1', 'M','ABCDEF','ABC','ABC','ä','ä½ å¥½ï¼?è¿?','ä½ å¥½ï¼?è¿?',\ + 'AAAAaoAATAAABrXAAA',1234,1234.56789,\ + 1234.56789,1234.56789,1234.56789,1234.56789,1234.56789,1234.56789,1234.5679,\ + 1234.56789,1234.5679,1234.5679,1234.56789,TIMESTAMP'2023-01-01 2:00:00',TIMESTAMP'2023-01-01 2:00:00',\ + TIMESTAMP '2023-01-01 00:00:00.000000') \ No newline at end of file From c17324612077d3b67b2ff0a1ed8b239bc48d7396 Mon Sep 17 00:00:00 2001 From: yjhawar Date: Thu, 6 Apr 2023 19:05:25 +0530 Subject: [PATCH 02/14] Refactored the code. --- src/e2e-test/features/Pipeline.feature | 38 ++---------- .../actions/ReplicationActions.java | 31 +++++----- .../io.cdap.plugin/hooks/TestSetUpHooks.java | 47 ++++++++++---- .../stepsdesign/StepDefinition.java | 8 +-- .../io.cdap.plugin/utils/OracleClient.java | 8 +-- .../utils/ValidationHelper.java | 62 +++++-------------- .../pluginDataCyAttributes.properties | 2 +- .../resources/pluginParameters.properties | 58 +++++++++-------- 8 files changed, 113 insertions(+), 141 deletions(-) diff --git a/src/e2e-test/features/Pipeline.feature b/src/e2e-test/features/Pipeline.feature index 51b8224..549dc38 100644 --- a/src/e2e-test/features/Pipeline.feature +++ b/src/e2e-test/features/Pipeline.feature @@ -14,37 +14,9 @@ # the License. # -Feature: Oracle - Verify Oracle source data transfer - -# Scenario: Sanity test from Oracle to Big Query -# Given Open DataFusion Project with replication to configure pipeline -# When Enter input plugin property: "name" with value: "pipelineName" -# And Click on the Next button -# And Select Oracle as Source -# Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields -# Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields -# Then Click plugin property: "region" -# Then Click plugin property: "regionOption" -# Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields -# Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields -# Then Replace input plugin property: "sid" with value: "database" for Credentials and Authorization related fields -# Then Click on the Next button -# Then Replace input plugin property: "loadInterval" with value: "loadInterval" -# Then Click on the Next button -# Then Validate Table is available and can be selected "ABC.E2E-sanity" -# And Click on the Next button -# And Click on the Next button -# And Click on the Next button -# Then Deploy the replication pipeline -# And Run the replication Pipeline -# Then Open the logs -# And Wait till pipeline is in running state and check if no errors occurred -# Then Verify expected Oracle records in target BigQuery table -# And Capture raw logs -# Then Close the pipeline logs and stop the pipeline - -# @ORACLE_SOURCE - Scenario: To verify snapshot and cdc from Oracle to Big Query successfully +Feature: Oracle - Verify Oracle source data transfer to Big Query + @ORACLE_SOURCE @BIGQUERY_TARGET + Scenario: To verify replication of snapshot and cdc data from Oracle to Big Query successfully with Sanity test Given Open DataFusion Project with replication to configure pipeline When Enter input plugin property: "name" with value: "pipelineName" And Click on the Next button @@ -55,11 +27,11 @@ Feature: Oracle - Verify Oracle source data transfer Then Click plugin property: "regionOption" Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields - Then Replace input plugin property: "sid" with value: "database" for Credentials and Authorization related fields + Then Replace input plugin property: "sid" with value: "dataset" for Credentials and Authorization related fields Then Click on the Next button Then Replace input plugin property: "loadInterval" with value: "loadInterval" Then Click on the Next button - Then Validate Table is available and can be selected "ABC.E2E1" + Then Validate Source table is available and select it And Click on the Next button And Click on the Next button And Click on the Next button diff --git a/src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java b/src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java index b92acb1..a5ae274 100644 --- a/src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java +++ b/src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java @@ -36,10 +36,9 @@ public class ReplicationActions { private static String parentWindow = StringUtils.EMPTY; private static final String projectId = PluginPropertyUtils.pluginProp("projectId"); - private static final String database = PluginPropertyUtils.pluginProp("database"); + private static final String database = PluginPropertyUtils.pluginProp("dataset"); public static String tableName = PluginPropertyUtils.pluginProp("sourceTable"); public static String schemaName = PluginPropertyUtils.pluginProp("schema"); - public static String datatypeColumnNames = PluginPropertyUtils.pluginProp("datatypeColumnNames"); public static String datatypeValues = PluginPropertyUtils.pluginProp("datatypeValuesForInsertOperation"); public static String deleteCondition = PluginPropertyUtils.pluginProp("deleteRowCondition"); public static String updateCondition = PluginPropertyUtils.pluginProp("updateRowCondition"); @@ -53,14 +52,16 @@ public static void clickNextButton() throws InterruptedException { time.sleep(1); ElementHelper.clickOnElement(ReplicationLocators.next); } + public static void clickOnOraclePlugin() { ElementHelper.clickOnElement(ReplicationLocators.oraclePlugin); } - public static void selectTable(String tableName) { - WaitHelper.waitForElementToBeDisplayed(ReplicationLocators.selectTable(tableName)); - AssertionHelper.verifyElementDisplayed(ReplicationLocators.selectTable(tableName)); - ElementHelper.clickOnElement(ReplicationLocators.selectTable(tableName)); + public static void selectTable() { + String table = schemaName + "." + tableName; + WaitHelper.waitForElementToBeDisplayed(ReplicationLocators.selectTable(table)); + AssertionHelper.verifyElementDisplayed(ReplicationLocators.selectTable(table)); + ElementHelper.clickOnElement(ReplicationLocators.selectTable(table)); } public static void deployPipeline() { @@ -109,9 +110,10 @@ public static String getRawLogs() { public static void waitTillPipelineIsRunningAndCheckForErrors() throws InterruptedException { //wait for datastream to startup - int defaultTimeout = Integer.parseInt(PluginPropertyUtils.pluginProp("datastream.timeout")); + int defaultTimeout = Integer.parseInt(PluginPropertyUtils.pluginProp("pipeline-initialization")); TimeUnit time = TimeUnit.SECONDS; time.sleep(defaultTimeout); + ValidationHelper.waitForFlush(); // Checking if an error message is displayed. Assert.assertFalse(ElementHelper.isElementDisplayed(ReplicationLocators.error)); } @@ -121,6 +123,7 @@ public static void closeTheLogsAndClickOnStopButton() { SeleniumDriver.getDriver().switchTo().window(parentWindow); //Stopping the pipeline ElementHelper.clickOnElement(ReplicationLocators.stop); + SeleniumDriver.getDriver().navigate().refresh(); WaitHelper.waitForElementToBeDisplayed(ReplicationLocators.stopped); } public static void verifyTargetBigQueryRecordMatchesExpectedOracleRecord() @@ -129,25 +132,25 @@ public static void verifyTargetBigQueryRecordMatchesExpectedOracleRecord() Assert.assertFalse(ElementHelper.isElementDisplayed(ReplicationLocators.error)); List> sourceOracleRecords = OracleClient.getOracleRecordsAsMap(tableName, schemaName); - List> targetBigQueryRecords = ValidationHelper.getBigQueryRecordsAsMap(projectId, database, tableName); //+ "_v1`" + List> targetBigQueryRecords = ValidationHelper.getBigQueryRecordsAsMap(projectId, database, tableName); ValidationHelper.validateRecords(sourceOracleRecords, targetBigQueryRecords); } public static void insertRecordAndWait() - throws IOException, InterruptedException, SQLException, ClassNotFoundException { //JCoException, - OracleClient.insertRow(tableName,schemaName, datatypeColumnNames,datatypeValues); + throws InterruptedException, SQLException, ClassNotFoundException { + OracleClient.insertRow(tableName, schemaName, datatypeValues); OracleClient.forceFlushCDC(); ValidationHelper.waitForFlush(); } - public static void deleteRecordAndWait() throws SQLException, ClassNotFoundException, IOException, InterruptedException { - OracleClient.deleteRow(tableName,schemaName, deleteCondition); + public static void deleteRecordAndWait() throws SQLException, ClassNotFoundException, InterruptedException { + OracleClient.deleteRow(tableName, schemaName, deleteCondition); OracleClient.forceFlushCDC(); ValidationHelper.waitForFlush(); } - public static void updateRecordAndWait() throws SQLException, ClassNotFoundException, IOException, InterruptedException { - OracleClient.updateRow(tableName,schemaName, updateCondition, updatedValue ); + public static void updateRecordAndWait() throws SQLException, ClassNotFoundException, InterruptedException { + OracleClient.updateRow(tableName, schemaName, updateCondition, updatedValue ); OracleClient.forceFlushCDC(); ValidationHelper.waitForFlush(); } diff --git a/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java b/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java index dbffc05..5972bf8 100644 --- a/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java +++ b/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java @@ -15,12 +15,16 @@ */ package io.cdap.plugin.hooks; +import com.google.cloud.bigquery.BigQueryException; +import io.cdap.e2e.utils.BigQueryClient; import io.cdap.e2e.utils.PluginPropertyUtils; import io.cdap.plugin.utils.OracleClient; import io.cucumber.java.After; import io.cucumber.java.Before; +import org.junit.Assert; import stepsdesign.BeforeActions; +import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; @@ -36,13 +40,15 @@ public class TestSetUpHooks { public static String schemaName = PluginPropertyUtils.pluginProp("schema"); public static String primaryKey = PluginPropertyUtils.pluginProp("primaryKey"); public static String datatypeColumns = PluginPropertyUtils.pluginProp("datatypeColumns"); - public static String datatypeColumnNames = PluginPropertyUtils.pluginProp("datatypeColumnNames"); public static String row1 = PluginPropertyUtils.pluginProp("datatypeValuesRow1"); public static String row2= PluginPropertyUtils.pluginProp("datatypeValuesRow2"); - @Before(order = 1, value = "@ORACLE_SOURCE") public static void overridePropertiesFromEnvVarsIfProvided() { + String projectId = System.getenv("PROJECT_ID"); + if (projectId != null && !projectId.isEmpty()) { + PluginPropertyUtils.addPluginProp("projectId", projectId); + } String username = System.getenv("ORACLE_USERNAME"); if (username != null && !username.isEmpty()) { PluginPropertyUtils.addPluginProp("username", username); @@ -55,22 +61,26 @@ public static void overridePropertiesFromEnvVarsIfProvided() { if (port!= null && !port.isEmpty()) { PluginPropertyUtils.addPluginProp("port", port); } - String sapHost = System.getenv("ORACLE_HOST"); - if (sapHost != null && !sapHost.isEmpty()) { - PluginPropertyUtils.addPluginProp("host", sapHost); + String oracleHost = System.getenv("ORACLE_HOST"); + if (oracleHost != null && !oracleHost.isEmpty()) { + PluginPropertyUtils.addPluginProp("host", oracleHost); + } + String sourceTable = System.getenv("SOURCE_TABLE"); + if (sourceTable != null && !sourceTable.isEmpty()) { + PluginPropertyUtils.addPluginProp("sourceTable", sourceTable); + tableName = PluginPropertyUtils.pluginProp("sourceTable"); } } @Before(order = 2, value = "@ORACLE_SOURCE") public static void createTable() throws SQLException, ClassNotFoundException { - OracleClient.createTable(tableName,schemaName,datatypeColumns, primaryKey); + OracleClient.createTable(tableName, schemaName, datatypeColumns, primaryKey); } @Before(order = 3, value = "@ORACLE_SOURCE") public static void insertRow() throws SQLException, ClassNotFoundException { - OracleClient.insertRow(tableName, schemaName, datatypeColumnNames, row1); - OracleClient.insertRow(tableName, schemaName, datatypeColumnNames, row2); - sourceOracleRecords = OracleClient.getOracleRecordsAsMap(tableName, schemaName); + OracleClient.insertRow(tableName, schemaName, row1); + OracleClient.insertRow(tableName, schemaName, row2); } @Before(order = 4, value = "@ORACLE_SOURCE") @@ -79,8 +89,23 @@ public static void getOracleRecordsAsMap() throws SQLException, ClassNotFoundExc BeforeActions.scenario.write("Expected Oracle records : " + sourceOracleRecords); } - @After(order = 1, value = "@ORACLE_SOURCE_TEMP") + @After(order = 1, value = "@ORACLE_SOURCE") public static void dropTables() throws SQLException, ClassNotFoundException { - OracleClient.deleteTables(schemaName, tableName); + OracleClient.deleteTables(schemaName, tableName); } + + @After(order = 1, value = "@BIGQUERY_TARGET") + public static void deleteTempTargetBQTable() throws IOException, InterruptedException { + try { + BigQueryClient.dropBqQuery(tableName); + BeforeActions.scenario.write("BQ Target table - " + tableName + " deleted successfully"); + } catch (BigQueryException e) { + if (e.getMessage().contains("Not found: Table")) { + BeforeActions.scenario.write("BQ Target Table does not exist"); + } else { + Assert.fail(e.getMessage()); + } + } + } + } \ No newline at end of file diff --git a/src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java b/src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java index 5b73e6a..caa42af 100644 --- a/src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java +++ b/src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java @@ -42,9 +42,10 @@ public void clickOnTheNextButton() throws InterruptedException { public void selectSourceAsOracle() { ReplicationActions.clickOnOraclePlugin(); } - @Then("Validate Table is available and can be selected {string}") - public void selectTable(String table) { - ReplicationActions.selectTable(table); + + @Then("Validate Source table is available and select it") + public void selectTable1() { + ReplicationActions.selectTable(); } @Then("Deploy the replication pipeline") public void deployPipeline() { @@ -95,5 +96,4 @@ public void verifyExpectedOracleRecordsInTargetBigQueryTable() throws IOExceptio ReplicationActions.verifyTargetBigQueryRecordMatchesExpectedOracleRecord(); } - } diff --git a/src/e2e-test/java/io.cdap.plugin/utils/OracleClient.java b/src/e2e-test/java/io.cdap.plugin/utils/OracleClient.java index b803f4d..1b8a37a 100644 --- a/src/e2e-test/java/io.cdap.plugin/utils/OracleClient.java +++ b/src/e2e-test/java/io.cdap.plugin/utils/OracleClient.java @@ -33,7 +33,7 @@ private static Connection getOracleConnection() throws SQLException, ClassNotFou TimeZone timezone = TimeZone.getTimeZone("UTC"); TimeZone.setDefault(timezone); Class.forName("oracle.jdbc.driver.OracleDriver"); - String databaseName = PluginPropertyUtils.pluginProp("database"); + String databaseName = PluginPropertyUtils.pluginProp("dataset"); String host = PluginPropertyUtils.pluginProp("host"); String port = PluginPropertyUtils.pluginProp("port"); String username = PluginPropertyUtils.pluginProp("username"); @@ -61,10 +61,10 @@ public static void forceFlushCDC() throws SQLException, ClassNotFoundException{ - public static void insertRow(String table, String schema, String datatypeColumns, String datatypeValues) throws SQLException, ClassNotFoundException { + public static void insertRow(String table, String schema, String datatypeValues) throws SQLException, ClassNotFoundException { try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { // Insert dummy data. - statement.executeUpdate("INSERT INTO " + schema + "." + table + " " + datatypeColumns + + statement.executeUpdate("INSERT INTO " + schema + "." + table + " " + " VALUES " + datatypeValues); } @@ -122,7 +122,7 @@ public static List> getOracleRecordsAsMap(String table, Stri } } - public static void deleteTables(String table, String schema) + public static void deleteTables(String schema, String table) throws SQLException, ClassNotFoundException { try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { String dropTableQuery = "DROP TABLE " + schema + "." + table; diff --git a/src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java b/src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java index 00344d9..8c15eae 100644 --- a/src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java +++ b/src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java @@ -21,15 +21,9 @@ import com.google.cloud.bigquery.FieldValueList; import com.google.cloud.bigquery.TableResult; import io.cdap.e2e.utils.BigQueryClient; -import io.cdap.e2e.utils.ConstantsUtil; import io.cdap.e2e.utils.PluginPropertyUtils; -import net.jodah.failsafe.Failsafe; -import net.jodah.failsafe.RetryPolicy; import org.junit.Assert; - import java.io.IOException; -import java.time.Duration; -import java.time.temporal.ChronoUnit; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -41,19 +35,7 @@ public static List> getBigQueryRecordsAsMap(String projectId String query = "SELECT * EXCEPT ( _row_id, _source_timestamp, _sort) FROM `" + projectId + "." + database + "." + tableName + "`"; List> bqRecords = new ArrayList<>(); - //Let us account for 90 seconds flush here.And for every flush, how long does it take to reflect the changes in bg by running queries. - //handle exceptions and retry - RetryPolicy retryPolicy = new RetryPolicy<>() - .withMaxAttempts(5) - .onRetry( e -> System.out.println("Retrying exception")) - .withDelay(Duration.ofSeconds(60)); - //.withMaxDuration(Duration.of(5, ChronoUnit.MINUTES)); - TableResult results = Failsafe.with(retryPolicy).get(() -> { - //Let changes reflect in bq making sure flush cycle is included -// TimeUnit time = TimeUnit.SECONDS; -// time.sleep(180); - return BigQueryClient.getQueryResult(query); - }); + TableResult results = BigQueryClient.getQueryResult(query); List columns = new ArrayList<>(); for (Field field : results.getSchema().getFields()) { columns.add(field.getName() + "#" + field.getType()); @@ -67,8 +49,6 @@ public static List> getBigQueryRecordsAsMap(String projectId Object value; if(dataType.equalsIgnoreCase("TIMESTAMP")){ value = fieldValue.getTimestampValue(); -// } else if (dataType.toUpperCase().equals("FLOAT")) { -// value = Float.parseFloat(fieldValue.getStringValue()); } else { value = fieldValue.getValue(); value = value != null ? value.toString() : null; @@ -85,14 +65,13 @@ public static void validateRecords(List> sourceOracleRecords List> targetBigQueryRecords) { String uniqueField = PluginPropertyUtils.pluginProp("primaryKey"); - //Assert.assertEquals("Checking the size of lists ", targetBigQueryRecords.size(), sourceOracleRecords.size()); //doesn't make sense in our case, since few datatypes are dropped during assessment - // Logic to maintain the order of both lists and validate records based on that order. Map BqUniqueIdMap = (Map)targetBigQueryRecords.stream() .filter(t -> t.get("_is_deleted")==null) .collect(Collectors.toMap( t -> t.get(uniqueField), t -> t, + //This logic is to handle duplication scenario in bq target (x,y) -> { Long xSeqNum = Long.parseLong(x.get("_sequence_num").toString()); Long ySeqNum = Long.parseLong(y.get("_sequence_num").toString()); @@ -102,48 +81,35 @@ public static void validateRecords(List> sourceOracleRecords return y; })); - for (int record = 0; record < sourceOracleRecords.size(); record++) { Map oracleRecord = sourceOracleRecords.get(record); Object uniqueId = oracleRecord.get(uniqueField); - //EXP : In case data type not matched, convert to string and compare -// if( !uniqueId.getClass().equals(targetBigQueryRecords.get(0).get(uniqueField).getClass()) ){ -// uniqueId = uniqueId.toString(); -// } Map bqRow = (Map) BqUniqueIdMap.get(uniqueId); - bqRow.remove("_is_deleted"); - bqRow.remove("_sequence_num"); - - ValidationHelper.compareBothResponses( bqRow, oracleRecord); + if (bqRow != null) { + bqRow.remove("_is_deleted"); + bqRow.remove("_sequence_num"); + bqRow.forEach((key, value) -> System.out.println("Bq record" + key + ":" + value)); + } + compareRecords(bqRow, oracleRecord); } } - public static void compareBothResponses(Map targetBigQueryRecords, - Map sourceOracleRecord) { - Assert.assertNotNull("Checking if target BigQuery Record is null", targetBigQueryRecords); + public static void compareRecords(Map targetBigQueryRecords, + Map sourceOracleRecord) { Set bigQueryKeySet = targetBigQueryRecords.keySet(); for (String field : bigQueryKeySet) { Object bigQueryFieldValue = targetBigQueryRecords.get(field); - Object sapFieldValue = sourceOracleRecord.get(field); -// if(!bigQueryFieldValue.getClass().equals(sapFieldValue.getClass())){ -// sapFieldValue = sapFieldValue.toString(); -// bigQueryFieldValue = bigQueryFieldValue.toString(); -// } + Object oracleFieldValue = sourceOracleRecord.get(field); Assert.assertEquals(String.format("Field %s is not equal: expected %s but got %s", field, - sapFieldValue, bigQueryFieldValue), - sapFieldValue, bigQueryFieldValue); + oracleFieldValue, bigQueryFieldValue), + oracleFieldValue, bigQueryFieldValue); } } - - - public static void waitForFlush() throws InterruptedException { int flushInterval = Integer.parseInt(PluginPropertyUtils.pluginProp("loadInterval")); TimeUnit time = TimeUnit.SECONDS; - time.sleep(2*flushInterval); + time.sleep(2 * flushInterval); } - - } diff --git a/src/e2e-test/resources/pluginDataCyAttributes.properties b/src/e2e-test/resources/pluginDataCyAttributes.properties index 2608ac4..7ebda94 100644 --- a/src/e2e-test/resources/pluginDataCyAttributes.properties +++ b/src/e2e-test/resources/pluginDataCyAttributes.properties @@ -2,5 +2,5 @@ host=host user=user password=password region=select-region -regionOption=option-us-west1 +regionOption=option-us-west2 diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index 7fa82fc..7bf3452 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -1,42 +1,48 @@ -pipelineName=e2e-ds3 -projectId= - -database= -schema= -sourceTable=E2E1 -host= -port= -username= -password= - -datastream.timeout=300 + +projectId=PROJECT_ID +dataset=XE +schema=ABC +sourceTable=SOURCE_TABLE +host=ORACLE_HOST +port=ORACLE_PORT +username=ORACLE_USERNAME +password=ORACLE_PASSWORD + +pipelineName=e2e-sanity +pipeline-initialization=200 loadInterval=30 replication.url=http://localhost:11011/cdap/ns/default/replication/create cdfUrl=http://localhost:11011/pipelines/ns/default/studio -datatypeColumns= (ID number(38), LASTNAME varchar2(100), PRIMARY KEY (ID)) -datatypeColumnNames=(ID, LASTNAME) primaryKey=ID -datatypeValuesRow1=(1, 'Sheldon') -datatypeValuesRow2=(2, 'Shelby') - -datatypeValuesForInsertOperation=(3, 'Simpson') +deleteRowCondition=ID='USER1' +updateRowCondition=ID='USER2' +updatedRow=COL2='SUCCEEDED' -deleteRowCondition=ID=3 +datatypeColumns=(ID VARCHAR2(100) PRIMARY KEY, COL1 CHAR, COL2 CHAR(10), COL3 VARCHAR(3), COL4 VARCHAR2(3), \ +COL5 NCHAR, COL6 NCHAR(12), COL7 NVARCHAR2(12), COL11 ROWID, COL12 NUMBER(4), \ +COL13 NUMBER(*), COL15 NUMBER(10,-3), COL16 NUMBER, COL17 DECIMAL(4), COL18 DECIMAL(*),\ + COL20 DECIMAL(10,-3), COL21 DECIMAL, COL22 FLOAT, COL24 INTEGER, \ +COL25 DOUBLE PRECISION, COL26 REAL, COL27 SMALLINT, COL28 TIMESTAMP, COL29 TIMESTAMP(9), \ +COL33 DATE, COL35 BFILE) -updateRowCondition=ID=1 -updatedRow=LASTNAME='Leonard' +datatypeValuesRow1=('USER1', 'M','ABCDEF','ABC','ABC','ä','ä½ å¥½ï¼?è¿?','ä½ å¥½ï¼?è¿?',\ + 'AAAAaoAATAAABrXAAA',1234,1234.56789,\ + 1234.56789,1234.56789,1234.56789,1234.56789,1234.56789,1234.56789,1234.5679,\ + 1234.56789,1234.5679,1234.5679,1234.56789,TIMESTAMP'2023-01-01 2:00:00',TIMESTAMP'2023-01-01 2:00:00',\ + TIMESTAMP '2023-01-01 00:00:00.000000', NULL) -datatypeColumnsForSanityTesting=('USER1', 'M','ABCDEF','ABC','ABC','ä','ä½ å¥½ï¼?è¿?','ä½ å¥½ï¼?è¿?','AAAAaoAATAAABrXAAA',1234,1234.56789,\ +datatypeValuesRow2=('USER2', 'M','ABCDEF','ABC','ABC','ä','ä½ å¥½ï¼?è¿?','ä½ å¥½ï¼?è¿?',\ + 'AAAAaoAATAAABrXAAA',1234,1234.56789,\ 1234.56789,1234.56789,1234.56789,1234.56789,1234.56789,1234.56789,1234.5679,\ - 1234.56789,1234.5679,1234.5679,1234.56789,TIMESTAMP'2023-01-01 2:00:00',TIMESTAMP'2023-01-01 2:00:00',\ - TIMESTAMP '2023-01-01 00:00:00.000000') + 1234.56789,1234.5679,1234.5679,1234.56789,TIMESTAMP'2023-01-01 2:00:00',TIMESTAMP'2023-01-01 2:00:00',\ + TIMESTAMP '2023-01-01 00:00:00.000000', NULL) -datatypeValuesForSanityTesting=('USER1', 'M','ABCDEF','ABC','ABC','ä','ä½ å¥½ï¼?è¿?','ä½ å¥½ï¼?è¿?',\ +datatypeValuesForInsertOperation=('USER3', 'M','ABCDEF','ABC','ABC','ä','ä½ å¥½ï¼?è¿?','ä½ å¥½ï¼?è¿?',\ 'AAAAaoAATAAABrXAAA',1234,1234.56789,\ 1234.56789,1234.56789,1234.56789,1234.56789,1234.56789,1234.56789,1234.5679,\ 1234.56789,1234.5679,1234.5679,1234.56789,TIMESTAMP'2023-01-01 2:00:00',TIMESTAMP'2023-01-01 2:00:00',\ - TIMESTAMP '2023-01-01 00:00:00.000000') \ No newline at end of file + TIMESTAMP '2023-01-01 00:00:00.000000', NULL) \ No newline at end of file From 644226607ec28bdaf3ad186b42b77e93c514d49b Mon Sep 17 00:00:00 2001 From: Sumit Jain Date: Mon, 10 Apr 2023 14:06:47 +0530 Subject: [PATCH 03/14] Add build helper plugin to configure test sources for e2e profile --- pom.xml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pom.xml b/pom.xml index 97fdc74..ec21d44 100644 --- a/pom.xml +++ b/pom.xml @@ -472,6 +472,25 @@ + + org.codehaus.mojo + build-helper-maven-plugin + 3.3.0 + + + add-test-source + generate-test-sources + + add-test-source + + + + ${testSourceLocation} + + + + + From 700cf890c242d174793d48c39a24a7da6b4702ee Mon Sep 17 00:00:00 2001 From: Sumit Jain Date: Mon, 10 Apr 2023 14:54:31 +0530 Subject: [PATCH 04/14] Add build helper plugin to configure test sources for e2e profile --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ec21d44..5f0a95d 100644 --- a/pom.xml +++ b/pom.xml @@ -485,7 +485,7 @@ - ${testSourceLocation} + ${project.basedir}/${testSourceLocation} From 28bcdf638ac32f915e2051f6d7dfc5babe5c2c5c Mon Sep 17 00:00:00 2001 From: yjhawar Date: Tue, 11 Apr 2023 14:48:28 +0530 Subject: [PATCH 05/14] resolved comments --- src/e2e-test/features/Pipeline.feature | 2 +- .../actions/ReplicationActions.java | 11 +++-- .../io.cdap.plugin/hooks/TestSetUpHooks.java | 7 +-- .../stepsdesign/StepDefinition.java | 2 +- .../java/io.cdap.plugin/utils/BigQuery.java | 47 +++++++++++++++++++ .../utils/ValidationHelper.java | 11 +---- .../resources/pluginParameters.properties | 6 +-- 7 files changed, 64 insertions(+), 22 deletions(-) create mode 100644 src/e2e-test/java/io.cdap.plugin/utils/BigQuery.java diff --git a/src/e2e-test/features/Pipeline.feature b/src/e2e-test/features/Pipeline.feature index 549dc38..0b6a7e5 100644 --- a/src/e2e-test/features/Pipeline.feature +++ b/src/e2e-test/features/Pipeline.feature @@ -15,7 +15,7 @@ # Feature: Oracle - Verify Oracle source data transfer to Big Query - @ORACLE_SOURCE @BIGQUERY_TARGET + @ENV_VARIABLES #@ORACLE_SOURCE @ORACLE_DELETE @BIGQUERY_DELETE Scenario: To verify replication of snapshot and cdc data from Oracle to Big Query successfully with Sanity test Given Open DataFusion Project with replication to configure pipeline When Enter input plugin property: "name" with value: "pipelineName" diff --git a/src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java b/src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java index a5ae274..ad659f4 100644 --- a/src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java +++ b/src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java @@ -20,6 +20,7 @@ import io.cdap.e2e.pages.locators.CdfPipelineRunLocators; import io.cdap.e2e.utils.*; import io.cdap.plugin.locators.ReplicationLocators; +import io.cdap.plugin.utils.BigQuery; import io.cdap.plugin.utils.OracleClient; import io.cdap.plugin.utils.ValidationHelper; import org.apache.commons.lang.StringUtils; @@ -59,7 +60,7 @@ public static void clickOnOraclePlugin() { public static void selectTable() { String table = schemaName + "." + tableName; - WaitHelper.waitForElementToBeDisplayed(ReplicationLocators.selectTable(table)); + WaitHelper.waitForElementToBeDisplayed(ReplicationLocators.selectTable(table),300); AssertionHelper.verifyElementDisplayed(ReplicationLocators.selectTable(table)); ElementHelper.clickOnElement(ReplicationLocators.selectTable(table)); } @@ -113,7 +114,7 @@ public static void waitTillPipelineIsRunningAndCheckForErrors() throws Interrupt int defaultTimeout = Integer.parseInt(PluginPropertyUtils.pluginProp("pipeline-initialization")); TimeUnit time = TimeUnit.SECONDS; time.sleep(defaultTimeout); - ValidationHelper.waitForFlush(); + BigQuery.waitForFlush(); // Checking if an error message is displayed. Assert.assertFalse(ElementHelper.isElementDisplayed(ReplicationLocators.error)); } @@ -140,18 +141,18 @@ public static void insertRecordAndWait() throws InterruptedException, SQLException, ClassNotFoundException { OracleClient.insertRow(tableName, schemaName, datatypeValues); OracleClient.forceFlushCDC(); - ValidationHelper.waitForFlush(); + BigQuery.waitForFlush(); } public static void deleteRecordAndWait() throws SQLException, ClassNotFoundException, InterruptedException { OracleClient.deleteRow(tableName, schemaName, deleteCondition); OracleClient.forceFlushCDC(); - ValidationHelper.waitForFlush(); + BigQuery.waitForFlush(); } public static void updateRecordAndWait() throws SQLException, ClassNotFoundException, InterruptedException { OracleClient.updateRow(tableName, schemaName, updateCondition, updatedValue ); OracleClient.forceFlushCDC(); - ValidationHelper.waitForFlush(); + BigQuery.waitForFlush(); } } \ No newline at end of file diff --git a/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java b/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java index 5972bf8..ac59b3b 100644 --- a/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java +++ b/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java @@ -18,6 +18,7 @@ import com.google.cloud.bigquery.BigQueryException; import io.cdap.e2e.utils.BigQueryClient; import io.cdap.e2e.utils.PluginPropertyUtils; +import io.cdap.plugin.utils.BigQuery; import io.cdap.plugin.utils.OracleClient; import io.cucumber.java.After; import io.cucumber.java.Before; @@ -43,7 +44,7 @@ public class TestSetUpHooks { public static String row1 = PluginPropertyUtils.pluginProp("datatypeValuesRow1"); public static String row2= PluginPropertyUtils.pluginProp("datatypeValuesRow2"); - @Before(order = 1, value = "@ORACLE_SOURCE") + @Before(order = 1, value = "@ENV_VARIABLES") public static void overridePropertiesFromEnvVarsIfProvided() { String projectId = System.getenv("PROJECT_ID"); if (projectId != null && !projectId.isEmpty()) { @@ -89,12 +90,12 @@ public static void getOracleRecordsAsMap() throws SQLException, ClassNotFoundExc BeforeActions.scenario.write("Expected Oracle records : " + sourceOracleRecords); } - @After(order = 1, value = "@ORACLE_SOURCE") + @After(order = 1, value = "@ORACLE_DELETE") public static void dropTables() throws SQLException, ClassNotFoundException { OracleClient.deleteTables(schemaName, tableName); } - @After(order = 1, value = "@BIGQUERY_TARGET") + @After(order = 1, value = "@BIGQUERY_DELETE") public static void deleteTempTargetBQTable() throws IOException, InterruptedException { try { BigQueryClient.dropBqQuery(tableName); diff --git a/src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java b/src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java index caa42af..b3af423 100644 --- a/src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java +++ b/src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java @@ -44,7 +44,7 @@ public void selectSourceAsOracle() { } @Then("Validate Source table is available and select it") - public void selectTable1() { + public void selectTable() { ReplicationActions.selectTable(); } @Then("Deploy the replication pipeline") diff --git a/src/e2e-test/java/io.cdap.plugin/utils/BigQuery.java b/src/e2e-test/java/io.cdap.plugin/utils/BigQuery.java new file mode 100644 index 0000000..3cd3df7 --- /dev/null +++ b/src/e2e-test/java/io.cdap.plugin/utils/BigQuery.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.utils; + +import com.google.cloud.bigquery.BigQueryException; +import io.cdap.e2e.utils.BigQueryClient; +import io.cdap.e2e.utils.PluginPropertyUtils; +import org.junit.Assert; +import stepsdesign.BeforeActions; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +public class BigQuery { + public static void waitForFlush() throws InterruptedException { + int flushInterval = Integer.parseInt(PluginPropertyUtils.pluginProp("loadInterval")); + TimeUnit time = TimeUnit.SECONDS; + time.sleep(2 * flushInterval + 60); + } + + public static void deleteTable(String tableName) throws IOException, InterruptedException{ + try { + BigQueryClient.dropBqQuery(tableName); + BeforeActions.scenario.write("BQ Target table - " + tableName + " deleted successfully"); + } catch (BigQueryException e) { + if (e.getMessage().contains("Not found: Table")) { + BeforeActions.scenario.write("BQ Target Table does not exist"); + } else { + Assert.fail(e.getMessage()); + } + } + } +} \ No newline at end of file diff --git a/src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java b/src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java index 8c15eae..b351b36 100644 --- a/src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java +++ b/src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java @@ -66,7 +66,7 @@ public static void validateRecords(List> sourceOracleRecords String uniqueField = PluginPropertyUtils.pluginProp("primaryKey"); // Logic to maintain the order of both lists and validate records based on that order. - Map BqUniqueIdMap = (Map)targetBigQueryRecords.stream() + Map bqUniqueIdMap = (Map)targetBigQueryRecords.stream() .filter(t -> t.get("_is_deleted")==null) .collect(Collectors.toMap( t -> t.get(uniqueField), @@ -84,11 +84,10 @@ public static void validateRecords(List> sourceOracleRecords for (int record = 0; record < sourceOracleRecords.size(); record++) { Map oracleRecord = sourceOracleRecords.get(record); Object uniqueId = oracleRecord.get(uniqueField); - Map bqRow = (Map) BqUniqueIdMap.get(uniqueId); + Map bqRow = (Map) bqUniqueIdMap.get(uniqueId); if (bqRow != null) { bqRow.remove("_is_deleted"); bqRow.remove("_sequence_num"); - bqRow.forEach((key, value) -> System.out.println("Bq record" + key + ":" + value)); } compareRecords(bqRow, oracleRecord); } @@ -106,10 +105,4 @@ public static void compareRecords(Map targetBigQueryRecords, } } - public static void waitForFlush() throws InterruptedException { - int flushInterval = Integer.parseInt(PluginPropertyUtils.pluginProp("loadInterval")); - TimeUnit time = TimeUnit.SECONDS; - time.sleep(2 * flushInterval); - } - } diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index 7bf3452..57b6385 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -1,15 +1,15 @@ projectId=PROJECT_ID -dataset=XE -schema=ABC +dataset=ORCL +schema=HR sourceTable=SOURCE_TABLE host=ORACLE_HOST port=ORACLE_PORT username=ORACLE_USERNAME password=ORACLE_PASSWORD -pipelineName=e2e-sanity +pipelineName=e2e-sanity-existing-table pipeline-initialization=200 loadInterval=30 From db19820b7dfeeeb459562e18cf89d0cea9f494c1 Mon Sep 17 00:00:00 2001 From: yjhawar Date: Tue, 11 Apr 2023 15:19:42 +0530 Subject: [PATCH 06/14] Add github workflow --- .github/workflows/e2e.yml | 108 ++++++++++++++++++ .../java/io.cdap.plugin/tests/TestRunner.java | 4 +- 2 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..e5d84e4 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,108 @@ +name: Build e2e tests + +on: + push: + branches: [ e2e ] + pull_request: + branches: [ e2e ] + types: [ opened, synchronize, reopened, labeled ] + workflow_dispatch: + +jobs: + build: + runs-on: k8s-runner-e2e + # We allow builds: + # 1) When triggered manually + # 2) When it's a merge into a branch + # 3) For PRs that are labeled as build and + # - It's a code change + # - A build label was just added + # A bit complex, but prevents builds when other labels are manipulated + if: > + github.event_name == 'workflow_dispatch' + || github.event_name == 'push' + || (contains(github.event.pull_request.labels.*.name, 'build') + && (github.event.action != 'labeled' || github.event.label.name == 'build') + ) + strategy: + fail-fast: false + + steps: + # Pinned 1.0.0 version + - uses: actions/checkout@v3 + with: + path: plugin + submodules: 'recursive' + ref: ${{ github.event.workflow_run.head_sha }} + + - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 + if: github.event_name != 'workflow_dispatch' && github.event_name != 'push' + id: filter + with: + working-directory: plugin + filters: | + e2e-test: + - '**/e2e-test/**' + - name: Checkout e2e test repo + uses: actions/checkout@v3 + with: + repository: cdapio/cdap-e2e-tests + path: e2e + + - name: Cache + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven-${{ github.workflow }} + - name: Get Secrets from GCP Secret Manager + id: secrets + uses: 'google-github-actions/get-secretmanager-secrets@v0' + with: + secrets: |- + ORACLE_HOST:cdapio-github-builds/ORACLE_HOST + ORACLE_USERNAME:cdapio-github-builds/ORACLE_USERNAME + ORACLE_PASSWORD:cdapio-github-builds/ORACLE_PASSWORD + ORACLE_PORT:cdapio-github-builds/ORACLE_PORT + - name: Run required e2e tests + if: github.event_name != 'workflow_dispatch' && github.event_name != 'push' && steps.filter.outputs.e2e-test == 'false' + run: python3 e2e/src/main/scripts/run_e2e_test.py --testRunner TestRunnerRequired.java + env: + ORACLE_HOST: ${{ steps.secrets.outputs.ORACLE_HOST }} + ORACLE_USERNAME: ${{ steps.secrets.outputs.ORACLE_USERNAME }} + ORACLE_PASSWORD: ${{ steps.secrets.outputs.ORACLE_PASSWORD }} + ORACLE_PORT: ${{ steps.secrets.outputs.ORACLE_PORT }} + + + - name: Run all e2e tests + if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || steps.filter.outputs.e2e-test == 'true' + run: python3 e2e/src/main/scripts/run_e2e_test.py --testRunner **/**/TestRunner.java + env: + ORACLE_HOST: ${{ steps.secrets.outputs.ORACLE_HOST }} + ORACLE_USERNAME: ${{ steps.secrets.outputs.ORACLE_USERNAME }} + ORACLE_PASSWORD: ${{ steps.secrets.outputs.ORACLE_PASSWORD }} + ORACLE_PORT: ${{ steps.secrets.outputs.ORACLE_PORT }} + + + - name: Upload report + uses: actions/upload-artifact@v3 + if: always() + with: + name: Cucumber report + path: ./**/target/cucumber-reports + + - name: Upload debug files + uses: actions/upload-artifact@v3 + if: always() + with: + name: Debug files + path: ./**/target/e2e-debug + + - name: Upload files to GCS + uses: google-github-actions/upload-cloud-storage@v0 + if: always() + with: + path: ./plugin + destination: e2e-tests-cucumber-reports/${{ github.event.repository.name }}/${{ github.ref }} + glob: '**/target/cucumber-reports/**' \ No newline at end of file diff --git a/src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java b/src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java index 2959b32..fd0307d 100644 --- a/src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java +++ b/src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java @@ -25,8 +25,8 @@ @RunWith(Cucumber.class) @CucumberOptions( features = {"src/e2e-test/features"}, - glue = {"stepsdesign", "io.cdap.plugin.common.stepsdesign", "io.cdap.plugin.oracle.stepsdesign"}, - tags = {"@Oracle"}, + glue = {"stepsdesign", "io.cdap.plugin.stepsdesign"}, + tags = {"@Oracle"}, monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/oracle", "json:target/cucumber-reports/cucumber-oracle.json", "junit:target/cucumber-reports/cucumber-oracle.xml"} From 8106d032d4857f41d443f481dd454937c14c3a96 Mon Sep 17 00:00:00 2001 From: yjhawar Date: Tue, 11 Apr 2023 16:36:12 +0530 Subject: [PATCH 07/14] fixed build issues --- .github/workflows/e2e.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e5d84e4..f8a790d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,3 +1,17 @@ +# Copyright © 2023 Cask Data, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# This workflow will build a Java project with Maven +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven +# Note: Any changes to this workflow would be used only after merging into develop name: Build e2e tests on: From 52949417d49cb80427eeb8611fc5f7c3120ef521 Mon Sep 17 00:00:00 2001 From: yjhawar Date: Tue, 11 Apr 2023 18:17:41 +0530 Subject: [PATCH 08/14] Fixed checkstyle issues and minor fix --- .../actions/ReplicationActions.java | 22 +++++-- .../io.cdap.plugin/actions/package-info.java | 7 ++- .../io.cdap.plugin/hooks/TestSetUpHooks.java | 21 ++----- .../io.cdap.plugin/hooks/package-info.java | 19 ++++++ .../locators/ReplicationLocators.java | 8 ++- .../io.cdap.plugin/locators/package-info.java | 6 +- .../stepsdesign/StepDefinition.java | 7 ++- .../stepsdesign/package-info.java | 19 ++++++ .../java/io.cdap.plugin/tests/TestRunner.java | 9 ++- .../java/io.cdap.plugin/utils/BigQuery.java | 8 ++- .../io.cdap.plugin/utils/OracleClient.java | 58 +++++++++++++------ .../utils/ValidationHelper.java | 22 ++++--- .../io.cdap.plugin/utils/package-info.java | 19 ++++++ .../resources/errorMessage.properties | 2 +- .../resources/pluginParameters.properties | 6 +- 15 files changed, 163 insertions(+), 70 deletions(-) create mode 100644 src/e2e-test/java/io.cdap.plugin/hooks/package-info.java create mode 100644 src/e2e-test/java/io.cdap.plugin/stepsdesign/package-info.java create mode 100644 src/e2e-test/java/io.cdap.plugin/utils/package-info.java diff --git a/src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java b/src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java index ad659f4..e51c9be 100644 --- a/src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java +++ b/src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java @@ -18,7 +18,14 @@ import io.cdap.e2e.pages.actions.CdfPipelineRunAction; import io.cdap.e2e.pages.locators.CdfPipelineRunLocators; -import io.cdap.e2e.utils.*; +import io.cdap.e2e.utils.AssertionHelper; +import io.cdap.e2e.utils.ConstantsUtil; +import io.cdap.e2e.utils.ElementHelper; +import io.cdap.e2e.utils.PageHelper; +import io.cdap.e2e.utils.PluginPropertyUtils; +import io.cdap.e2e.utils.SeleniumDriver; +import io.cdap.e2e.utils.SeleniumHelper; +import io.cdap.e2e.utils.WaitHelper; import io.cdap.plugin.locators.ReplicationLocators; import io.cdap.plugin.utils.BigQuery; import io.cdap.plugin.utils.OracleClient; @@ -34,6 +41,9 @@ import java.util.Map; import java.util.concurrent.TimeUnit; +/** + * Replication oracle Actions. + */ public class ReplicationActions { private static String parentWindow = StringUtils.EMPTY; private static final String projectId = PluginPropertyUtils.pluginProp("projectId"); @@ -60,7 +70,7 @@ public static void clickOnOraclePlugin() { public static void selectTable() { String table = schemaName + "." + tableName; - WaitHelper.waitForElementToBeDisplayed(ReplicationLocators.selectTable(table),300); + WaitHelper.waitForElementToBeDisplayed(ReplicationLocators.selectTable(table), 300); AssertionHelper.verifyElementDisplayed(ReplicationLocators.selectTable(table)); ElementHelper.clickOnElement(ReplicationLocators.selectTable(table)); } @@ -120,7 +130,6 @@ public static void waitTillPipelineIsRunningAndCheckForErrors() throws Interrupt } public static void closeTheLogsAndClickOnStopButton() { - //As the logs get opened in a new window in this plugin so after closing them we have to switch to parent window. SeleniumDriver.getDriver().switchTo().window(parentWindow); //Stopping the pipeline ElementHelper.clickOnElement(ReplicationLocators.stop); @@ -133,7 +142,8 @@ public static void verifyTargetBigQueryRecordMatchesExpectedOracleRecord() Assert.assertFalse(ElementHelper.isElementDisplayed(ReplicationLocators.error)); List> sourceOracleRecords = OracleClient.getOracleRecordsAsMap(tableName, schemaName); - List> targetBigQueryRecords = ValidationHelper.getBigQueryRecordsAsMap(projectId, database, tableName); + List> targetBigQueryRecords = + ValidationHelper.getBigQueryRecordsAsMap(projectId, database, tableName); ValidationHelper.validateRecords(sourceOracleRecords, targetBigQueryRecords); } @@ -151,8 +161,8 @@ public static void deleteRecordAndWait() throws SQLException, ClassNotFoundExcep } public static void updateRecordAndWait() throws SQLException, ClassNotFoundException, InterruptedException { - OracleClient.updateRow(tableName, schemaName, updateCondition, updatedValue ); + OracleClient.updateRow(tableName, schemaName, updateCondition, updatedValue); OracleClient.forceFlushCDC(); BigQuery.waitForFlush(); } -} \ No newline at end of file +} diff --git a/src/e2e-test/java/io.cdap.plugin/actions/package-info.java b/src/e2e-test/java/io.cdap.plugin/actions/package-info.java index 8e01adf..0bf092a 100644 --- a/src/e2e-test/java/io.cdap.plugin/actions/package-info.java +++ b/src/e2e-test/java/io.cdap.plugin/actions/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023. + * Copyright © 2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -14,4 +14,7 @@ * the License. */ -package io.cdap.plugin.actions; \ No newline at end of file +/** + * contains Actions methods. + */ +package io.cdap.plugin.actions; diff --git a/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java b/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java index ac59b3b..721d0ac 100644 --- a/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java +++ b/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java @@ -39,10 +39,9 @@ public class TestSetUpHooks { public static List> sourceOracleRecords = new ArrayList<>(); public static String tableName = PluginPropertyUtils.pluginProp("sourceTable"); public static String schemaName = PluginPropertyUtils.pluginProp("schema"); - public static String primaryKey = PluginPropertyUtils.pluginProp("primaryKey"); public static String datatypeColumns = PluginPropertyUtils.pluginProp("datatypeColumns"); public static String row1 = PluginPropertyUtils.pluginProp("datatypeValuesRow1"); - public static String row2= PluginPropertyUtils.pluginProp("datatypeValuesRow2"); + public static String row2 = PluginPropertyUtils.pluginProp("datatypeValuesRow2"); @Before(order = 1, value = "@ENV_VARIABLES") public static void overridePropertiesFromEnvVarsIfProvided() { @@ -59,7 +58,7 @@ public static void overridePropertiesFromEnvVarsIfProvided() { PluginPropertyUtils.addPluginProp("password", password); } String port = System.getenv("ORACLE_PORT"); - if (port!= null && !port.isEmpty()) { + if (port != null && !port.isEmpty()) { PluginPropertyUtils.addPluginProp("port", port); } String oracleHost = System.getenv("ORACLE_HOST"); @@ -75,7 +74,7 @@ public static void overridePropertiesFromEnvVarsIfProvided() { @Before(order = 2, value = "@ORACLE_SOURCE") public static void createTable() throws SQLException, ClassNotFoundException { - OracleClient.createTable(tableName, schemaName, datatypeColumns, primaryKey); + OracleClient.createTable(tableName, schemaName, datatypeColumns); } @Before(order = 3, value = "@ORACLE_SOURCE") @@ -97,16 +96,6 @@ public static void dropTables() throws SQLException, ClassNotFoundException { @After(order = 1, value = "@BIGQUERY_DELETE") public static void deleteTempTargetBQTable() throws IOException, InterruptedException { - try { - BigQueryClient.dropBqQuery(tableName); - BeforeActions.scenario.write("BQ Target table - " + tableName + " deleted successfully"); - } catch (BigQueryException e) { - if (e.getMessage().contains("Not found: Table")) { - BeforeActions.scenario.write("BQ Target Table does not exist"); - } else { - Assert.fail(e.getMessage()); - } - } + BigQuery.deleteTable(tableName); } - -} \ No newline at end of file +} diff --git a/src/e2e-test/java/io.cdap.plugin/hooks/package-info.java b/src/e2e-test/java/io.cdap.plugin/hooks/package-info.java new file mode 100644 index 0000000..46e9944 --- /dev/null +++ b/src/e2e-test/java/io.cdap.plugin/hooks/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +/** + * Represent Test Setup/Clean up hooks. + */ +package io.cdap.plugin.hooks; diff --git a/src/e2e-test/java/io.cdap.plugin/locators/ReplicationLocators.java b/src/e2e-test/java/io.cdap.plugin/locators/ReplicationLocators.java index a5690b2..cce2e17 100644 --- a/src/e2e-test/java/io.cdap.plugin/locators/ReplicationLocators.java +++ b/src/e2e-test/java/io.cdap.plugin/locators/ReplicationLocators.java @@ -21,11 +21,13 @@ import org.openqa.selenium.WebElement; import org.openqa.selenium.support.FindBy; import org.openqa.selenium.support.How; - +/** + * Oracle Plugin Locators. + */ public class ReplicationLocators { @FindBy(how = How.XPATH, using = "//*[contains(text(),'Next')]") public static WebElement next; - @FindBy(how = How.XPATH, using ="//div[contains(text(),'Oracle')]") + @FindBy(how = How.XPATH, using = "//div[contains(text(),'Oracle')]") public static WebElement oraclePlugin; public static WebElement selectTable(String tableName) { return SeleniumDriver.getDriver().findElement(By.xpath("//div[contains(text(),'" + tableName + "')]" + @@ -46,4 +48,4 @@ public static WebElement selectTable(String tableName) { @FindBy(how = How.XPATH, using = "//*[contains(text(),'Stopped')]") public static WebElement stopped; public static By start = By.xpath("//*[contains(@class, 'icon-play ')]"); -} \ No newline at end of file +} diff --git a/src/e2e-test/java/io.cdap.plugin/locators/package-info.java b/src/e2e-test/java/io.cdap.plugin/locators/package-info.java index e9988de..f4dfbbf 100644 --- a/src/e2e-test/java/io.cdap.plugin/locators/package-info.java +++ b/src/e2e-test/java/io.cdap.plugin/locators/package-info.java @@ -13,5 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ - -package io.cdap.plugin.locators; \ No newline at end of file +/** + * contains CDF Oracle Replicator plugin locators. + */ +package io.cdap.plugin.locators; diff --git a/src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java b/src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java index b3af423..ee7fa66 100644 --- a/src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java +++ b/src/e2e-test/java/io.cdap.plugin/stepsdesign/StepDefinition.java @@ -26,7 +26,9 @@ import java.io.IOException; import java.sql.SQLException; - +/** + * Contains Oracle replication test scenarios step definitions. + */ public class StepDefinition implements CdfHelper { @Given("Open DataFusion Project with replication to configure pipeline") public void openDataFusionProjectWithReplicationToConfigurePipeline() throws IOException, InterruptedException { @@ -92,7 +94,8 @@ public void triggerUpdateCdcEvent() throws IOException, InterruptedException, SQ ReplicationActions.updateRecordAndWait(); //JCoException, } @Then("Verify expected Oracle records in target BigQuery table") - public void verifyExpectedOracleRecordsInTargetBigQueryTable() throws IOException, InterruptedException, SQLException, ClassNotFoundException { + public void verifyExpectedOracleRecordsInTargetBigQueryTable() throws + IOException, InterruptedException, SQLException, ClassNotFoundException { ReplicationActions.verifyTargetBigQueryRecordMatchesExpectedOracleRecord(); } diff --git a/src/e2e-test/java/io.cdap.plugin/stepsdesign/package-info.java b/src/e2e-test/java/io.cdap.plugin/stepsdesign/package-info.java new file mode 100644 index 0000000..8c1da89 --- /dev/null +++ b/src/e2e-test/java/io.cdap.plugin/stepsdesign/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +/** + * Contains Oracle replication test scenarios step definitions. + */ +package io.cdap.plugin.stepsdesign; diff --git a/src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java b/src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java index fd0307d..007cd9a 100644 --- a/src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java +++ b/src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java @@ -24,12 +24,11 @@ */ @RunWith(Cucumber.class) @CucumberOptions( - features = {"src/e2e-test/features"}, - glue = {"stepsdesign", "io.cdap.plugin.stepsdesign"}, + features = {"e2e-test/features"}, + glue = {"stepsdesign", "io.cdap.plugin.stepsdesign", "io.cdap.plugin.hooks"}, tags = {"@Oracle"}, monochrome = true, - plugin = {"pretty", "html:target/cucumber-html-report/oracle", - "json:target/cucumber-reports/cucumber-oracle.json", - "junit:target/cucumber-reports/cucumber-oracle.xml"} + plugin = {"pretty", "html:target/cucumber-html-report", "json:target/cucumber-reports/cucumber.json", + "junit:target/cucumber-reports/cucumber.xml"} ) public class TestRunner { } diff --git a/src/e2e-test/java/io.cdap.plugin/utils/BigQuery.java b/src/e2e-test/java/io.cdap.plugin/utils/BigQuery.java index 3cd3df7..46d5cd0 100644 --- a/src/e2e-test/java/io.cdap.plugin/utils/BigQuery.java +++ b/src/e2e-test/java/io.cdap.plugin/utils/BigQuery.java @@ -24,7 +24,9 @@ import java.io.IOException; import java.util.concurrent.TimeUnit; - +/** + * Contains bq helper methods used in e2e tests. + */ public class BigQuery { public static void waitForFlush() throws InterruptedException { int flushInterval = Integer.parseInt(PluginPropertyUtils.pluginProp("loadInterval")); @@ -32,7 +34,7 @@ public static void waitForFlush() throws InterruptedException { time.sleep(2 * flushInterval + 60); } - public static void deleteTable(String tableName) throws IOException, InterruptedException{ + public static void deleteTable(String tableName) throws IOException, InterruptedException { try { BigQueryClient.dropBqQuery(tableName); BeforeActions.scenario.write("BQ Target table - " + tableName + " deleted successfully"); @@ -44,4 +46,4 @@ public static void deleteTable(String tableName) throws IOException, Interrupted } } } -} \ No newline at end of file +} diff --git a/src/e2e-test/java/io.cdap.plugin/utils/OracleClient.java b/src/e2e-test/java/io.cdap.plugin/utils/OracleClient.java index 1b8a37a..0bc5828 100644 --- a/src/e2e-test/java/io.cdap.plugin/utils/OracleClient.java +++ b/src/e2e-test/java/io.cdap.plugin/utils/OracleClient.java @@ -19,10 +19,20 @@ import io.cdap.e2e.utils.PluginPropertyUtils; -import java.sql.*; -import java.time.Instant; -import java.util.*; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; import java.util.concurrent.TimeUnit; /** @@ -39,29 +49,34 @@ private static Connection getOracleConnection() throws SQLException, ClassNotFou String username = PluginPropertyUtils.pluginProp("username"); String password = PluginPropertyUtils.pluginProp("password"); - return DriverManager.getConnection("jdbc:oracle:thin:@//" +host + return DriverManager.getConnection("jdbc:oracle:thin:@//" + host + ":" + port + "/" + databaseName, username, password); } - public static void createTable(String table, String schema, String datatypeColumns, String primaryKey) throws SQLException, ClassNotFoundException { + public static void createTable(String table, String schema, String datatypeColumns) + throws SQLException, ClassNotFoundException { try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { String createTableQuery = "CREATE TABLE " + schema + "." + table + datatypeColumns; statement.executeUpdate(createTableQuery); } } - public static void forceFlushCDC() throws SQLException, ClassNotFoundException{ + public static void forceFlushCDC() throws SQLException, ClassNotFoundException { try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { - //Oracle doesn't immediately flush CDC events, it can automatically happen based on time/log file size or you can force it + /* + Oracle doesn't immediately flush CDC events, it can automatically happen based + on time/log file size or you can force it + */ statement.executeUpdate("ALTER SYSTEM SWITCH LOGFILE"); } } - public static void insertRow(String table, String schema, String datatypeValues) throws SQLException, ClassNotFoundException { + public static void insertRow (String table, String schema, String datatypeValues) throws + SQLException, ClassNotFoundException { try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { // Insert dummy data. statement.executeUpdate("INSERT INTO " + schema + "." + table + " " + @@ -69,13 +84,15 @@ public static void insertRow(String table, String schema, String datatypeValues) } } - public static void deleteRow(String table, String schema, String deleteCondition) throws SQLException, ClassNotFoundException { + public static void deleteRow(String table, String schema, String deleteCondition) throws SQLException, + ClassNotFoundException { try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { // Insert dummy data. statement.executeUpdate("DELETE FROM " + schema + "." + table + " WHERE " + deleteCondition); } } - public static void updateRow(String table, String schema, String updateCondition, String updatedValue) throws SQLException, ClassNotFoundException { + public static void updateRow(String table, String schema, String updateCondition, String updatedValue) throws + SQLException, ClassNotFoundException { try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { // Insert dummy data. statement.executeUpdate("UPDATE " + schema + "." + table + " SET " + updatedValue + @@ -83,7 +100,8 @@ public static void updateRow(String table, String schema, String updateCondition } } - public static List> getOracleRecordsAsMap(String table, String schema)throws SQLException, ClassNotFoundException { + public static List> getOracleRecordsAsMap(String table, String schema) throws SQLException, + ClassNotFoundException { try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { // Insert dummy data. List> oracleRecords = new ArrayList<>(); @@ -94,26 +112,30 @@ public static List> getOracleRecordsAsMap(String table, Stri int numberOfColumns = rsmd.getColumnCount(); List columns = new ArrayList<>(); columns.add(""); - for(int colIndex = 1 ; colIndex <= numberOfColumns ; colIndex++){ + for (int colIndex = 1; colIndex <= numberOfColumns; colIndex++) { columns.add(rsmd.getColumnName(colIndex) + "#" + rsmd.getColumnType(colIndex)); } while (result.next()) { Map record = new HashMap<>(); - for(int colIndex = 1 ; colIndex <= numberOfColumns ; colIndex++){ + for (int colIndex = 1; colIndex <= numberOfColumns; colIndex++) { String columnName = columns.get(colIndex).split("#")[0]; int type = Integer.parseInt(columns.get(colIndex).split("#")[1]); Object value; - switch (type){ + switch (type) { case Types.TIMESTAMP: Instant instant = result.getTimestamp(colIndex).toInstant(); //Rounding off as BQ supports till microseconds - value = TimeUnit.SECONDS.toMicros(instant.getEpochSecond()) + TimeUnit.NANOSECONDS.toMicros(instant.getNano()); + value = TimeUnit.SECONDS.toMicros(instant.getEpochSecond()) + + TimeUnit.NANOSECONDS.toMicros(instant.getNano()); break; default: - //else convert all data types toString as bq converts certain data types to string to preserve precision and scale + /* + Convert all data types toString as bq converts certain data types to string + to preserve precision and scale + */ value = result.getString(colIndex); } - record.put(columnName,value); + record.put(columnName, value); } oracleRecords.add(record); } @@ -129,4 +151,4 @@ public static void deleteTables(String schema, String table) statement.execute(dropTableQuery); } } -} \ No newline at end of file +} diff --git a/src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java b/src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java index b351b36..c21c8a8 100644 --- a/src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java +++ b/src/e2e-test/java/io.cdap.plugin/utils/ValidationHelper.java @@ -23,11 +23,17 @@ import io.cdap.e2e.utils.BigQueryClient; import io.cdap.e2e.utils.PluginPropertyUtils; import org.junit.Assert; + import java.io.IOException; -import java.util.*; -import java.util.concurrent.TimeUnit; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; - +/** + * Contains validation method from oracle to bq used in e2e tests. + */ public class ValidationHelper { public static List> getBigQueryRecordsAsMap(String projectId, String database, String tableName) @@ -47,7 +53,7 @@ public static List> getBigQueryRecordsAsMap(String projectId String columnName = columns.get(index).split("#")[0]; String dataType = columns.get(index).split("#")[1]; Object value; - if(dataType.equalsIgnoreCase("TIMESTAMP")){ + if (dataType.equalsIgnoreCase("TIMESTAMP")) { value = fieldValue.getTimestampValue(); } else { value = fieldValue.getValue(); @@ -66,16 +72,16 @@ public static void validateRecords(List> sourceOracleRecords String uniqueField = PluginPropertyUtils.pluginProp("primaryKey"); // Logic to maintain the order of both lists and validate records based on that order. - Map bqUniqueIdMap = (Map)targetBigQueryRecords.stream() - .filter(t -> t.get("_is_deleted")==null) + Map bqUniqueIdMap = (Map) targetBigQueryRecords.stream() + .filter(t -> t.get("_is_deleted") == null) .collect(Collectors.toMap( t -> t.get(uniqueField), t -> t, //This logic is to handle duplication scenario in bq target - (x,y) -> { + (x, y) -> { Long xSeqNum = Long.parseLong(x.get("_sequence_num").toString()); Long ySeqNum = Long.parseLong(y.get("_sequence_num").toString()); - if(xSeqNum > ySeqNum){ + if (xSeqNum > ySeqNum) { return x; } return y; diff --git a/src/e2e-test/java/io.cdap.plugin/utils/package-info.java b/src/e2e-test/java/io.cdap.plugin/utils/package-info.java new file mode 100644 index 0000000..185a9c1 --- /dev/null +++ b/src/e2e-test/java/io.cdap.plugin/utils/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +/** + * Contains helper methods used in e2e tests. + */ +package io.cdap.plugin.utils; diff --git a/src/e2e-test/resources/errorMessage.properties b/src/e2e-test/resources/errorMessage.properties index 283352d..0aa0344 100644 --- a/src/e2e-test/resources/errorMessage.properties +++ b/src/e2e-test/resources/errorMessage.properties @@ -1 +1 @@ -validationSuccessMessage=No errors found \ No newline at end of file +validationSuccessMessage=No errors found diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index 57b6385..b89dab0 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -1,5 +1,3 @@ - - projectId=PROJECT_ID dataset=ORCL schema=HR @@ -9,7 +7,7 @@ port=ORACLE_PORT username=ORACLE_USERNAME password=ORACLE_PASSWORD -pipelineName=e2e-sanity-existing-table +pipelineName=e2e-sanity pipeline-initialization=200 loadInterval=30 @@ -45,4 +43,4 @@ datatypeValuesForInsertOperation=('USER3', 'M','ABCDEF','ABC','ABC',' 'AAAAaoAATAAABrXAAA',1234,1234.56789,\ 1234.56789,1234.56789,1234.56789,1234.56789,1234.56789,1234.56789,1234.5679,\ 1234.56789,1234.5679,1234.5679,1234.56789,TIMESTAMP'2023-01-01 2:00:00',TIMESTAMP'2023-01-01 2:00:00',\ - TIMESTAMP '2023-01-01 00:00:00.000000', NULL) \ No newline at end of file + TIMESTAMP '2023-01-01 00:00:00.000000', NULL) From 38df2106bdf60e8b6f7e516195855f65a8cda3f7 Mon Sep 17 00:00:00 2001 From: yjhawar Date: Wed, 12 Apr 2023 14:34:36 +0530 Subject: [PATCH 09/14] corrected path for feature --- src/e2e-test/features/Pipeline.feature | 2 +- src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java | 6 ++---- src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/e2e-test/features/Pipeline.feature b/src/e2e-test/features/Pipeline.feature index 0b6a7e5..da6a28c 100644 --- a/src/e2e-test/features/Pipeline.feature +++ b/src/e2e-test/features/Pipeline.feature @@ -15,7 +15,7 @@ # Feature: Oracle - Verify Oracle source data transfer to Big Query - @ENV_VARIABLES #@ORACLE_SOURCE @ORACLE_DELETE @BIGQUERY_DELETE + @ENV_VARIABLES @ORACLE_SOURCE @ORACLE_DELETE @BIGQUERY_DELETE Scenario: To verify replication of snapshot and cdc data from Oracle to Big Query successfully with Sanity test Given Open DataFusion Project with replication to configure pipeline When Enter input plugin property: "name" with value: "pipelineName" diff --git a/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java b/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java index 721d0ac..926a7dd 100644 --- a/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java +++ b/src/e2e-test/java/io.cdap.plugin/hooks/TestSetUpHooks.java @@ -15,14 +15,12 @@ */ package io.cdap.plugin.hooks; -import com.google.cloud.bigquery.BigQueryException; -import io.cdap.e2e.utils.BigQueryClient; + import io.cdap.e2e.utils.PluginPropertyUtils; import io.cdap.plugin.utils.BigQuery; import io.cdap.plugin.utils.OracleClient; import io.cucumber.java.After; import io.cucumber.java.Before; -import org.junit.Assert; import stepsdesign.BeforeActions; import java.io.IOException; @@ -95,7 +93,7 @@ public static void dropTables() throws SQLException, ClassNotFoundException { } @After(order = 1, value = "@BIGQUERY_DELETE") - public static void deleteTempTargetBQTable() throws IOException, InterruptedException { + public static void deleteTargetBQTable() throws IOException, InterruptedException { BigQuery.deleteTable(tableName); } } diff --git a/src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java b/src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java index 007cd9a..5c063ed 100644 --- a/src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java +++ b/src/e2e-test/java/io.cdap.plugin/tests/TestRunner.java @@ -24,7 +24,7 @@ */ @RunWith(Cucumber.class) @CucumberOptions( - features = {"e2e-test/features"}, + features = {"src/e2e-test/features"}, glue = {"stepsdesign", "io.cdap.plugin.stepsdesign", "io.cdap.plugin.hooks"}, tags = {"@Oracle"}, monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report", "json:target/cucumber-reports/cucumber.json", From 3fed35f18e6f55c87931d17b31e7f19016bdb671 Mon Sep 17 00:00:00 2001 From: yjhawar Date: Wed, 12 Apr 2023 16:08:21 +0530 Subject: [PATCH 10/14] Removing runner path --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f8a790d..54c8189 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -91,7 +91,7 @@ jobs: - name: Run all e2e tests if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || steps.filter.outputs.e2e-test == 'true' - run: python3 e2e/src/main/scripts/run_e2e_test.py --testRunner **/**/TestRunner.java + run: python3 e2e/src/main/scripts/run_e2e_test.py env: ORACLE_HOST: ${{ steps.secrets.outputs.ORACLE_HOST }} ORACLE_USERNAME: ${{ steps.secrets.outputs.ORACLE_USERNAME }} From 375534947534d4745065310041d9c8655389c713 Mon Sep 17 00:00:00 2001 From: yjhawar Date: Wed, 12 Apr 2023 16:58:14 +0530 Subject: [PATCH 11/14] add correct tag to runner file --- src/e2e-test/features/Pipeline.feature | 1 + 1 file changed, 1 insertion(+) diff --git a/src/e2e-test/features/Pipeline.feature b/src/e2e-test/features/Pipeline.feature index da6a28c..f6e2189 100644 --- a/src/e2e-test/features/Pipeline.feature +++ b/src/e2e-test/features/Pipeline.feature @@ -14,6 +14,7 @@ # the License. # +@Oracle Feature: Oracle - Verify Oracle source data transfer to Big Query @ENV_VARIABLES @ORACLE_SOURCE @ORACLE_DELETE @BIGQUERY_DELETE Scenario: To verify replication of snapshot and cdc data from Oracle to Big Query successfully with Sanity test From 9891050524d7a373936b2a493eb91b725ba59ce7 Mon Sep 17 00:00:00 2001 From: Sumit Jain Date: Thu, 13 Apr 2023 09:45:33 +0530 Subject: [PATCH 12/14] Fix cucumber tests not executing due to junit 5 --- pom.xml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 5f0a95d..79a519b 100644 --- a/pom.xml +++ b/pom.xml @@ -153,7 +153,7 @@ org.junit.jupiter junit-jupiter-engine - 5.4.0 + 5.9.1 test @@ -401,7 +401,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.18.1 + 2.22.0 true @@ -410,7 +410,14 @@ org.apache.maven.plugins maven-failsafe-plugin - 3.0.0-M5 + 3.0.0 + + + org.apache.maven.surefire + surefire-junit47 + 3.0.0 + + ${TEST_RUNNER} From 62167e0261605f277c9e158a287ddbe3967bd471 Mon Sep 17 00:00:00 2001 From: yjhawar Date: Thu, 13 Apr 2023 12:55:12 +0530 Subject: [PATCH 13/14] added secret manager values --- .github/workflows/e2e.yml | 9 +++++---- src/e2e-test/features/Pipeline.feature | 2 +- src/e2e-test/resources/pluginParameters.properties | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 54c8189..7373e08 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -76,9 +76,10 @@ jobs: with: secrets: |- ORACLE_HOST:cdapio-github-builds/ORACLE_HOST - ORACLE_USERNAME:cdapio-github-builds/ORACLE_USERNAME - ORACLE_PASSWORD:cdapio-github-builds/ORACLE_PASSWORD + ORACLE_USERNAME:cdapio-github-builds/ORACLE_NORMAL_USERNAME + ORACLE_PASSWORD:cdapio-github-builds/ORACLE_NORMAL_PASSWORD ORACLE_PORT:cdapio-github-builds/ORACLE_PORT + PROJECT_ID:cdapio-github-builds/PROJECT_ID - name: Run required e2e tests if: github.event_name != 'workflow_dispatch' && github.event_name != 'push' && steps.filter.outputs.e2e-test == 'false' run: python3 e2e/src/main/scripts/run_e2e_test.py --testRunner TestRunnerRequired.java @@ -91,13 +92,13 @@ jobs: - name: Run all e2e tests if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || steps.filter.outputs.e2e-test == 'true' - run: python3 e2e/src/main/scripts/run_e2e_test.py + run: python3 e2e/src/main/scripts/run_e2e_test.py --testRunner TestRunner.java env: ORACLE_HOST: ${{ steps.secrets.outputs.ORACLE_HOST }} ORACLE_USERNAME: ${{ steps.secrets.outputs.ORACLE_USERNAME }} ORACLE_PASSWORD: ${{ steps.secrets.outputs.ORACLE_PASSWORD }} ORACLE_PORT: ${{ steps.secrets.outputs.ORACLE_PORT }} - + PROJECT_ID : ${{ steps.secrets.outputs.PROJECT_ID }} - name: Upload report uses: actions/upload-artifact@v3 diff --git a/src/e2e-test/features/Pipeline.feature b/src/e2e-test/features/Pipeline.feature index f6e2189..f2c8547 100644 --- a/src/e2e-test/features/Pipeline.feature +++ b/src/e2e-test/features/Pipeline.feature @@ -16,7 +16,7 @@ @Oracle Feature: Oracle - Verify Oracle source data transfer to Big Query - @ENV_VARIABLES @ORACLE_SOURCE @ORACLE_DELETE @BIGQUERY_DELETE + @ENV_VARIABLES Scenario: To verify replication of snapshot and cdc data from Oracle to Big Query successfully with Sanity test Given Open DataFusion Project with replication to configure pipeline When Enter input plugin property: "name" with value: "pipelineName" diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index b89dab0..371e202 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -1,7 +1,7 @@ projectId=PROJECT_ID -dataset=ORCL -schema=HR -sourceTable=SOURCE_TABLE +dataset=XE +schema=ABC +sourceTable=SANITY host=ORACLE_HOST port=ORACLE_PORT username=ORACLE_USERNAME From 847262501a23846440662fb259381a4c9b522cc8 Mon Sep 17 00:00:00 2001 From: yjhawar Date: Thu, 13 Apr 2023 16:39:23 +0530 Subject: [PATCH 14/14] added different project id --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 7373e08..ecbb045 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -79,7 +79,7 @@ jobs: ORACLE_USERNAME:cdapio-github-builds/ORACLE_NORMAL_USERNAME ORACLE_PASSWORD:cdapio-github-builds/ORACLE_NORMAL_PASSWORD ORACLE_PORT:cdapio-github-builds/ORACLE_PORT - PROJECT_ID:cdapio-github-builds/PROJECT_ID + PROJECT_ID:cdapio-github-builds/PROJECT_ID_FOR_REPL - name: Run required e2e tests if: github.event_name != 'workflow_dispatch' && github.event_name != 'push' && steps.filter.outputs.e2e-test == 'false' run: python3 e2e/src/main/scripts/run_e2e_test.py --testRunner TestRunnerRequired.java