From ef616a353c343fbd28eb1814477ce3a17a73b96a Mon Sep 17 00:00:00 2001 From: Pritam Sawant Date: Wed, 11 May 2022 22:04:25 +0530 Subject: [PATCH] e2e tests CDAP-19055 - Override service account details --- pom.xml | 14 ++++++++ .../bigquery/sink/BigQuerySinkError.feature | 4 +++ .../bigquery/sink/GCSToBigQuery.feature | 8 +++++ .../sink/GCSToBigQuery_WithMacro.feature | 16 ++++++---- .../source/BigQuerySourceError.feature | 2 ++ .../source/BigQueryToBigQuery.feature | 12 +++++++ .../bigquery/source/BigQueryToGCS.feature | 6 ++++ .../source/BigQueryToGCS_WithMacro.feature | 16 ++++++---- .../source/BigQueryToMultipleSinks.feature | 4 +++ .../features/gcs/sink/GCSSink.feature | 2 ++ .../gcs/source/GCSSourceError.feature | 5 +++ .../gcs/source/GCSSourceToBigQuery.feature | 12 +++++++ .../gcs/source/GCSourceSchema.feature | 3 ++ .../features/gcsdelete/GCSDelete.feature | 4 +++ .../gcsdelete/GCSDelete_WithMacro.feature | 14 +++++--- src/e2e-test/features/gcsmove/GCSMove.feature | 6 ++++ .../gcsmove/GCSMove_WithMacro.feature | 7 ++-- .../features/pubsub/sink/BQToPubSub.feature | 2 ++ .../features/pubsub/sink/GCSToPubSub.feature | 12 +++++++ .../pubsub/sink/GCSToPubSub_WithMacro.feature | 32 ++++++++++++------- .../sink/PubSubSinkErrorScnerios.feature | 2 ++ .../features/spanner/sink/BQToSpanner.feature | 4 +++ .../spanner/sink/GCSToSpanner.feature | 4 +++ .../SpannerSourceErrorScenarios.feature | 1 + .../spanner/source/SpannertoBQ.feature | 4 +++ .../spanner/source/SpannertoGCS.feature | 4 +++ .../common/stepsdesign/TestSetupHooks.java | 30 +++++++++++++++++ .../utils/CdfPluginPropertyLocator.java | 4 +++ .../resources/pluginParameters.properties | 3 +- 29 files changed, 206 insertions(+), 31 deletions(-) diff --git a/pom.xml b/pom.xml index 45de5b917..7fe99a9a3 100644 --- a/pom.xml +++ b/pom.xml @@ -1005,6 +1005,20 @@ 2 true + + + ${GOOGLE_APPLICATION_CREDENTIALS} + + + ${SERVICE_ACCOUNT_TYPE} + + + ${SERVICE_ACCOUNT_FILE_PATH} + + + ${SERVICE_ACCOUNT_JSON} + + diff --git a/src/e2e-test/features/bigquery/sink/BigQuerySinkError.feature b/src/e2e-test/features/bigquery/sink/BigQuerySinkError.feature index 719c42f34..ee87a1cbd 100644 --- a/src/e2e-test/features/bigquery/sink/BigQuerySinkError.feature +++ b/src/e2e-test/features/bigquery/sink/BigQuerySinkError.feature @@ -21,6 +21,7 @@ Feature: BigQuery sink - Validate BigQuery sink plugin error scenarios Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery sink property table name Then Enter BigQuery sink property GCS upload request chunk size "bqInvalidChunkSize" @@ -32,6 +33,7 @@ Feature: BigQuery sink - Validate BigQuery sink plugin error scenarios When Sink is BigQuery Then Open BigQuery sink properties Then Enter BigQuery property reference name + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "bqInvalidSinkDataset" Then Enter BigQuery sink property table name Then Verify the BigQuery validation error message for invalid property "dataset" @@ -42,6 +44,7 @@ Feature: BigQuery sink - Validate BigQuery sink plugin error scenarios When Sink is BigQuery Then Open BigQuery sink properties Then Enter BigQuery property reference name + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery property table "bqInvalidSinkTable" Then Verify the BigQuery validation error message for invalid property "table" @@ -54,6 +57,7 @@ Feature: BigQuery sink - Validate BigQuery sink plugin error scenarios Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery sink property table name Then Enter BigQuery property temporary bucket name "bqInvalidTemporaryBucket" diff --git a/src/e2e-test/features/bigquery/sink/GCSToBigQuery.feature b/src/e2e-test/features/bigquery/sink/GCSToBigQuery.feature index 75fe8db00..7a20e4d61 100644 --- a/src/e2e-test/features/bigquery/sink/GCSToBigQuery.feature +++ b/src/e2e-test/features/bigquery/sink/GCSToBigQuery.feature @@ -7,6 +7,7 @@ Feature: BigQuery sink - Verification of GCS to BigQuery successful data transfe When Source is GCS When Sink is BigQuery Then Open GCS source properties + Then Override Service account details if set in environment variables Then Enter the GCS source mandatory properties Then Validate "GCS" plugin properties Then Close the GCS properties @@ -14,6 +15,7 @@ Feature: BigQuery sink - Verification of GCS to BigQuery successful data transfe Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery sink property table name Then Enter BiqQuery property encryption key name "cmekBQ" if cmek is enabled @@ -41,6 +43,7 @@ Feature: BigQuery sink - Verification of GCS to BigQuery successful data transfe When Sink is BigQuery Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "gcsCsvRangeFile" Then Toggle GCS source property skip header to true Then Select GCS property format "csv" @@ -51,6 +54,7 @@ Feature: BigQuery sink - Verification of GCS to BigQuery successful data transfe Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery sink property table name Then Toggle BigQuery sink property truncateTable to true @@ -82,6 +86,7 @@ Feature: BigQuery sink - Verification of GCS to BigQuery successful data transfe When Sink is BigQuery Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "gcsCsvRangeFile" Then Toggle GCS source property skip header to true Then Select GCS property format "csv" @@ -92,6 +97,7 @@ Feature: BigQuery sink - Verification of GCS to BigQuery successful data transfe Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery sink property table name Then Toggle BigQuery sink property truncateTable to true @@ -118,6 +124,7 @@ Feature: BigQuery sink - Verification of GCS to BigQuery successful data transfe When Sink is BigQuery Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "gcsCsvRangeFile" Then Toggle GCS source property skip header to true Then Select GCS property format "csv" @@ -128,6 +135,7 @@ Feature: BigQuery sink - Verification of GCS to BigQuery successful data transfe Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery sink property table name Then Enter BigQuery sink property GCS upload request chunk size "bqChunkSize" diff --git a/src/e2e-test/features/bigquery/sink/GCSToBigQuery_WithMacro.feature b/src/e2e-test/features/bigquery/sink/GCSToBigQuery_WithMacro.feature index a725553b1..b05a86134 100644 --- a/src/e2e-test/features/bigquery/sink/GCSToBigQuery_WithMacro.feature +++ b/src/e2e-test/features/bigquery/sink/GCSToBigQuery_WithMacro.feature @@ -9,7 +9,9 @@ Feature: BigQuery sink - Verification of GCS to BigQuery successful data transfe Then Open GCS source properties Then Enter GCS property reference name Then Enter GCS property "projectId" as macro argument "gcsProjectId" - Then Enter GCS property "serviceAccountFilePath" as macro argument "gcsServiceAccount" + Then Enter GCS property "serviceAccountType" as macro argument "serviceAccountType" + Then Enter GCS property "serviceAccountFilePath" as macro argument "serviceAccount" + Then Enter GCS property "serviceAccountJSON" as macro argument "serviceAccount" Then Enter GCS property "path" as macro argument "gcsSourcePath" Then Enter GCS source property "skipHeader" as macro argument "gcsSkipHeader" Then Enter GCS property "format" as macro argument "gcsFormat" @@ -20,7 +22,9 @@ Feature: BigQuery sink - Verification of GCS to BigQuery successful data transfe Then Enter BigQuery property reference name Then Enter BigQuery property "projectId" as macro argument "bqProjectId" Then Enter BigQuery property "datasetProjectId" as macro argument "bqDatasetProjectId" - Then Enter BigQuery property "serviceAccountFilePath" as macro argument "bqServiceAccount" + Then Enter GCS property "serviceAccountType" as macro argument "serviceAccountType" + Then Enter GCS property "serviceAccountFilePath" as macro argument "serviceAccount" + Then Enter GCS property "serviceAccountJSON" as macro argument "serviceAccount" Then Enter BigQuery property "dataset" as macro argument "bqDataset" Then Enter BigQuery property "table" as macro argument "bqTargetTable" Then Enter BigQuery cmek property "encryptionKeyName" as macro argument "cmekBQ" if cmek is enabled @@ -32,14 +36,14 @@ Feature: BigQuery sink - Verification of GCS to BigQuery successful data transfe Then Save the pipeline Then Preview and run the pipeline Then Enter runtime argument value "projectId" for key "gcsProjectId" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "gcsServiceAccount" + Then Enter runtime argument value "serviceAccountType" for key "serviceAccountType" + Then Enter runtime argument value "serviceAccount" for key "serviceAccount" Then Enter runtime argument value "gcsCsvFile" for GCS source property path key "gcsSourcePath" Then Enter runtime argument value "gcsSkipHeaderTrue" for key "gcsSkipHeader" Then Enter runtime argument value "csvFormat" for key "gcsFormat" Then Enter runtime argument value "gcsCSVFileOutputSchema" for key "gcsOutputSchema" Then Enter runtime argument value "projectId" for key "bqProjectId" Then Enter runtime argument value "projectId" for key "bqDatasetProjectId" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "bqServiceAccount" Then Enter runtime argument value "dataset" for key "bqDataset" Then Enter runtime argument value for BigQuery sink table name key "bqTargetTable" Then Enter runtime argument value "cmekBQ" for BigQuery cmek property key "cmekBQ" if BQ cmek is enabled @@ -51,14 +55,14 @@ Feature: BigQuery sink - Verification of GCS to BigQuery successful data transfe Then Deploy the pipeline Then Run the Pipeline in Runtime Then Enter runtime argument value "projectId" for key "gcsProjectId" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "gcsServiceAccount" + Then Enter runtime argument value "serviceAccountType" for key "serviceAccountType" + Then Enter runtime argument value "serviceAccount" for key "serviceAccount" Then Enter runtime argument value "gcsCsvFile" for GCS source property path key "gcsSourcePath" Then Enter runtime argument value "gcsSkipHeaderTrue" for key "gcsSkipHeader" Then Enter runtime argument value "csvFormat" for key "gcsFormat" Then Enter runtime argument value "gcsCSVFileOutputSchema" for key "gcsOutputSchema" Then Enter runtime argument value "projectId" for key "bqProjectId" Then Enter runtime argument value "projectId" for key "bqDatasetProjectId" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "bqServiceAccount" Then Enter runtime argument value "dataset" for key "bqDataset" Then Enter runtime argument value for BigQuery sink table name key "bqTargetTable" Then Enter runtime argument value "cmekBQ" for BigQuery cmek property key "cmekBQ" if BQ cmek is enabled diff --git a/src/e2e-test/features/bigquery/source/BigQuerySourceError.feature b/src/e2e-test/features/bigquery/source/BigQuerySourceError.feature index 8fcfcec44..873c4495e 100644 --- a/src/e2e-test/features/bigquery/source/BigQuerySourceError.feature +++ b/src/e2e-test/features/bigquery/source/BigQuerySourceError.feature @@ -19,6 +19,7 @@ Feature: BigQuery source - Validate BigQuery source plugin error scenarios When Source is BigQuery Then Open BigQuery source properties Then Enter BigQuery property reference name + Then Override Service account details if set in environment variables Then Enter the BigQuery source properties with incorrect property "" value "" Then Validate BigQuery source incorrect property error for table "" value "" Examples: @@ -50,6 +51,7 @@ Feature: BigQuery source - Validate BigQuery source plugin error scenarios Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery source property table name Then Enter BigQuery property temporary bucket name "bqInvalidTemporaryBucket" diff --git a/src/e2e-test/features/bigquery/source/BigQueryToBigQuery.feature b/src/e2e-test/features/bigquery/source/BigQueryToBigQuery.feature index b5e729355..d5c9aafbc 100644 --- a/src/e2e-test/features/bigquery/source/BigQueryToBigQuery.feature +++ b/src/e2e-test/features/bigquery/source/BigQueryToBigQuery.feature @@ -10,6 +10,7 @@ Feature: BigQuery source - Verification of BigQuery to BigQuery successful data Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery source property table name Then Enter BigQuery source property filter "bqFilter" @@ -17,6 +18,7 @@ Feature: BigQuery source - Verification of BigQuery to BigQuery successful data Then Validate "BigQuery" plugin properties Then Close the BigQuery properties Then Open BigQuery sink properties + Then Override Service account details if set in environment variables Then Enter the BigQuery sink mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties @@ -44,6 +46,7 @@ Feature: BigQuery source - Verification of BigQuery to BigQuery successful data Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery source property table name Then Enter BigQuery source property filter "bqInvalidFilter" @@ -51,6 +54,7 @@ Feature: BigQuery source - Verification of BigQuery to BigQuery successful data Then Validate "BigQuery" plugin properties Then Close the BigQuery properties Then Open BigQuery sink properties + Then Override Service account details if set in environment variables Then Enter the BigQuery sink mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties @@ -68,6 +72,7 @@ Feature: BigQuery source - Verification of BigQuery to BigQuery successful data Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery source property table name Then Enter BigQuery source properties partitionStartDate and partitionEndDate @@ -75,6 +80,7 @@ Feature: BigQuery source - Verification of BigQuery to BigQuery successful data Then Validate "BigQuery" plugin properties Then Close the BigQuery properties Then Open BigQuery sink properties + Then Override Service account details if set in environment variables Then Enter the BigQuery sink mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties @@ -102,6 +108,7 @@ Feature: BigQuery source - Verification of BigQuery to BigQuery successful data Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery source property table name Then Enter BigQuery source properties partitionStartDate "bqFuturePartitionStartDate" and partitionEndDate "bqFuturePartitionEndDate" @@ -109,6 +116,7 @@ Feature: BigQuery source - Verification of BigQuery to BigQuery successful data Then Validate "BigQuery" plugin properties Then Close the BigQuery properties Then Open BigQuery sink properties + Then Override Service account details if set in environment variables Then Enter the BigQuery sink mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties @@ -130,6 +138,7 @@ Feature: BigQuery source - Verification of BigQuery to BigQuery successful data Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery source property table name Then Validate output schema with expectedSchema "bqSourceSchemaDatatype" @@ -139,6 +148,7 @@ Feature: BigQuery source - Verification of BigQuery to BigQuery successful data Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery sink property table name Then Toggle BigQuery sink property truncateTable to true @@ -167,6 +177,7 @@ Feature: BigQuery source - Verification of BigQuery to BigQuery successful data Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery source property table name Then Validate output schema with expectedSchema "bqSourceSchemaDatatype" @@ -176,6 +187,7 @@ Feature: BigQuery source - Verification of BigQuery to BigQuery successful data Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery sink property table name Then Toggle BigQuery sink property truncateTable to true diff --git a/src/e2e-test/features/bigquery/source/BigQueryToGCS.feature b/src/e2e-test/features/bigquery/source/BigQueryToGCS.feature index de2ed47d1..c7a1bed32 100644 --- a/src/e2e-test/features/bigquery/source/BigQueryToGCS.feature +++ b/src/e2e-test/features/bigquery/source/BigQueryToGCS.feature @@ -12,11 +12,13 @@ Feature: BigQuery source - Verification of BigQuery to GCS successful data trans Then Enter BigQuery property datasetProjectId "projectId" Then Enter BigQuery property dataset "dataset" Then Enter BigQuery source property table name + Then Override Service account details if set in environment variables Then Enter BiqQuery property encryption key name "cmekBQ" if cmek is enabled Then Validate output schema with expectedSchema "bqSourceSchema" Then Validate "BigQuery" plugin properties Then Close the BigQuery properties Then Open GCS sink properties + Then Override Service account details if set in environment variables Then Enter the GCS sink mandatory properties Then Validate "GCS" plugin properties Then Close the GCS properties @@ -43,12 +45,14 @@ Feature: BigQuery source - Verification of BigQuery to GCS successful data trans Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery source property table name Then Validate output schema with expectedSchema "bqSourceSchemaDatatype" Then Validate "BigQuery" plugin properties Then Close the BigQuery properties Then Open GCS sink properties + Then Override Service account details if set in environment variables Then Enter the GCS sink mandatory properties Then Validate "GCS" plugin properties Then Close the GCS properties @@ -75,6 +79,7 @@ Feature: BigQuery source - Verification of BigQuery to GCS successful data trans Then Enter BigQuery property reference name Then Enter BigQuery property projectId "projectId" Then Enter BigQuery property datasetProjectId "projectId" + Then Override Service account details if set in environment variables Then Enter BigQuery property dataset "dataset" Then Enter BigQuery source property table name as view Then Toggle BigQuery source property enable querying views to true @@ -84,6 +89,7 @@ Feature: BigQuery source - Verification of BigQuery to GCS successful data trans Then Validate "BigQuery" plugin properties Then Close the BigQuery properties Then Open GCS sink properties + Then Override Service account details if set in environment variables Then Enter the GCS sink mandatory properties Then Validate "GCS" plugin properties Then Close the GCS properties diff --git a/src/e2e-test/features/bigquery/source/BigQueryToGCS_WithMacro.feature b/src/e2e-test/features/bigquery/source/BigQueryToGCS_WithMacro.feature index 0a3f58ef0..2c52ac9ba 100644 --- a/src/e2e-test/features/bigquery/source/BigQueryToGCS_WithMacro.feature +++ b/src/e2e-test/features/bigquery/source/BigQueryToGCS_WithMacro.feature @@ -10,7 +10,9 @@ Feature: BigQuery source - Verification of BigQuery to GCS successful data trans Then Enter BigQuery property reference name Then Enter BigQuery property "projectId" as macro argument "bqProjectId" Then Enter BigQuery property "datasetProjectId" as macro argument "bqDatasetProjectId" - Then Enter BigQuery property "serviceAccountFilePath" as macro argument "bqServiceAccount" + Then Enter BigQuery property "serviceAccountType" as macro argument "serviceAccountType" + Then Enter BigQuery property "serviceAccountFilePath" as macro argument "serviceAccount" + Then Enter BigQuery property "serviceAccountJSON" as macro argument "serviceAccount" Then Enter BigQuery property "dataset" as macro argument "bqDataset" Then Enter BigQuery property "table" as macro argument "bqSourceTable" Then Validate "BigQuery" plugin properties @@ -18,7 +20,9 @@ Feature: BigQuery source - Verification of BigQuery to GCS successful data trans Then Open GCS sink properties Then Enter GCS property reference name Then Enter GCS property "projectId" as macro argument "gcsProjectId" - Then Enter GCS property "serviceAccountFilePath" as macro argument "gcsServiceAccount" + Then Enter GCS property "serviceAccountType" as macro argument "serviceAccountType" + Then Enter GCS property "serviceAccountFilePath" as macro argument "serviceAccount" + Then Enter GCS property "serviceAccountJSON" as macro argument "serviceAccount" Then Enter GCS property "path" as macro argument "gcsSinkPath" Then Enter GCS sink property "pathSuffix" as macro argument "gcsPathSuffix" Then Enter GCS property "format" as macro argument "gcsFormat" @@ -30,11 +34,11 @@ Feature: BigQuery source - Verification of BigQuery to GCS successful data trans Then Preview and run the pipeline Then Enter runtime argument value "projectId" for key "bqProjectId" Then Enter runtime argument value "projectId" for key "bqDatasetProjectId" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "bqServiceAccount" + Then Enter runtime argument value "serviceAccountType" for key "serviceAccountType" + Then Enter runtime argument value "serviceAccount" for key "serviceAccount" Then Enter runtime argument value "dataset" for key "bqDataset" Then Enter runtime argument value for BigQuery source table name key "bqSourceTable" Then Enter runtime argument value "projectId" for key "gcsProjectId" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "gcsServiceAccount" Then Enter runtime argument value for GCS sink property path key "gcsSinkPath" Then Enter runtime argument value "gcsPathDateSuffix" for key "gcsPathSuffix" Then Enter runtime argument value "csvFormat" for key "gcsFormat" @@ -47,11 +51,11 @@ Feature: BigQuery source - Verification of BigQuery to GCS successful data trans Then Run the Pipeline in Runtime Then Enter runtime argument value "projectId" for key "bqProjectId" Then Enter runtime argument value "projectId" for key "bqDatasetProjectId" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "bqServiceAccount" + Then Enter runtime argument value "serviceAccountType" for key "serviceAccountType" + Then Enter runtime argument value "serviceAccount" for key "serviceAccount" Then Enter runtime argument value "dataset" for key "bqDataset" Then Enter runtime argument value for BigQuery source table name key "bqSourceTable" Then Enter runtime argument value "projectId" for key "gcsProjectId" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "gcsServiceAccount" Then Enter runtime argument value for GCS sink property path key "gcsSinkPath" Then Enter runtime argument value "gcsPathDateSuffix" for key "gcsPathSuffix" Then Enter runtime argument value "csvFormat" for key "gcsFormat" diff --git a/src/e2e-test/features/bigquery/source/BigQueryToMultipleSinks.feature b/src/e2e-test/features/bigquery/source/BigQueryToMultipleSinks.feature index b08474e91..cdfc0178d 100644 --- a/src/e2e-test/features/bigquery/source/BigQueryToMultipleSinks.feature +++ b/src/e2e-test/features/bigquery/source/BigQueryToMultipleSinks.feature @@ -12,20 +12,24 @@ Feature: BigQuery source - Verification of BigQuery to Multiple sinks successful When Sink is PubSub Then Connect source as "BigQuery" having title "BigQuery" and sink3 as "GooglePublisher" having title "Pub/Sub" to establish connection Then Open BigQuery source properties + Then Override Service account details if set in environment variables Then Enter the BigQuery source mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties Then Open GCS sink properties + Then Override Service account details if set in environment variables Then Enter the GCS sink mandatory properties Then Enter GCS property encryption key name "cmekGCS" if cmek is enabled Then Validate "GCS" plugin properties Then Close the GCS properties Then Open BigQuery sink properties + Then Override Service account details if set in environment variables Then Enter the BigQuery sink mandatory properties Then Enter BiqQuery property encryption key name "cmekBQ" if cmek is enabled Then Validate "BigQuery" plugin properties Then Close the BigQuery properties Then Open the PubSub sink properties + Then Override Service account details if set in environment variables Then Enter the PubSub sink mandatory properties Then Enter PubSub sink property encryption key name "cmekPubSub" if cmek is enabled Then Validate "PubSub" plugin properties diff --git a/src/e2e-test/features/gcs/sink/GCSSink.feature b/src/e2e-test/features/gcs/sink/GCSSink.feature index f9f6b63cf..b0f079433 100644 --- a/src/e2e-test/features/gcs/sink/GCSSink.feature +++ b/src/e2e-test/features/gcs/sink/GCSSink.feature @@ -8,11 +8,13 @@ Feature: GCS sink - Verification of GCS Sink plugin When Sink is GCS Then Connect source as "BigQuery" and sink as "GCS" to establish connection Then Open BigQuery source properties + Then Override Service account details if set in environment variables Then Enter the BigQuery source mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties Then Open GCS sink properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS sink property path Then Select GCS property format "csv" Then Enter GCS property encryption key name "cmekGCS" if cmek is enabled diff --git a/src/e2e-test/features/gcs/source/GCSSourceError.feature b/src/e2e-test/features/gcs/source/GCSSourceError.feature index a21a0978c..e61fa1281 100644 --- a/src/e2e-test/features/gcs/source/GCSSourceError.feature +++ b/src/e2e-test/features/gcs/source/GCSSourceError.feature @@ -18,6 +18,7 @@ Feature: GCS source - Verify GCS Source plugin error scenarios When Source is GCS Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS property path "gcsInvalidBucketName" Then Select GCS property format "csv" Then Verify invalid bucket name error message is displayed for bucket "gcsInvalidBucketName" @@ -28,6 +29,7 @@ Feature: GCS source - Verify GCS Source plugin error scenarios When Source is GCS Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "gcsOutputFieldTestFile" Then Select GCS property format "csv" Then Enter GCS source property path field "gcsInvalidPathField" @@ -39,6 +41,7 @@ Feature: GCS source - Verify GCS Source plugin error scenarios When Source is GCS Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "gcsOutputFieldTestFile" Then Select GCS property format "csv" Then Enter GCS source property override field "gcsInvalidOverrideField" and data type "gcsOverrideDataType" @@ -53,6 +56,7 @@ Feature: GCS source - Verify GCS Source plugin error scenarios Then Connect source as "GCS" and sink as "BigQuery" to establish connection Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "gcsCsvFile" Then Select GCS property format "csv" Then Enter GCS source property regex path filter "gcsIncorrectRegexPath" @@ -61,6 +65,7 @@ Feature: GCS source - Verify GCS Source plugin error scenarios Then Validate "GCS" plugin properties Then Close the GCS properties Then Open BigQuery sink properties + Then Override Service account details if set in environment variables Then Enter the BigQuery sink mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties diff --git a/src/e2e-test/features/gcs/source/GCSSourceToBigQuery.feature b/src/e2e-test/features/gcs/source/GCSSourceToBigQuery.feature index 7853d96e0..75c3390ac 100644 --- a/src/e2e-test/features/gcs/source/GCSSourceToBigQuery.feature +++ b/src/e2e-test/features/gcs/source/GCSSourceToBigQuery.feature @@ -9,6 +9,7 @@ Feature: GCS source - Verification of GCS to BQ successful data transfer Then Connect source as "GCS" and sink as "BigQuery" to establish connection Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "gcsCsvFile" Then Toggle GCS source property skip header to true Then Select GCS property format "csv" @@ -16,6 +17,7 @@ Feature: GCS source - Verification of GCS to BQ successful data transfer Then Validate "GCS" plugin properties Then Close the GCS properties Then Open BigQuery sink properties + Then Override Service account details if set in environment variables Then Enter the BigQuery sink mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties @@ -40,6 +42,7 @@ Feature: GCS source - Verification of GCS to BQ successful data transfer Then Connect source as "GCS" and sink as "BigQuery" to establish connection Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "gcsOutputFieldTestFile" Then Select GCS property format "csv" Then Enter GCS source property path field "gcsPathField" @@ -48,6 +51,7 @@ Feature: GCS source - Verification of GCS to BQ successful data transfer Then Validate "GCS" plugin properties Then Close the GCS properties Then Open BigQuery sink properties + Then Override Service account details if set in environment variables Then Enter the BigQuery sink mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties @@ -67,6 +71,7 @@ Feature: GCS source - Verification of GCS to BQ successful data transfer Then Connect source as "GCS" and sink as "BigQuery" to establish connection Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "gcsOutputFieldTestFile" Then Select GCS property format "csv" Then Enter GCS source property override field "gcsOverrideField" and data type "gcsOverrideDataType" @@ -75,6 +80,7 @@ Feature: GCS source - Verification of GCS to BQ successful data transfer Then Validate "GCS" plugin properties Then Close the GCS properties Then Open BigQuery sink properties + Then Override Service account details if set in environment variables Then Enter the BigQuery sink mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties @@ -94,6 +100,7 @@ Feature: GCS source - Verification of GCS to BQ successful data transfer Then Connect source as "GCS" and sink as "BigQuery" to establish connection Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "gcsDelimitedFile" Then Select GCS property format "delimited" Then Enter GCS property delimiter "gcsDelimiter" @@ -102,6 +109,7 @@ Feature: GCS source - Verification of GCS to BQ successful data transfer Then Validate "GCS" plugin properties Then Close the GCS properties Then Open BigQuery sink properties + Then Override Service account details if set in environment variables Then Enter the BigQuery sink mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties @@ -120,6 +128,7 @@ Feature: GCS source - Verification of GCS to BQ successful data transfer Then Connect source as "GCS" and sink as "BigQuery" to establish connection Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "gcsBlobFile" Then Select GCS property format "blob" Then Enter GCS source property minimum split size "gcsMinSplitSize" and maximum split size "gcsMaxSplitSize" @@ -128,6 +137,7 @@ Feature: GCS source - Verification of GCS to BQ successful data transfer Then Validate "GCS" plugin properties Then Close the GCS properties Then Open BigQuery sink properties + Then Override Service account details if set in environment variables Then Enter the BigQuery sink mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties @@ -146,6 +156,7 @@ Feature: GCS source - Verification of GCS to BQ successful data transfer Then Connect source as "GCS" and sink as "BigQuery" to establish connection Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "gcsCsvFile" Then Select GCS property format "csv" Then Enter GCS source property regex path filter "gcsRegexPathFilter" @@ -154,6 +165,7 @@ Feature: GCS source - Verification of GCS to BQ successful data transfer Then Validate "GCS" plugin properties Then Close the GCS properties Then Open BigQuery sink properties + Then Override Service account details if set in environment variables Then Enter the BigQuery sink mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties diff --git a/src/e2e-test/features/gcs/source/GCSourceSchema.feature b/src/e2e-test/features/gcs/source/GCSourceSchema.feature index e4a279470..fd9f2309e 100644 --- a/src/e2e-test/features/gcs/source/GCSourceSchema.feature +++ b/src/e2e-test/features/gcs/source/GCSourceSchema.feature @@ -6,6 +6,7 @@ Feature: GCS source - Validate GCS plugin output schema for different formats When Source is GCS Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "" Then Select GCS property format "" Then Toggle GCS source property skip header to true @@ -33,6 +34,7 @@ Feature: GCS source - Validate GCS plugin output schema for different formats When Source is GCS Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "" Then Select GCS property format "" Then Enter GCS property delimiter "" @@ -50,6 +52,7 @@ Feature: GCS source - Validate GCS plugin output schema for different formats When Source is GCS Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "gcsDelimitedFile" Then Select GCS property format "delimited" Then Toggle GCS source property skip header to true diff --git a/src/e2e-test/features/gcsdelete/GCSDelete.feature b/src/e2e-test/features/gcsdelete/GCSDelete.feature index 1bc567f03..e54b02c74 100644 --- a/src/e2e-test/features/gcsdelete/GCSDelete.feature +++ b/src/e2e-test/features/gcsdelete/GCSDelete.feature @@ -9,6 +9,7 @@ Feature: GCS Delete - Verification of GCS Delete plugin When Navigate to the properties page of plugin: "GCS Delete" Then Enter the GCS Delete property projectId "projectId" Then Enter the GCS Delete property objects to delete as bucketName + Then Override Service account details if set in environment variables Then Validate "GCS Delete" plugin properties Then Close the GCS Delete properties Then Save and Deploy Pipeline @@ -26,6 +27,7 @@ Feature: GCS Delete - Verification of GCS Delete plugin When Navigate to the properties page of plugin: "GCS Delete" Then Enter the GCS Delete property projectId "projectId" Then Enter the GCS Delete property objects to delete as path "gcsCsvFile" + Then Override Service account details if set in environment variables Then Validate "GCS Delete" plugin properties Then Close the GCS Delete properties Then Save and Deploy Pipeline @@ -43,6 +45,7 @@ Feature: GCS Delete - Verification of GCS Delete plugin When Navigate to the properties page of plugin: "GCS Delete" Then Enter the GCS Delete property projectId "projectId" Then Enter the GCS Delete property objects to delete as list of objects "gcsDeleteObjectsList" + Then Override Service account details if set in environment variables Then Validate "GCS Delete" plugin properties Then Close the GCS Delete properties Then Save and Deploy Pipeline @@ -65,4 +68,5 @@ Feature: GCS Delete - Verification of GCS Delete plugin When Select plugin: "GCS Delete" from the plugins list as: "Conditions and Actions" When Navigate to the properties page of plugin: "GCS Delete" Then Enter the GCS Delete property objects to delete as path "gcsInvalidBucketName" + Then Override Service account details if set in environment variables Then Verify invalid bucket name error message is displayed for GCS Delete objects to delete path "gcsInvalidBucketName" diff --git a/src/e2e-test/features/gcsdelete/GCSDelete_WithMacro.feature b/src/e2e-test/features/gcsdelete/GCSDelete_WithMacro.feature index 49110fcdb..7d9622a6f 100644 --- a/src/e2e-test/features/gcsdelete/GCSDelete_WithMacro.feature +++ b/src/e2e-test/features/gcsdelete/GCSDelete_WithMacro.feature @@ -9,14 +9,17 @@ Feature: GCS Delete - Verification of GCS Delete plugin with macro arguments When Navigate to the properties page of plugin: "GCS Delete" Then Enter the GCS Delete property "projectId" as macro argument "gcsDeleteProjectId" Then Enter the GCS Delete property "objectsToDelete" as macro argument "gcsDeleteObjectsToDelete" - Then Enter the GCS Delete property "serviceAccountFilePath" as macro argument "gcsServiceAccount" + Then Enter the GCS Delete property "serviceAccountType" as macro argument "serviceAccountType" + Then Enter the GCS Delete property "serviceAccountFilePath" as macro argument "serviceAccount" + Then Enter the GCS Delete property "serviceAccountJSON" as macro argument "serviceAccount" Then Validate "GCS Delete" plugin properties Then Close the GCS Delete properties Then Save and Deploy Pipeline Then Run the Pipeline in Runtime Then Enter runtime argument value "projectId" for key "gcsDeleteProjectId" Then Enter runtime argument value source bucket name for key "gcsDeleteObjectsToDelete" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "gcsServiceAccount" + Then Enter runtime argument value "serviceAccountType" for key "serviceAccountType" + Then Enter runtime argument value "serviceAccount" for key "serviceAccount" Then Run the Pipeline in Runtime with runtime arguments Then Wait till pipeline is in running state Then Open and capture logs @@ -31,14 +34,17 @@ Feature: GCS Delete - Verification of GCS Delete plugin with macro arguments When Navigate to the properties page of plugin: "GCS Delete" Then Enter the GCS Delete property "projectId" as macro argument "gcsDeleteProjectId" Then Enter the GCS Delete property "objectsToDelete" as macro argument "gcsDeleteObjectsToDelete" - Then Enter the GCS Delete property "serviceAccountFilePath" as macro argument "gcsServiceAccount" + Then Enter the GCS Delete property "serviceAccountType" as macro argument "serviceAccountType" + Then Enter the GCS Delete property "serviceAccountFilePath" as macro argument "serviceAccount" + Then Enter the GCS Delete property "serviceAccountJSON" as macro argument "serviceAccount" Then Validate "GCS Delete" plugin properties Then Close the GCS Delete properties Then Save and Deploy Pipeline Then Run the Pipeline in Runtime Then Enter runtime argument value "projectId" for key "gcsDeleteProjectId" Then Enter runtime argument value "gcsDeleteObjectsList" as comma separated objects for key "gcsDeleteObjectsToDelete" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "gcsServiceAccount" + Then Enter runtime argument value "serviceAccountType" for key "serviceAccountType" + Then Enter runtime argument value "serviceAccount" for key "serviceAccount" Then Run the Pipeline in Runtime with runtime arguments Then Wait till pipeline is in running state Then Open and capture logs diff --git a/src/e2e-test/features/gcsmove/GCSMove.feature b/src/e2e-test/features/gcsmove/GCSMove.feature index 1fdf19f33..91a4b0e74 100644 --- a/src/e2e-test/features/gcsmove/GCSMove.feature +++ b/src/e2e-test/features/gcsmove/GCSMove.feature @@ -10,6 +10,7 @@ Feature:GCSMove - Verification of successful objects move from one bucket to ano Then Enter GCSMove property projectId "projectId" Then Enter GCSMove property source path "gcsCsvFile" Then Enter GCSMove property destination path + Then Override Service account details if set in environment variables Then Enter GCSMove property encryption key name "cmekGCS" if cmek is enabled Then Validate "GCSMove" plugin properties Then Close GCSMove properties @@ -31,6 +32,7 @@ Feature:GCSMove - Verification of successful objects move from one bucket to ano Then Enter GCSMove property source path "gcsMoveReadRecursivePath" Then Enter GCSMove property destination path Then Select GCSMove property move all subdirectories as "true" + Then Override Service account details if set in environment variables Then Validate "GCSMove" plugin properties Then Close GCSMove properties Then Save and Deploy Pipeline @@ -50,6 +52,7 @@ Feature:GCSMove - Verification of successful objects move from one bucket to ano Then Enter GCSMove property source path "gcsMoveReadRecursivePath" Then Enter GCSMove property destination path Then Select GCSMove property move all subdirectories as "false" + Then Override Service account details if set in environment variables Then Validate "GCSMove" plugin properties Then Close GCSMove properties Then Save and Deploy Pipeline @@ -69,6 +72,7 @@ Feature:GCSMove - Verification of successful objects move from one bucket to ano Then Enter GCSMove property source path "gcsCsvFile" Then Enter GCSMove property destination path "gcsMoveReadRecursivePath" Then Select GCSMove property overwrite existing files as "true" + Then Override Service account details if set in environment variables Then Validate "GCSMove" plugin properties Then Close GCSMove properties Then Save and Deploy Pipeline @@ -88,6 +92,7 @@ Feature:GCSMove - Verification of successful objects move from one bucket to ano Then Enter GCSMove property source path "gcsCsvFile" Then Enter GCSMove property destination path "gcsMoveReadRecursivePath" Then Select GCSMove property overwrite existing files as "false" + Then Override Service account details if set in environment variables Then Validate "GCSMove" plugin properties Then Close GCSMove properties Then Save and Deploy Pipeline @@ -106,6 +111,7 @@ Feature:GCSMove - Verification of successful objects move from one bucket to ano Then Enter GCSMove property projectId "projectId" Then Enter GCSMove property source path "gcsCsvFile" Then Enter GCSMove property destination path + Then Override Service account details if set in environment variables Then Enter GCSMove property location "locationEU" Then Validate "GCSMove" plugin properties Then Close GCSMove properties diff --git a/src/e2e-test/features/gcsmove/GCSMove_WithMacro.feature b/src/e2e-test/features/gcsmove/GCSMove_WithMacro.feature index 47af3625b..7b7a73640 100644 --- a/src/e2e-test/features/gcsmove/GCSMove_WithMacro.feature +++ b/src/e2e-test/features/gcsmove/GCSMove_WithMacro.feature @@ -10,7 +10,9 @@ Feature:GCSMove - Verification of successful objects move from one bucket to ano Then Enter GCSMove property "projectId" as macro argument "gcsMoveProjectId" Then Enter GCSMove property "gcsMoveSourcePath" as macro argument "gcsMoveSourcePath" Then Enter GCSMove property "gcsMoveDestinationPath" as macro argument "gcsMoveDestinationPath" - Then Enter GCSMove property "serviceAccountFilePath" as macro argument "gcsMoveServiceAccount" + Then Enter GCSMove property "serviceAccountType" as macro argument "serviceAccountType" + Then Enter GCSMove property "serviceAccountFilePath" as macro argument "serviceAccount" + Then Enter GCSMove property "serviceAccountJSON" as macro argument "serviceAccount" Then Enter GCSMove cmek property "encryptionKeyName" as macro argument "cmekGCS" if cmek is enabled Then Validate "GCSMove" plugin properties Then Close GCSMove properties @@ -19,7 +21,8 @@ Feature:GCSMove - Verification of successful objects move from one bucket to ano Then Enter runtime argument value "projectId" for key "gcsMoveProjectId" Then Enter runtime argument value "gcsCsvFile" for GCSMove property sourcePath key "gcsMoveSourcePath" Then Enter runtime argument value for GCSMove property destination path key "gcsMoveDestinationPath" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "gcsMoveServiceAccount" + Then Enter runtime argument value "serviceAccountType" for key "serviceAccountType" + Then Enter runtime argument value "serviceAccount" for key "serviceAccount" Then Enter runtime argument value "cmekGCS" for GCSMove cmek property key "cmekGCS" if GCS cmek is enabled Then Run the Pipeline in Runtime with runtime arguments Then Wait till pipeline is in running state diff --git a/src/e2e-test/features/pubsub/sink/BQToPubSub.feature b/src/e2e-test/features/pubsub/sink/BQToPubSub.feature index 947221912..8a1b5aeab 100644 --- a/src/e2e-test/features/pubsub/sink/BQToPubSub.feature +++ b/src/e2e-test/features/pubsub/sink/BQToPubSub.feature @@ -8,11 +8,13 @@ Feature: PubSub-Sink - Verification of BigQuery to PubSub successful data transf When Sink is PubSub Then Connect source as "BigQuery" and sink as "GooglePublisher" to establish connection Then Open BigQuery source properties + Then Override Service account details if set in environment variables Then Enter the BigQuery source mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties Then Open the PubSub sink properties Then Enter PubSub property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter PubSub property reference name Then Enter PubSub sink property topic name Then Select PubSub property format "csv" diff --git a/src/e2e-test/features/pubsub/sink/GCSToPubSub.feature b/src/e2e-test/features/pubsub/sink/GCSToPubSub.feature index 0227b47fc..462bd7d91 100644 --- a/src/e2e-test/features/pubsub/sink/GCSToPubSub.feature +++ b/src/e2e-test/features/pubsub/sink/GCSToPubSub.feature @@ -7,11 +7,13 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer When Source is GCS When Sink is PubSub Then Open GCS source properties + Then Override Service account details if set in environment variables Then Enter the GCS source mandatory properties Then Validate "GCS" plugin properties Then Close the GCS properties Then Open the PubSub sink properties Then Enter PubSub property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter PubSub property reference name Then Enter PubSub sink property topic name Then Select PubSub property format "csv" @@ -45,11 +47,13 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer When Source is GCS When Sink is PubSub Then Open GCS source properties + Then Override Service account details if set in environment variables Then Enter the GCS source mandatory properties Then Validate "GCS" plugin properties Then Close the GCS properties Then Open the PubSub sink properties Then Enter PubSub property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter PubSub property reference name Then Enter PubSub sink property topic name Then Select PubSub property format "" @@ -91,6 +95,7 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer Then Connect source as "GCS" and sink as "GooglePublisher" to establish connection Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "gcsDataTypeTest1File" Then Toggle GCS source property skip header to true Then Select GCS property format "csv" @@ -99,6 +104,7 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer Then Close the GCS properties Then Open the PubSub sink properties Then Enter PubSub property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter PubSub property reference name Then Enter PubSub sink property topic name Then Select PubSub property format "csv" @@ -126,6 +132,7 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer Then Connect source as "GCS" and sink as "GooglePublisher" to establish connection Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "gcsDataTypeTest2File" Then Toggle GCS source property skip header to true Then Select GCS property format "csv" @@ -134,6 +141,7 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer Then Close the GCS properties Then Open the PubSub sink properties Then Enter PubSub property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter PubSub property reference name Then Enter PubSub sink property topic name Then Select PubSub property format "csv" @@ -160,6 +168,7 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer Then Connect source as "GCS" and sink as "GooglePublisher" to establish connection Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "" Then Toggle GCS source property skip header to true Then Select GCS property format "" @@ -168,6 +177,7 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer Then Close the GCS properties Then Open the PubSub sink properties Then Enter PubSub property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter PubSub property reference name Then Enter PubSub sink property topic name Then Select PubSub property format "" @@ -218,6 +228,7 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer Then Connect source as "GCS" and sink as "GooglePublisher" to establish connection Then Open GCS source properties Then Enter GCS property projectId and reference name + Then Override Service account details if set in environment variables Then Enter GCS source property path "" Then Toggle GCS source property skip header to true Then Select GCS property format "" @@ -226,6 +237,7 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer Then Close the GCS properties Then Open the PubSub sink properties Then Enter PubSub property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter PubSub property reference name Then Enter PubSub sink property topic name Then Select PubSub property format "" diff --git a/src/e2e-test/features/pubsub/sink/GCSToPubSub_WithMacro.feature b/src/e2e-test/features/pubsub/sink/GCSToPubSub_WithMacro.feature index 984b829f9..d3a15f490 100644 --- a/src/e2e-test/features/pubsub/sink/GCSToPubSub_WithMacro.feature +++ b/src/e2e-test/features/pubsub/sink/GCSToPubSub_WithMacro.feature @@ -9,7 +9,9 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer wi Then Open GCS source properties Then Enter GCS property reference name Then Enter GCS property "projectId" as macro argument "gcsProjectId" - Then Enter GCS property "serviceAccountFilePath" as macro argument "gcsServiceAccount" + Then Enter GCS property "serviceAccountType" as macro argument "serviceAccountType" + Then Enter GCS property "serviceAccountFilePath" as macro argument "serviceAccount" + Then Enter GCS property "serviceAccountJSON" as macro argument "serviceAccount" Then Enter GCS property "path" as macro argument "gcsSourcePath" Then Enter GCS source property "skipHeader" as macro argument "gcsSkipHeader" Then Enter GCS property "format" as macro argument "gcsFormat" @@ -19,7 +21,9 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer wi Then Open the PubSub sink properties Then Enter PubSub property reference name Then Enter PubSub property "projectId" as macro argument "PubSubProjectId" - Then Enter PubSub property "serviceAccountFilePath" as macro argument "PubSubServiceAccount" + Then Enter PubSub property "serviceAccountType" as macro argument "serviceAccountType" + Then Enter PubSub property "serviceAccountFilePath" as macro argument "serviceAccount" + Then Enter PubSub property "serviceAccountJSON" as macro argument "serviceAccount" Then Enter PubSub property "topic" as macro argument "PubSubSinkTopic" Then Enter PubSub property "format" as macro argument "PubSubFormat" Then Enter PubSub sink cmek property "encryptionKeyName" as macro argument "cmekPubSub" if cmek is enabled @@ -29,12 +33,12 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer wi Then Save the pipeline Then Preview and run the pipeline Then Enter runtime argument value "projectId" for key "PubSubProjectId" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "PubSubServiceAccount" + Then Enter runtime argument value "serviceAccountType" for key "serviceAccountType" + Then Enter runtime argument value "serviceAccount" for key "serviceAccount" Then Enter runtime argument value for PubSub sink property topic key "PubSubSinkTopic" Then Enter runtime argument value "csvFormat" for key "PubSubFormat" Then Enter runtime argument value "cmekPubSub" for PubSub Sink cmek property key "cmekPubSub" if PubSub Sink cmek is enabled Then Enter runtime argument value "projectId" for key "gcsProjectId" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "gcsServiceAccount" Then Enter runtime argument value "gcsCsvFile" for GCS source property path key "gcsSourcePath" Then Enter runtime argument value "gcsSkipHeaderTrue" for key "gcsSkipHeader" Then Enter runtime argument value "csvFormat" for key "gcsFormat" @@ -48,12 +52,12 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer wi Then Deploy the pipeline Then Run the Pipeline in Runtime Then Enter runtime argument value "projectId" for key "PubSubProjectId" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "PubSubServiceAccount" + Then Enter runtime argument value "serviceAccountType" for key "serviceAccountType" + Then Enter runtime argument value "serviceAccount" for key "serviceAccount" Then Enter runtime argument value for PubSub sink property topic key "PubSubSinkTopic" Then Enter runtime argument value "csvFormat" for key "PubSubFormat" Then Enter runtime argument value "cmekPubSub" for PubSub Sink cmek property key "cmekPubSub" if PubSub Sink cmek is enabled Then Enter runtime argument value "projectId" for key "gcsProjectId" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "gcsServiceAccount" Then Enter runtime argument value "gcsCsvFile" for GCS source property path key "gcsSourcePath" Then Enter runtime argument value "gcsSkipHeaderTrue" for key "gcsSkipHeader" Then Enter runtime argument value "csvFormat" for key "gcsFormat" @@ -71,7 +75,9 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer wi Then Open GCS source properties Then Enter GCS property reference name Then Enter GCS property "projectId" as macro argument "gcsProjectId" - Then Enter GCS property "serviceAccountFilePath" as macro argument "gcsServiceAccount" + Then Enter GCS property "serviceAccountType" as macro argument "serviceAccountType" + Then Enter GCS property "serviceAccountFilePath" as macro argument "serviceAccount" + Then Enter GCS property "serviceAccountJSON" as macro argument "serviceAccount" Then Enter GCS property "path" as macro argument "gcsSourcePath" Then Enter GCS source property "skipHeader" as macro argument "gcsSkipHeader" Then Enter GCS property "format" as macro argument "gcsFormat" @@ -81,7 +87,9 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer wi Then Open the PubSub sink properties Then Enter PubSub property reference name Then Enter PubSub property "projectId" as macro argument "PubSubProjectId" - Then Enter PubSub property "serviceAccountFilePath" as macro argument "PubSubServiceAccount" + Then Enter PubSub property "serviceAccountType" as macro argument "serviceAccountType" + Then Enter PubSub property "serviceAccountFilePath" as macro argument "serviceAccount" + Then Enter PubSub property "serviceAccountJSON" as macro argument "serviceAccount" Then Enter PubSub property "topic" as macro argument "PubSubSinkTopic" Then Enter PubSub property "format" as macro argument "PubSubFormat" Then Enter PubSub property "maximumBatchCount" as macro argument "maximumBatchCount" @@ -95,7 +103,8 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer wi Then Save the pipeline Then Preview and run the pipeline Then Enter runtime argument value "projectId" for key "PubSubProjectId" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "PubSubServiceAccount" + Then Enter runtime argument value "serviceAccountType" for key "serviceAccountType" + Then Enter runtime argument value "serviceAccount" for key "serviceAccount" Then Enter runtime argument value for PubSub sink property topic key "PubSubSinkTopic" Then Enter runtime argument value "csvFormat" for key "PubSubFormat" Then Enter runtime argument value "pubSubMaximumBatchCount" for key "maximumBatchCount" @@ -103,7 +112,6 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer wi Then Enter runtime argument value "pubSubPublishDelayThreshold" for key "publishDelayThreshold" Then Enter runtime argument value "pubSubRetryTimeOut" for key "retryTimeout" Then Enter runtime argument value "pubSubErrorThreshold" for key "errorThreshold" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "gcsServiceAccount" Then Enter runtime argument value "projectId" for key "gcsProjectId" Then Enter runtime argument value "gcsCsvFile" for GCS source property path key "gcsSourcePath" Then Enter runtime argument value "gcsSkipHeaderTrue" for key "gcsSkipHeader" @@ -118,7 +126,8 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer wi Then Deploy the pipeline Then Run the Pipeline in Runtime Then Enter runtime argument value "projectId" for key "PubSubProjectId" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "PubSubServiceAccount" + Then Enter runtime argument value "serviceAccountType" for key "serviceAccountType" + Then Enter runtime argument value "serviceAccount" for key "serviceAccount" Then Enter runtime argument value for PubSub sink property topic key "PubSubSinkTopic" Then Enter runtime argument value "csvFormat" for key "PubSubFormat" Then Enter runtime argument value "pubSubMaximumBatchCount" for key "maximumBatchCount" @@ -126,7 +135,6 @@ Feature: PubSub-Sink - Verification of GCS to PubSub successful data transfer wi Then Enter runtime argument value "pubSubPublishDelayThreshold" for key "publishDelayThreshold" Then Enter runtime argument value "pubSubRetryTimeOut" for key "retryTimeout" Then Enter runtime argument value "pubSubErrorThreshold" for key "errorThreshold" - Then Enter runtime argument value "serviceAccountAutoDetect" for key "gcsServiceAccount" Then Enter runtime argument value "projectId" for key "gcsProjectId" Then Enter runtime argument value "gcsCsvFile" for GCS source property path key "gcsSourcePath" Then Enter runtime argument value "gcsSkipHeaderTrue" for key "gcsSkipHeader" diff --git a/src/e2e-test/features/pubsub/sink/PubSubSinkErrorScnerios.feature b/src/e2e-test/features/pubsub/sink/PubSubSinkErrorScnerios.feature index 98be5d8c5..2511ab54b 100644 --- a/src/e2e-test/features/pubsub/sink/PubSubSinkErrorScnerios.feature +++ b/src/e2e-test/features/pubsub/sink/PubSubSinkErrorScnerios.feature @@ -18,6 +18,7 @@ Feature: PubSub-Sink - Verify PubSub sink plugin error Scenarios Then Open the PubSub sink properties Then Enter PubSub property reference name Then Enter PubSub property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter PubSub topic name "dummytopic" Then Select PubSub property format "csv" Then Enter the PubSub sink advanced properties with incorrect property "" @@ -36,6 +37,7 @@ Feature: PubSub-Sink - Verify PubSub sink plugin error Scenarios Then Open the PubSub sink properties Then Enter PubSub property reference name Then Enter PubSub property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter PubSub topic name "dummytopic" Then Select PubSub property format "csv" Then Enter the PubSub sink advanced properties with invalid number for property "" diff --git a/src/e2e-test/features/spanner/sink/BQToSpanner.feature b/src/e2e-test/features/spanner/sink/BQToSpanner.feature index 8606cdf86..e2b5bc030 100644 --- a/src/e2e-test/features/spanner/sink/BQToSpanner.feature +++ b/src/e2e-test/features/spanner/sink/BQToSpanner.feature @@ -8,12 +8,14 @@ Feature: Spanner Sink - Verification of BigQuery to Spanner data transfer When Sink is Spanner Then Connect source as "BigQuery" and sink as "Spanner" to establish connection Then Open BigQuery source properties + Then Override Service account details if set in environment variables Then Enter the BigQuery source mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties Then Open Spanner sink properties Then Enter Spanner property reference name Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter Spanner property InstanceId Then Enter Spanner sink property DatabaseName Then Enter Spanner sink property TableName @@ -38,12 +40,14 @@ Feature: Spanner Sink - Verification of BigQuery to Spanner data transfer When Sink is Spanner Then Connect source as "BigQuery" and sink as "Spanner" to establish connection Then Open BigQuery source properties + Then Override Service account details if set in environment variables Then Enter the BigQuery source mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties Then Open Spanner sink properties Then Enter Spanner property reference name Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter Spanner property InstanceId Then Enter Spanner sink property DatabaseName Then Enter Spanner sink property TableName diff --git a/src/e2e-test/features/spanner/sink/GCSToSpanner.feature b/src/e2e-test/features/spanner/sink/GCSToSpanner.feature index c47c5da63..7dac37693 100644 --- a/src/e2e-test/features/spanner/sink/GCSToSpanner.feature +++ b/src/e2e-test/features/spanner/sink/GCSToSpanner.feature @@ -8,12 +8,14 @@ Feature: Spanner Sink - Verification of GCS to Spanner data transfer When Sink is Spanner Then Connect source as "GCS" and sink as "Spanner" to establish connection Then Open GCS source properties + Then Override Service account details if set in environment variables Then Enter the GCS source mandatory properties Then Validate "GCS" plugin properties Then Close the GCS properties Then Open Spanner sink properties Then Enter Spanner property reference name Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter Spanner property InstanceId Then Enter Spanner sink property DatabaseName Then Enter Spanner sink property TableName @@ -37,12 +39,14 @@ Feature: Spanner Sink - Verification of GCS to Spanner data transfer When Sink is Spanner Then Connect source as "GCS" and sink as "Spanner" to establish connection Then Open GCS source properties + Then Override Service account details if set in environment variables Then Enter the GCS source mandatory properties Then Validate "GCS" plugin properties Then Close the GCS properties Then Open Spanner sink properties Then Enter Spanner property reference name Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter Spanner property InstanceId Then Enter Spanner sink property DatabaseName Then Enter Spanner sink property TableName diff --git a/src/e2e-test/features/spanner/source/SpannerSourceErrorScenarios.feature b/src/e2e-test/features/spanner/source/SpannerSourceErrorScenarios.feature index 978b6e3fe..8149a19b5 100644 --- a/src/e2e-test/features/spanner/source/SpannerSourceErrorScenarios.feature +++ b/src/e2e-test/features/spanner/source/SpannerSourceErrorScenarios.feature @@ -20,6 +20,7 @@ Feature: Spanner Source - Verify Spanner source plugin error scenarios Then Open Spanner source properties Then Enter Spanner property reference name Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter the Spanner properties with incorrect property "" Then Verify plugin properties validation fails with 1 error Examples: diff --git a/src/e2e-test/features/spanner/source/SpannertoBQ.feature b/src/e2e-test/features/spanner/source/SpannertoBQ.feature index 42acec235..79ef6bfc9 100644 --- a/src/e2e-test/features/spanner/source/SpannertoBQ.feature +++ b/src/e2e-test/features/spanner/source/SpannertoBQ.feature @@ -10,6 +10,7 @@ Feature: Spanner Source - Verification of Spanner to BigQuery successful data tr Then Open Spanner source properties Then Enter Spanner property reference name Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter Spanner property InstanceId Then Enter Spanner source property DatabaseName Then Enter Spanner source property TableName @@ -17,6 +18,7 @@ Feature: Spanner Source - Verification of Spanner to BigQuery successful data tr Then Validate "Spanner" plugin properties Then Close the Spanner properties Then Open BigQuery sink properties + Then Override Service account details if set in environment variables Then Enter the BigQuery sink mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties @@ -42,6 +44,7 @@ Feature: Spanner Source - Verification of Spanner to BigQuery successful data tr Then Open Spanner source properties Then Enter Spanner property reference name Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter Spanner property InstanceId Then Enter Spanner source property DatabaseName Then Enter Spanner source property TableName @@ -50,6 +53,7 @@ Feature: Spanner Source - Verification of Spanner to BigQuery successful data tr Then Validate "Spanner" plugin properties Then Close the Spanner properties Then Open BigQuery sink properties + Then Override Service account details if set in environment variables Then Enter the BigQuery sink mandatory properties Then Validate "BigQuery" plugin properties Then Close the BigQuery properties diff --git a/src/e2e-test/features/spanner/source/SpannertoGCS.feature b/src/e2e-test/features/spanner/source/SpannertoGCS.feature index d615cd58e..4665bfcbc 100644 --- a/src/e2e-test/features/spanner/source/SpannertoGCS.feature +++ b/src/e2e-test/features/spanner/source/SpannertoGCS.feature @@ -10,6 +10,7 @@ Feature: Spanner Source - Verification of Spanner to GCS successful data transfe Then Open Spanner source properties Then Enter Spanner property reference name Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter Spanner property InstanceId Then Enter Spanner source property DatabaseName Then Enter Spanner source property TableName @@ -17,6 +18,7 @@ Feature: Spanner Source - Verification of Spanner to GCS successful data transfe Then Validate "Spanner" plugin properties Then Close the Spanner properties Then Open GCS sink properties + Then Override Service account details if set in environment variables Then Enter the GCS sink mandatory properties Then Validate "GCS" plugin properties Then Close the GCS properties @@ -42,6 +44,7 @@ Feature: Spanner Source - Verification of Spanner to GCS successful data transfe Then Open Spanner source properties Then Enter Spanner property reference name Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables Then Enter Spanner property InstanceId Then Enter Spanner source property DatabaseName Then Enter Spanner source property TableName @@ -50,6 +53,7 @@ Feature: Spanner Source - Verification of Spanner to GCS successful data transfe Then Validate "Spanner" plugin properties Then Close the Spanner properties Then Open GCS sink properties + Then Override Service account details if set in environment variables Then Enter the GCS sink mandatory properties Then Validate "GCS" plugin properties Then Close the GCS properties diff --git a/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java index 9e9ea39da..533fa2d44 100644 --- a/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java +++ b/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java @@ -50,6 +50,7 @@ */ public class TestSetupHooks { + public static boolean beforeAllFlag = true; public static String gcsSourceBucketName = StringUtils.EMPTY; public static String gcsTargetBucketName = StringUtils.EMPTY; public static String bqTargetTable = StringUtils.EMPTY; @@ -63,6 +64,35 @@ public class TestSetupHooks { public static String spannerTargetTable = StringUtils.EMPTY; public static boolean firstSpannerTestFlag = true; + @Before(order = 1) + public static void overrideServiceAccountFilePathIfProvided() { + if (beforeAllFlag) { + String serviceAccountType = System.getenv("SERVICE_ACCOUNT_TYPE"); + if (serviceAccountType != null && !serviceAccountType.isEmpty()) { + if (serviceAccountType.equalsIgnoreCase("FilePath")) { + PluginPropertyUtils.addPluginProp("serviceAccountType", "filePath"); + String serviceAccountFilePath = System.getenv("SERVICE_ACCOUNT_FILE_PATH"); + if (!(serviceAccountFilePath == null) && !serviceAccountFilePath.equalsIgnoreCase("auto-detect")) { + PluginPropertyUtils.addPluginProp("serviceAccount", serviceAccountFilePath); + } + return; + } + if (serviceAccountType.equalsIgnoreCase("JSON")) { + PluginPropertyUtils.addPluginProp("serviceAccountType", "JSON"); + String serviceAccountJSON = System.getenv("SERVICE_ACCOUNT_JSON"); + if (!(serviceAccountJSON == null) && !serviceAccountJSON.equalsIgnoreCase("auto-detect")) { + PluginPropertyUtils.addPluginProp("serviceAccount", serviceAccountJSON); + } + return; + } + Assert.fail("ServiceAccount override failed - ServiceAccount type set in environment variable " + + "'SERVICE_ACCOUNT_TYPE' with invalid value: '" + serviceAccountType + "'. " + + "Value should be either 'FilePath' or 'JSON'"); + } + beforeAllFlag = false; + } + } + @Before(order = 1, value = "@GCS_CSV_TEST") public static void createBucketWithCSVFile() throws IOException, URISyntaxException { gcsSourceBucketName = createGCSBucketWithFile(PluginPropertyUtils.pluginProp("gcsCsvFile")); diff --git a/src/e2e-test/java/io/cdap/plugin/utils/CdfPluginPropertyLocator.java b/src/e2e-test/java/io/cdap/plugin/utils/CdfPluginPropertyLocator.java index 0225acc09..dbb7add2e 100644 --- a/src/e2e-test/java/io/cdap/plugin/utils/CdfPluginPropertyLocator.java +++ b/src/e2e-test/java/io/cdap/plugin/utils/CdfPluginPropertyLocator.java @@ -20,7 +20,9 @@ public enum CdfPluginPropertyLocator { SKIP_HEADER("skipHeader"), SUFFIX("suffix"), CMEK_KEY("cmekKey"), + SERVICE_ACCOUNT_TYPE("serviceAccountType"), SERVICE_ACCOUNT_PATH("serviceFilePath"), + SERVICE_ACCOUNT_JSON("serviceAccountJSON"), TRUNCATE_TABLE("truncateTable"), UPDATE_TABLE_SCHEMA("allowSchemaRelaxation"), PUBSUB_TOPIC("topic"), @@ -53,7 +55,9 @@ public enum CdfPluginPropertyLocator { .put("skipHeader", CdfPluginPropertyLocator.SKIP_HEADER) .put("pathSuffix", CdfPluginPropertyLocator.SUFFIX) .put("encryptionKeyName", CdfPluginPropertyLocator.CMEK_KEY) + .put("serviceAccountType", CdfPluginPropertyLocator.SERVICE_ACCOUNT_TYPE) .put("serviceAccountFilePath", CdfPluginPropertyLocator.SERVICE_ACCOUNT_PATH) + .put("serviceAccountJSON", CdfPluginPropertyLocator.SERVICE_ACCOUNT_JSON) .put("truncateTable", CdfPluginPropertyLocator.TRUNCATE_TABLE) .put("updateTableSchema", CdfPluginPropertyLocator.UPDATE_TABLE_SCHEMA) .put("topic", CdfPluginPropertyLocator.PUBSUB_TOPIC) diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index 95df026a9..0d5079c88 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -1,6 +1,7 @@ projectId=cdf-athena dataset=test_automation -serviceAccountAutoDetect=auto-detect +serviceAccountType=filePath +serviceAccount=auto-detect csvFormat=csv ## GCS-PLUGIN-PROPERTIES-START