diff --git a/.github/workflows/run-e2e-automation-tests.yml b/.github/workflows/run-e2e-automation-tests.yml index c97533c52..424b5208b 100644 --- a/.github/workflows/run-e2e-automation-tests.yml +++ b/.github/workflows/run-e2e-automation-tests.yml @@ -67,6 +67,7 @@ on: - functional - sandbox - proxy_smoke + - Batch_File_Validation_Feature env: APIGEE_AUTH_ENV: ${{ inputs.apigee_environment == 'int' && inputs.apigee_environment || 'internal-dev' }} diff --git a/tests/e2e_automation/features/APITests/search.feature b/tests/e2e_automation/features/APITests/search.feature index a437476d0..d038e60ae 100644 --- a/tests/e2e_automation/features/APITests/search.feature +++ b/tests/e2e_automation/features/APITests/search.feature @@ -272,14 +272,14 @@ Feature: Search the immunization of a patient @Delete_cleanUp @supplier_name_TPP Scenario: Verify that Search API returns immunization events when searching by target-disease (GET) Given Valid vaccination record is created with Patient 'Random' and vaccine_type 'MMRV' - When Send a search request with GET method using target-disease for Immunization event created + When Send a search request with 'GET' method using target-disease for Immunization event created Then The request will be successful with the status code '200' And The Search Response JSONs should contain the detail of the immunization events created above @Delete_cleanUp @supplier_name_EMIS Scenario: Verify that Search API returns immunization events when searching by target-disease (POST) Given Valid vaccination record is created with Patient 'Random' and vaccine_type '3IN1' - When Send a search request with POST method using target-disease for Immunization event created + When Send a search request with 'POST' method using target-disease for Immunization event created Then The request will be successful with the status code '200' And The Search Response JSONs should contain the detail of the immunization events created above @@ -304,14 +304,14 @@ Feature: Search the immunization of a patient @Delete_cleanUp @supplier_name_Postman_Auth Scenario: Verify that Search API returns immunization events when searching by comma-separated target-disease (GET) Given Valid vaccination record is created with Patient 'Random' and vaccine_type 'HEPB' - When Send a search request with GET method using comma-separated target-disease for Immunization event created + When Send a search request with 'GET' method using comma-separated target-disease for Immunization event created Then The request will be successful with the status code '200' And The Search Response JSONs should contain the detail of the immunization events created above @Delete_cleanUp @supplier_name_Postman_Auth Scenario: Verify that Search API returns immunization events when searching by comma-separated target-disease (POST) Given Valid vaccination record is created with Patient 'Random' and vaccine_type 'COVID' - When Send a search request with POST method using comma-separated target-disease for Immunization event created + When Send a search request with 'POST' method using comma-separated target-disease for Immunization event created Then The request will be successful with the status code '200' And The Search Response JSONs should contain the detail of the immunization events created above @@ -334,10 +334,10 @@ Feature: Search the immunization of a patient @supplier_name_Postman_Auth Scenario: Verify that Search API returns 400 when all target-disease values are invalid SNOMED codes - When Send a search request with GET method with valid NHS Number and all invalid target-disease codes + When Send a search request with 'GET' method with valid NHS Number and all invalid target-disease codes Then The request will be unsuccessful with the status code '400' And The Response JSONs should contain correct error message for invalid target-disease codes - When Send a search request with POST method with valid NHS Number and all invalid target-disease codes + When Send a search request with 'POST' method with valid NHS Number and all invalid target-disease codes Then The request will be unsuccessful with the status code '400' And The Response JSONs should contain correct error message for invalid target-disease codes @@ -345,9 +345,9 @@ Feature: Search the immunization of a patient @Delete_cleanUp @supplier_name_Postman_Auth Scenario: Verify that Search API returns 200 with results and OperationOutcome when some target-disease values are invalid Given Valid vaccination record is created with Patient 'Random' and vaccine_type '6IN1' - When Send a search request with GET method using mixed valid and invalid target-disease codes for Immunization event created + When Send a search request with 'GET' method using mixed valid and invalid target-disease codes for Immunization event created Then The request will be successful with the status code '200' And The Search Response should contain search results and OperationOutcome for invalid target-disease codes - When Send a search request with POST method using mixed valid and invalid target-disease codes for Immunization event created + When Send a search request with 'POST' method using mixed valid and invalid target-disease codes for Immunization event created Then The request will be successful with the status code '200' And The Search Response should contain search results and OperationOutcome for invalid target-disease codes diff --git a/tests/e2e_automation/features/APITests/steps/test_search_steps.py b/tests/e2e_automation/features/APITests/steps/test_search_steps.py index e3a0e593f..bba09f981 100644 --- a/tests/e2e_automation/features/APITests/steps/test_search_steps.py +++ b/tests/e2e_automation/features/APITests/steps/test_search_steps.py @@ -13,7 +13,10 @@ validate_error_response, validate_to_compare_request_and_response, ) -from utilities.api_get_header import get_search_get_url_header, get_search_post_url_header +from utilities.api_get_header import ( + get_search_get_url_header, + get_search_post_url_header, +) from utilities.date_helper import iso_to_compact from utilities.http_requests_session import http_requests_session @@ -66,78 +69,51 @@ def trigger_search_request(context, httpMethod): trigger_search_request_by_httpMethod(context, httpMethod=httpMethod) -@when("Send a search request with GET method using target-disease for Immunization event created") -def send_search_get_with_target_disease(context): - get_search_get_url_header(context) - patient_ident = context.create_object.contained[1].identifier[0] - target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] - context.params = { - "patient.identifier": f"{patient_ident.system}|{patient_ident.value}", - "target-disease": f"{target.system}|{target.code}", - } - print(f"\n Search Get parameters (target-disease) - \n {context.params}") - context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) - - @when("Send a search request with POST method for Immunization event created") def TriggerSearchPostRequest(context): - get_search_post_url_header(context) context.request = convert_to_form_data( set_request_data( - context.patient.identifier[0].value, context.vaccine_type, datetime.today().strftime("%Y-%m-%d") + context.patient.identifier[0].value, + context.vaccine_type, + datetime.today().strftime("%Y-%m-%d"), ) ) - print(f"\n Search Post Request - \n {context.request}") - context.response = http_requests_session.post(context.url, headers=context.headers, data=context.request) - print(f"\n Search Post Response - \n {context.response.json()}") + trigger_search_request_by_httpMethod(context, httpMethod="POST") -@when("Send a search request with POST method using target-disease for Immunization event created") -def send_search_post_with_target_disease(context): - get_search_post_url_header(context) +@when( + parsers.parse("Send a search request with '{httpMethod}' method using target-disease for Immunization event created") +) +def send_search_with_target_disease(context, httpMethod): patient_ident = context.create_object.contained[1].identifier[0] target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] - context.request = { + context.params = context.request = { "patient.identifier": f"{patient_ident.system}|{patient_ident.value}", "target-disease": f"{target.system}|{target.code}", } - print(f"\n Search Post request (target-disease) - \n {context.request}") - context.response = http_requests_session.post(context.url, headers=context.headers, data=context.request) - - -@when("Send a search request with GET method using comma-separated target-disease for Immunization event created") -def send_search_get_with_comma_separated_target_disease(context): - get_search_get_url_header(context) - patient_ident = context.create_object.contained[1].identifier[0] - targets = context.create_object.protocolApplied[0].targetDisease - target_parts = [f"{t.coding[0].system}|{t.coding[0].code}" for t in targets[:2]] - context.params = { - "patient.identifier": f"{patient_ident.system}|{patient_ident.value}", - "target-disease": ",".join(target_parts), - } - print(f"\n Search Get parameters (comma-separated target-disease) - \n {context.params}") - context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) + trigger_search_request_by_httpMethod(context, httpMethod=httpMethod) -@when("Send a search request with POST method using comma-separated target-disease for Immunization event created") -def send_search_post_with_comma_separated_target_disease(context): - get_search_post_url_header(context) +@when( + parsers.parse( + "Send a search request with '{httpMethod}' method using comma-separated target-disease for Immunization event created" + ) +) +def send_search_post_with_comma_separated_target_disease(context, httpMethod): patient_ident = context.create_object.contained[1].identifier[0] targets = context.create_object.protocolApplied[0].targetDisease target_parts = [f"{t.coding[0].system}|{t.coding[0].code}" for t in targets[:2]] - context.request = { + context.params = context.request = { "patient.identifier": f"{patient_ident.system}|{patient_ident.value}", "target-disease": ",".join(target_parts), } - print(f"\n Search Post request (comma-separated target-disease) - \n {context.request}") - context.response = http_requests_session.post(context.url, headers=context.headers, data=context.request) + trigger_search_request_by_httpMethod(context, httpMethod=httpMethod) @when( "Send a search request with GET method using target-disease and Date From and Date To for Immunization event created" ) def send_search_get_with_target_disease_and_dates(context): - get_search_get_url_header(context) patient_ident = context.create_object.contained[1].identifier[0] target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] context.DateFrom = "2023-01-01" @@ -148,15 +124,13 @@ def send_search_get_with_target_disease_and_dates(context): "-date.from": context.DateFrom, "-date.to": context.DateTo, } - print(f"\n Search Get parameters (target-disease with dates) - \n {context.params}") - context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) + trigger_search_request_by_httpMethod(context, httpMethod="GET") @when( "Send a search request with POST method using target-disease and Date From and Date To for Immunization event created" ) def send_search_post_with_target_disease_and_dates(context): - get_search_post_url_header(context) patient_ident = context.create_object.contained[1].identifier[0] target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] context.DateFrom = "2023-01-01" @@ -167,72 +141,45 @@ def send_search_post_with_target_disease_and_dates(context): "-date.from": context.DateFrom, "-date.to": context.DateTo, } - print(f"\n Search Post request (target-disease with dates) - \n {context.request}") - context.response = http_requests_session.post(context.url, headers=context.headers, data=context.request) + trigger_search_request_by_httpMethod(context, httpMethod="POST") @when("Send a search request with GET method using target-disease for Immunization event created with valid NHS Number") def send_search_get_with_target_disease_unauthorised_supplier(context): - get_search_get_url_header(context) nhs_number = "9000000009" context.params = { "patient.identifier": f"{PATIENT_IDENTIFIER_SYSTEM}|{nhs_number}", "target-disease": f"{TARGET_DISEASE_SYSTEM}|14189004", } - print(f"\n Search Get parameters (target-disease, 403 check) - \n {context.params}") - context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) - - -@when("Send a search request with GET method with valid NHS Number and all invalid target-disease codes") -def send_search_get_with_all_invalid_target_disease_codes(context): - get_search_get_url_header(context) - context.params = { - "patient.identifier": f"{PATIENT_IDENTIFIER_SYSTEM}|9000000009", - "target-disease": "invalid-no-pipe,wrong_system|123", - } - print(f"\n Search Get parameters (all invalid target-disease) - \n {context.params}") - context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) - - -@when("Send a search request with POST method with valid NHS Number and all invalid target-disease codes") -def send_search_post_with_all_invalid_target_disease_codes(context): - get_search_post_url_header(context) - context.request = { - "patient.identifier": f"{PATIENT_IDENTIFIER_SYSTEM}|9000000009", - "target-disease": "invalid-no-pipe,wrong_system|123", - } - print(f"\n Search Post request (all invalid target-disease) - \n {context.request}") - context.response = http_requests_session.post(context.url, headers=context.headers, data=context.request) + trigger_search_request_by_httpMethod(context, httpMethod="GET") @when( - "Send a search request with GET method using mixed valid and invalid target-disease codes for Immunization event created" + parsers.parse( + "Send a search request with '{httpMethod}' method with valid NHS Number and all invalid target-disease codes" + ) ) -def send_search_get_with_mixed_valid_and_invalid_target_disease_codes(context): - get_search_get_url_header(context) - patient_ident = context.create_object.contained[1].identifier[0] - target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] - context.params = { - "patient.identifier": f"{patient_ident.system}|{patient_ident.value}", - "target-disease": f"{target.system}|{target.code},{TARGET_DISEASE_SYSTEM}|{INVALID_TARGET_DISEASE_CODE}", +def send_search_request_with_all_invalid_target_disease_codes(context, httpMethod): + context.params = context.request = { + "patient.identifier": f"{PATIENT_IDENTIFIER_SYSTEM}|9000000009", + "target-disease": "invalid-no-pipe,wrong_system|123", } - print(f"\n Search Get parameters (mixed valid/invalid target-disease) - \n {context.params}") - context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) + trigger_search_request_by_httpMethod(context, httpMethod=httpMethod) @when( - "Send a search request with POST method using mixed valid and invalid target-disease codes for Immunization event created" + parsers.parse( + "Send a search request with '{httpMethod}' method using mixed valid and invalid target-disease codes for Immunization event created" + ) ) -def send_search_post_with_mixed_valid_and_invalid_target_disease_codes(context): - get_search_post_url_header(context) +def send_search_post_with_mixed_valid_and_invalid_target_disease_codes(context, httpMethod): patient_ident = context.create_object.contained[1].identifier[0] target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] - context.request = { + context.params = context.request = { "patient.identifier": f"{patient_ident.system}|{patient_ident.value}", "target-disease": f"{target.system}|{target.code},{TARGET_DISEASE_SYSTEM}|{INVALID_TARGET_DISEASE_CODE}", } - print(f"\n Search Post request (mixed valid/invalid target-disease) - \n {context.request}") - context.response = http_requests_session.post(context.url, headers=context.headers, data=context.request) + trigger_search_request_by_httpMethod(context, httpMethod=httpMethod) @when( @@ -284,7 +231,6 @@ def send_search_with_target_disease_and_immunization_target(context, httpMethod) @when("Send a search request with GET method using target-disease and identifier for Immunization event created") def send_search_get_with_target_disease_and_identifier(context): - get_search_get_url_header(context) patient_ident = context.create_object.contained[1].identifier[0] target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] context.params = { @@ -292,8 +238,7 @@ def send_search_get_with_target_disease_and_identifier(context): "target-disease": f"{target.system}|{target.code}", "identifier": "https://example.org|abc-123", } - print(f"\n Search Get parameters (target-disease with identifier) - \n {context.params}") - context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) + trigger_search_request_by_httpMethod(context, httpMethod="GET") @when( diff --git a/tests/e2e_automation/features/batchTests/Steps/test_batch_file_validation_steps.py b/tests/e2e_automation/features/batchTests/Steps/test_batch_file_validation_steps.py index 357997c0e..e9908b191 100644 --- a/tests/e2e_automation/features/batchTests/Steps/test_batch_file_validation_steps.py +++ b/tests/e2e_automation/features/batchTests/Steps/test_batch_file_validation_steps.py @@ -9,7 +9,11 @@ from utilities.batch_file_helper import validate_inf_ack_file from utilities.batch_S3_buckets import wait_and_read_ack_file -from .batch_common_steps import build_dataFrame_using_datatable, create_batch_file, ignore_if_local_run +from .batch_common_steps import ( + build_dataFrame_using_datatable, + create_batch_file, + ignore_if_local_run, +) scenarios("batchTests/batch_file_validation.feature") @@ -77,7 +81,7 @@ def batch_file_with_additional_column_is_created(datatable, context): def file_will_be_moved_to_destination_bucket(context): result = wait_and_read_ack_file(context, "ack", duplicate_inf_files=True) assert result is not None, f"File not found in destination bucket after timeout: {context.forwarded_prefix}" - context.fileContent = result["csv"] + context.fileContent = result["csv"]["content"] assert context.fileContent, f"File not found in destination bucket after timeout: {context.forwarded_prefix}" diff --git a/tests/e2e_automation/utilities/batch_S3_buckets.py b/tests/e2e_automation/utilities/batch_S3_buckets.py index 0be1deb8f..c08677cee 100644 --- a/tests/e2e_automation/utilities/batch_S3_buckets.py +++ b/tests/e2e_automation/utilities/batch_S3_buckets.py @@ -73,6 +73,7 @@ def wait_and_read_ack_file( expected_extensions = {".csv"} print("[MODE] Expecting ONLY CSV ACK file") + expected_count = len(expected_extensions) found_files = {} while elapsed < timeout: @@ -80,41 +81,50 @@ def wait_and_read_ack_file( response = s3.list_objects_v2(Bucket=destination_bucket, Prefix=forwarded_prefix) contents = response.get("Contents", []) - if not contents: - print(f"[WAIT] No files found yet... ({elapsed}s)") - else: - if duplicate_inf_files and len(contents) == 1: - print(f"[WAIT] Waiting for more INF files... ({elapsed}s)") + contents = [obj for obj in contents if os.path.splitext(obj["Key"])[1].lower() in expected_extensions] + + if len(contents) < expected_count: + print(f"[WAIT] Not all ACK files arrived yet... ({elapsed}s)") + time.sleep(interval) + elapsed += interval + continue + + contents = sorted(contents, key=lambda x: x["LastModified"], reverse=True) - elif duplicate_bus_files: - if len(contents) > len(expected_extensions): - print(f"[ERROR] Unexpected extra BUS files detected: {contents}") - return "Unexpected duplicate BUS file found" - elif len(contents) < len(expected_extensions): - print(f"[WAIT] Not all BUS ACK files arrived yet... ({elapsed}s)") + if duplicate_inf_files and len(contents) == 1: + print(f"[WAIT] Waiting for more INF files... ({elapsed}s)") + time.sleep(interval) + elapsed += interval + continue - for obj in contents: - key = obj["Key"] - ext = os.path.splitext(key)[1].lower() + if duplicate_bus_files: + if len(contents) > expected_count: + print(f"[ERROR] Unexpected extra BUS files detected: {contents}") + return "Unexpected duplicate BUS file found" - if ext in expected_extensions and ext not in found_files: - print(f"[FOUND] {ext} file located: {key}") - s3_obj = s3.get_object(Bucket=destination_bucket, Key=key) - file_data = s3_obj["Body"].read().decode("utf-8") - found_files[ext] = {"key": key, "content": file_data} + print("[INFO] BUS mode: expected files already exist — skipping processing") + return None - print(f"[SUCCESS] Loaded {ext} file ({len(file_data)} bytes)") + for obj in contents: + key = obj["Key"] + ext = os.path.splitext(key)[1].lower() - if expected_extensions.issubset(found_files.keys()): - print("[COMPLETE] All expected ACK files received") + if ext in expected_extensions and ext not in found_files: + print(f"[FOUND] {ext} file located: {key}") + s3_obj = s3.get_object(Bucket=destination_bucket, Key=key) + file_data = s3_obj["Body"].read().decode("utf-8") + found_files[ext] = {"key": key, "content": file_data} - if expected_extensions == {".csv"}: - return {"csv": found_files[".csv"]} + if expected_extensions.issubset(found_files.keys()): + print("[COMPLETE] All expected ACK files received") - return {"csv": found_files[".csv"], "json": found_files[".json"]} + if expected_extensions == {".csv"}: + return {"csv": found_files[".csv"]} - time.sleep(interval) - elapsed += interval + return { + "csv": found_files[".csv"], + "json": found_files[".json"], + } except ClientError as e: print(f"[ERROR] S3 access failed: {e}") diff --git a/tests/e2e_automation/utilities/batch_file_helper.py b/tests/e2e_automation/utilities/batch_file_helper.py index 659276a41..b87e5e0f1 100644 --- a/tests/e2e_automation/utilities/batch_file_helper.py +++ b/tests/e2e_automation/utilities/batch_file_helper.py @@ -32,6 +32,7 @@ def validate_bus_ack_file_for_successful_records(context, file_rows) -> bool: def validate_inf_ack_file(context, success: bool = True) -> bool: content = context.fileContent + content = content.replace("\r\n", "\n") lines = content.strip().split("\n") header = lines[0].split("|") row = lines[1].split("|")