From 0afb45ffa5463ffc36d1643af9c8041cd447a097 Mon Sep 17 00:00:00 2001 From: FimranNHS Date: Mon, 9 Mar 2026 15:12:52 +0000 Subject: [PATCH 1/4] fix the broken test --- .../features/APITests/search.feature | 16 +-- .../APITests/steps/test_search_steps.py | 122 +++++------------- .../Steps/test_batch_file_validation_steps.py | 8 +- .../utilities/batch_S3_buckets.py | 24 +++- .../utilities/batch_file_helper.py | 1 + 5 files changed, 65 insertions(+), 106 deletions(-) diff --git a/tests/e2e_automation/features/APITests/search.feature b/tests/e2e_automation/features/APITests/search.feature index a437476d08..d038e60aef 100644 --- a/tests/e2e_automation/features/APITests/search.feature +++ b/tests/e2e_automation/features/APITests/search.feature @@ -272,14 +272,14 @@ Feature: Search the immunization of a patient @Delete_cleanUp @supplier_name_TPP Scenario: Verify that Search API returns immunization events when searching by target-disease (GET) Given Valid vaccination record is created with Patient 'Random' and vaccine_type 'MMRV' - When Send a search request with GET method using target-disease for Immunization event created + When Send a search request with 'GET' method using target-disease for Immunization event created Then The request will be successful with the status code '200' And The Search Response JSONs should contain the detail of the immunization events created above @Delete_cleanUp @supplier_name_EMIS Scenario: Verify that Search API returns immunization events when searching by target-disease (POST) Given Valid vaccination record is created with Patient 'Random' and vaccine_type '3IN1' - When Send a search request with POST method using target-disease for Immunization event created + When Send a search request with 'POST' method using target-disease for Immunization event created Then The request will be successful with the status code '200' And The Search Response JSONs should contain the detail of the immunization events created above @@ -304,14 +304,14 @@ Feature: Search the immunization of a patient @Delete_cleanUp @supplier_name_Postman_Auth Scenario: Verify that Search API returns immunization events when searching by comma-separated target-disease (GET) Given Valid vaccination record is created with Patient 'Random' and vaccine_type 'HEPB' - When Send a search request with GET method using comma-separated target-disease for Immunization event created + When Send a search request with 'GET' method using comma-separated target-disease for Immunization event created Then The request will be successful with the status code '200' And The Search Response JSONs should contain the detail of the immunization events created above @Delete_cleanUp @supplier_name_Postman_Auth Scenario: Verify that Search API returns immunization events when searching by comma-separated target-disease (POST) Given Valid vaccination record is created with Patient 'Random' and vaccine_type 'COVID' - When Send a search request with POST method using comma-separated target-disease for Immunization event created + When Send a search request with 'POST' method using comma-separated target-disease for Immunization event created Then The request will be successful with the status code '200' And The Search Response JSONs should contain the detail of the immunization events created above @@ -334,10 +334,10 @@ Feature: Search the immunization of a patient @supplier_name_Postman_Auth Scenario: Verify that Search API returns 400 when all target-disease values are invalid SNOMED codes - When Send a search request with GET method with valid NHS Number and all invalid target-disease codes + When Send a search request with 'GET' method with valid NHS Number and all invalid target-disease codes Then The request will be unsuccessful with the status code '400' And The Response JSONs should contain correct error message for invalid target-disease codes - When Send a search request with POST method with valid NHS Number and all invalid target-disease codes + When Send a search request with 'POST' method with valid NHS Number and all invalid target-disease codes Then The request will be unsuccessful with the status code '400' And The Response JSONs should contain correct error message for invalid target-disease codes @@ -345,9 +345,9 @@ Feature: Search the immunization of a patient @Delete_cleanUp @supplier_name_Postman_Auth Scenario: Verify that Search API returns 200 with results and OperationOutcome when some target-disease values are invalid Given Valid vaccination record is created with Patient 'Random' and vaccine_type '6IN1' - When Send a search request with GET method using mixed valid and invalid target-disease codes for Immunization event created + When Send a search request with 'GET' method using mixed valid and invalid target-disease codes for Immunization event created Then The request will be successful with the status code '200' And The Search Response should contain search results and OperationOutcome for invalid target-disease codes - When Send a search request with POST method using mixed valid and invalid target-disease codes for Immunization event created + When Send a search request with 'POST' method using mixed valid and invalid target-disease codes for Immunization event created Then The request will be successful with the status code '200' And The Search Response should contain search results and OperationOutcome for invalid target-disease codes diff --git a/tests/e2e_automation/features/APITests/steps/test_search_steps.py b/tests/e2e_automation/features/APITests/steps/test_search_steps.py index e3a0e593f0..bab9c9dd82 100644 --- a/tests/e2e_automation/features/APITests/steps/test_search_steps.py +++ b/tests/e2e_automation/features/APITests/steps/test_search_steps.py @@ -13,7 +13,10 @@ validate_error_response, validate_to_compare_request_and_response, ) -from utilities.api_get_header import get_search_get_url_header, get_search_post_url_header +from utilities.api_get_header import ( + get_search_get_url_header, + get_search_post_url_header, +) from utilities.date_helper import iso_to_compact from utilities.http_requests_session import http_requests_session @@ -66,62 +69,38 @@ def trigger_search_request(context, httpMethod): trigger_search_request_by_httpMethod(context, httpMethod=httpMethod) -@when("Send a search request with GET method using target-disease for Immunization event created") -def send_search_get_with_target_disease(context): - get_search_get_url_header(context) - patient_ident = context.create_object.contained[1].identifier[0] - target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] - context.params = { - "patient.identifier": f"{patient_ident.system}|{patient_ident.value}", - "target-disease": f"{target.system}|{target.code}", - } - print(f"\n Search Get parameters (target-disease) - \n {context.params}") - context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) - - @when("Send a search request with POST method for Immunization event created") def TriggerSearchPostRequest(context): - get_search_post_url_header(context) context.request = convert_to_form_data( set_request_data( - context.patient.identifier[0].value, context.vaccine_type, datetime.today().strftime("%Y-%m-%d") + context.patient.identifier[0].value, + context.vaccine_type, + datetime.today().strftime("%Y-%m-%d"), ) ) print(f"\n Search Post Request - \n {context.request}") - context.response = http_requests_session.post(context.url, headers=context.headers, data=context.request) - print(f"\n Search Post Response - \n {context.response.json()}") + trigger_search_request_by_httpMethod(context, httpMethod="POST") -@when("Send a search request with POST method using target-disease for Immunization event created") -def send_search_post_with_target_disease(context): - get_search_post_url_header(context) +@when( + parsers.parse("Send a search request with '{httpMethod}' method using target-disease for Immunization event created") +) +def send_search_with_target_disease(context, httpMethod): patient_ident = context.create_object.contained[1].identifier[0] target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] context.request = { "patient.identifier": f"{patient_ident.system}|{patient_ident.value}", "target-disease": f"{target.system}|{target.code}", } - print(f"\n Search Post request (target-disease) - \n {context.request}") - context.response = http_requests_session.post(context.url, headers=context.headers, data=context.request) - - -@when("Send a search request with GET method using comma-separated target-disease for Immunization event created") -def send_search_get_with_comma_separated_target_disease(context): - get_search_get_url_header(context) - patient_ident = context.create_object.contained[1].identifier[0] - targets = context.create_object.protocolApplied[0].targetDisease - target_parts = [f"{t.coding[0].system}|{t.coding[0].code}" for t in targets[:2]] - context.params = { - "patient.identifier": f"{patient_ident.system}|{patient_ident.value}", - "target-disease": ",".join(target_parts), - } - print(f"\n Search Get parameters (comma-separated target-disease) - \n {context.params}") - context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) + trigger_search_request_by_httpMethod(context, httpMethod=httpMethod) -@when("Send a search request with POST method using comma-separated target-disease for Immunization event created") -def send_search_post_with_comma_separated_target_disease(context): - get_search_post_url_header(context) +@when( + parsers.parse( + "Send a search request with '{httpMethod}' method using comma-separated target-disease for Immunization event created" + ) +) +def send_search_post_with_comma_separated_target_disease(context, httpMethod): patient_ident = context.create_object.contained[1].identifier[0] targets = context.create_object.protocolApplied[0].targetDisease target_parts = [f"{t.coding[0].system}|{t.coding[0].code}" for t in targets[:2]] @@ -129,15 +108,13 @@ def send_search_post_with_comma_separated_target_disease(context): "patient.identifier": f"{patient_ident.system}|{patient_ident.value}", "target-disease": ",".join(target_parts), } - print(f"\n Search Post request (comma-separated target-disease) - \n {context.request}") - context.response = http_requests_session.post(context.url, headers=context.headers, data=context.request) + trigger_search_request_by_httpMethod(context, httpMethod=httpMethod) @when( "Send a search request with GET method using target-disease and Date From and Date To for Immunization event created" ) def send_search_get_with_target_disease_and_dates(context): - get_search_get_url_header(context) patient_ident = context.create_object.contained[1].identifier[0] target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] context.DateFrom = "2023-01-01" @@ -148,15 +125,13 @@ def send_search_get_with_target_disease_and_dates(context): "-date.from": context.DateFrom, "-date.to": context.DateTo, } - print(f"\n Search Get parameters (target-disease with dates) - \n {context.params}") - context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) + trigger_search_request_by_httpMethod(context, httpMethod="GET") @when( "Send a search request with POST method using target-disease and Date From and Date To for Immunization event created" ) def send_search_post_with_target_disease_and_dates(context): - get_search_post_url_header(context) patient_ident = context.create_object.contained[1].identifier[0] target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] context.DateFrom = "2023-01-01" @@ -167,72 +142,41 @@ def send_search_post_with_target_disease_and_dates(context): "-date.from": context.DateFrom, "-date.to": context.DateTo, } - print(f"\n Search Post request (target-disease with dates) - \n {context.request}") - context.response = http_requests_session.post(context.url, headers=context.headers, data=context.request) + trigger_search_request_by_httpMethod(context, httpMethod="POST") @when("Send a search request with GET method using target-disease for Immunization event created with valid NHS Number") def send_search_get_with_target_disease_unauthorised_supplier(context): - get_search_get_url_header(context) nhs_number = "9000000009" context.params = { "patient.identifier": f"{PATIENT_IDENTIFIER_SYSTEM}|{nhs_number}", "target-disease": f"{TARGET_DISEASE_SYSTEM}|14189004", } - print(f"\n Search Get parameters (target-disease, 403 check) - \n {context.params}") - context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) + trigger_search_request_by_httpMethod(context, httpMethod="GET") -@when("Send a search request with GET method with valid NHS Number and all invalid target-disease codes") -def send_search_get_with_all_invalid_target_disease_codes(context): - get_search_get_url_header(context) - context.params = { - "patient.identifier": f"{PATIENT_IDENTIFIER_SYSTEM}|9000000009", - "target-disease": "invalid-no-pipe,wrong_system|123", - } - print(f"\n Search Get parameters (all invalid target-disease) - \n {context.params}") - context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) - - -@when("Send a search request with POST method with valid NHS Number and all invalid target-disease codes") -def send_search_post_with_all_invalid_target_disease_codes(context): - get_search_post_url_header(context) +@when("Send a search request with '{httpMethod}' method with valid NHS Number and all invalid target-disease codes") +def send_search_request_with_all_invalid_target_disease_codes(context, httpMethod): context.request = { "patient.identifier": f"{PATIENT_IDENTIFIER_SYSTEM}|9000000009", "target-disease": "invalid-no-pipe,wrong_system|123", } - print(f"\n Search Post request (all invalid target-disease) - \n {context.request}") - context.response = http_requests_session.post(context.url, headers=context.headers, data=context.request) - - -@when( - "Send a search request with GET method using mixed valid and invalid target-disease codes for Immunization event created" -) -def send_search_get_with_mixed_valid_and_invalid_target_disease_codes(context): - get_search_get_url_header(context) - patient_ident = context.create_object.contained[1].identifier[0] - target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] - context.params = { - "patient.identifier": f"{patient_ident.system}|{patient_ident.value}", - "target-disease": f"{target.system}|{target.code},{TARGET_DISEASE_SYSTEM}|{INVALID_TARGET_DISEASE_CODE}", - } - print(f"\n Search Get parameters (mixed valid/invalid target-disease) - \n {context.params}") - context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) + trigger_search_request_by_httpMethod(context, httpMethod=httpMethod) @when( - "Send a search request with POST method using mixed valid and invalid target-disease codes for Immunization event created" + parsers.parse( + "Send a search request with '{httpMethod}' method using mixed valid and invalid target-disease codes for Immunization event created" + ) ) -def send_search_post_with_mixed_valid_and_invalid_target_disease_codes(context): - get_search_post_url_header(context) +def send_search_post_with_mixed_valid_and_invalid_target_disease_codes(context, httpMethod): patient_ident = context.create_object.contained[1].identifier[0] target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] context.request = { "patient.identifier": f"{patient_ident.system}|{patient_ident.value}", "target-disease": f"{target.system}|{target.code},{TARGET_DISEASE_SYSTEM}|{INVALID_TARGET_DISEASE_CODE}", } - print(f"\n Search Post request (mixed valid/invalid target-disease) - \n {context.request}") - context.response = http_requests_session.post(context.url, headers=context.headers, data=context.request) + trigger_search_request_by_httpMethod(context, httpMethod=httpMethod) @when( @@ -284,7 +228,6 @@ def send_search_with_target_disease_and_immunization_target(context, httpMethod) @when("Send a search request with GET method using target-disease and identifier for Immunization event created") def send_search_get_with_target_disease_and_identifier(context): - get_search_get_url_header(context) patient_ident = context.create_object.contained[1].identifier[0] target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] context.params = { @@ -292,8 +235,7 @@ def send_search_get_with_target_disease_and_identifier(context): "target-disease": f"{target.system}|{target.code}", "identifier": "https://example.org|abc-123", } - print(f"\n Search Get parameters (target-disease with identifier) - \n {context.params}") - context.response = http_requests_session.get(context.url, params=context.params, headers=context.headers) + trigger_search_request_by_httpMethod(context, httpMethod="GET") @when( diff --git a/tests/e2e_automation/features/batchTests/Steps/test_batch_file_validation_steps.py b/tests/e2e_automation/features/batchTests/Steps/test_batch_file_validation_steps.py index 357997c0ee..e9908b1910 100644 --- a/tests/e2e_automation/features/batchTests/Steps/test_batch_file_validation_steps.py +++ b/tests/e2e_automation/features/batchTests/Steps/test_batch_file_validation_steps.py @@ -9,7 +9,11 @@ from utilities.batch_file_helper import validate_inf_ack_file from utilities.batch_S3_buckets import wait_and_read_ack_file -from .batch_common_steps import build_dataFrame_using_datatable, create_batch_file, ignore_if_local_run +from .batch_common_steps import ( + build_dataFrame_using_datatable, + create_batch_file, + ignore_if_local_run, +) scenarios("batchTests/batch_file_validation.feature") @@ -77,7 +81,7 @@ def batch_file_with_additional_column_is_created(datatable, context): def file_will_be_moved_to_destination_bucket(context): result = wait_and_read_ack_file(context, "ack", duplicate_inf_files=True) assert result is not None, f"File not found in destination bucket after timeout: {context.forwarded_prefix}" - context.fileContent = result["csv"] + context.fileContent = result["csv"]["content"] assert context.fileContent, f"File not found in destination bucket after timeout: {context.forwarded_prefix}" diff --git a/tests/e2e_automation/utilities/batch_S3_buckets.py b/tests/e2e_automation/utilities/batch_S3_buckets.py index 0be1deb8f9..af4578c88c 100644 --- a/tests/e2e_automation/utilities/batch_S3_buckets.py +++ b/tests/e2e_automation/utilities/batch_S3_buckets.py @@ -83,15 +83,27 @@ def wait_and_read_ack_file( if not contents: print(f"[WAIT] No files found yet... ({elapsed}s)") else: + contents = sorted(contents, key=lambda x: x["LastModified"], reverse=True) + if duplicate_inf_files and len(contents) == 1: print(f"[WAIT] Waiting for more INF files... ({elapsed}s)") + time.sleep(interval) + elapsed += interval + continue elif duplicate_bus_files: if len(contents) > len(expected_extensions): print(f"[ERROR] Unexpected extra BUS files detected: {contents}") return "Unexpected duplicate BUS file found" - elif len(contents) < len(expected_extensions): - print(f"[WAIT] Not all BUS ACK files arrived yet... ({elapsed}s)") + + print("[INFO] BUS mode: expected files already exist — skipping processing") + return None + + if len(contents) < len(expected_extensions): + print(f"[WAIT] Not all BUS ACK files arrived yet... ({elapsed}s)") + time.sleep(interval) + elapsed += interval + continue for obj in contents: key = obj["Key"] @@ -111,10 +123,10 @@ def wait_and_read_ack_file( if expected_extensions == {".csv"}: return {"csv": found_files[".csv"]} - return {"csv": found_files[".csv"], "json": found_files[".json"]} - - time.sleep(interval) - elapsed += interval + return { + "csv": found_files[".csv"], + "json": found_files[".json"], + } except ClientError as e: print(f"[ERROR] S3 access failed: {e}") diff --git a/tests/e2e_automation/utilities/batch_file_helper.py b/tests/e2e_automation/utilities/batch_file_helper.py index 659276a415..b87e5e0f11 100644 --- a/tests/e2e_automation/utilities/batch_file_helper.py +++ b/tests/e2e_automation/utilities/batch_file_helper.py @@ -32,6 +32,7 @@ def validate_bus_ack_file_for_successful_records(context, file_rows) -> bool: def validate_inf_ack_file(context, success: bool = True) -> bool: content = context.fileContent + content = content.replace("\r\n", "\n") lines = content.strip().split("\n") header = lines[0].split("|") row = lines[1].split("|") From 1a9cfb8168ee2eeda13509aa1ac6f57b8e8a2647 Mon Sep 17 00:00:00 2001 From: FimranNHS Date: Mon, 9 Mar 2026 18:50:48 +0000 Subject: [PATCH 2/4] fix search broken tests --- .../features/APITests/steps/test_search_steps.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/e2e_automation/features/APITests/steps/test_search_steps.py b/tests/e2e_automation/features/APITests/steps/test_search_steps.py index bab9c9dd82..bba09f9810 100644 --- a/tests/e2e_automation/features/APITests/steps/test_search_steps.py +++ b/tests/e2e_automation/features/APITests/steps/test_search_steps.py @@ -78,7 +78,6 @@ def TriggerSearchPostRequest(context): datetime.today().strftime("%Y-%m-%d"), ) ) - print(f"\n Search Post Request - \n {context.request}") trigger_search_request_by_httpMethod(context, httpMethod="POST") @@ -88,7 +87,7 @@ def TriggerSearchPostRequest(context): def send_search_with_target_disease(context, httpMethod): patient_ident = context.create_object.contained[1].identifier[0] target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] - context.request = { + context.params = context.request = { "patient.identifier": f"{patient_ident.system}|{patient_ident.value}", "target-disease": f"{target.system}|{target.code}", } @@ -104,7 +103,7 @@ def send_search_post_with_comma_separated_target_disease(context, httpMethod): patient_ident = context.create_object.contained[1].identifier[0] targets = context.create_object.protocolApplied[0].targetDisease target_parts = [f"{t.coding[0].system}|{t.coding[0].code}" for t in targets[:2]] - context.request = { + context.params = context.request = { "patient.identifier": f"{patient_ident.system}|{patient_ident.value}", "target-disease": ",".join(target_parts), } @@ -155,9 +154,13 @@ def send_search_get_with_target_disease_unauthorised_supplier(context): trigger_search_request_by_httpMethod(context, httpMethod="GET") -@when("Send a search request with '{httpMethod}' method with valid NHS Number and all invalid target-disease codes") +@when( + parsers.parse( + "Send a search request with '{httpMethod}' method with valid NHS Number and all invalid target-disease codes" + ) +) def send_search_request_with_all_invalid_target_disease_codes(context, httpMethod): - context.request = { + context.params = context.request = { "patient.identifier": f"{PATIENT_IDENTIFIER_SYSTEM}|9000000009", "target-disease": "invalid-no-pipe,wrong_system|123", } @@ -172,7 +175,7 @@ def send_search_request_with_all_invalid_target_disease_codes(context, httpMetho def send_search_post_with_mixed_valid_and_invalid_target_disease_codes(context, httpMethod): patient_ident = context.create_object.contained[1].identifier[0] target = context.create_object.protocolApplied[0].targetDisease[0].coding[0] - context.request = { + context.params = context.request = { "patient.identifier": f"{patient_ident.system}|{patient_ident.value}", "target-disease": f"{target.system}|{target.code},{TARGET_DISEASE_SYSTEM}|{INVALID_TARGET_DISEASE_CODE}", } From f613a69936b233578d72f934c76f4450d42a908e Mon Sep 17 00:00:00 2001 From: FimranNHS Date: Mon, 9 Mar 2026 19:09:03 +0000 Subject: [PATCH 3/4] added missing option for batch file validation scenarios --- .github/workflows/run-e2e-automation-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-e2e-automation-tests.yml b/.github/workflows/run-e2e-automation-tests.yml index c97533c52f..424b5208b0 100644 --- a/.github/workflows/run-e2e-automation-tests.yml +++ b/.github/workflows/run-e2e-automation-tests.yml @@ -67,6 +67,7 @@ on: - functional - sandbox - proxy_smoke + - Batch_File_Validation_Feature env: APIGEE_AUTH_ENV: ${{ inputs.apigee_environment == 'int' && inputs.apigee_environment || 'internal-dev' }} From e61595194c82df244aa84a57753a90801d1cec69 Mon Sep 17 00:00:00 2001 From: FimranNHS Date: Mon, 9 Mar 2026 20:55:56 +0000 Subject: [PATCH 4/4] fix --- .../utilities/batch_S3_buckets.py | 92 +++++++++---------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/tests/e2e_automation/utilities/batch_S3_buckets.py b/tests/e2e_automation/utilities/batch_S3_buckets.py index af4578c88c..c08677cee3 100644 --- a/tests/e2e_automation/utilities/batch_S3_buckets.py +++ b/tests/e2e_automation/utilities/batch_S3_buckets.py @@ -73,6 +73,7 @@ def wait_and_read_ack_file( expected_extensions = {".csv"} print("[MODE] Expecting ONLY CSV ACK file") + expected_count = len(expected_extensions) found_files = {} while elapsed < timeout: @@ -80,53 +81,50 @@ def wait_and_read_ack_file( response = s3.list_objects_v2(Bucket=destination_bucket, Prefix=forwarded_prefix) contents = response.get("Contents", []) - if not contents: - print(f"[WAIT] No files found yet... ({elapsed}s)") - else: - contents = sorted(contents, key=lambda x: x["LastModified"], reverse=True) - - if duplicate_inf_files and len(contents) == 1: - print(f"[WAIT] Waiting for more INF files... ({elapsed}s)") - time.sleep(interval) - elapsed += interval - continue - - elif duplicate_bus_files: - if len(contents) > len(expected_extensions): - print(f"[ERROR] Unexpected extra BUS files detected: {contents}") - return "Unexpected duplicate BUS file found" - - print("[INFO] BUS mode: expected files already exist — skipping processing") - return None - - if len(contents) < len(expected_extensions): - print(f"[WAIT] Not all BUS ACK files arrived yet... ({elapsed}s)") - time.sleep(interval) - elapsed += interval - continue - - for obj in contents: - key = obj["Key"] - ext = os.path.splitext(key)[1].lower() - - if ext in expected_extensions and ext not in found_files: - print(f"[FOUND] {ext} file located: {key}") - s3_obj = s3.get_object(Bucket=destination_bucket, Key=key) - file_data = s3_obj["Body"].read().decode("utf-8") - found_files[ext] = {"key": key, "content": file_data} - - print(f"[SUCCESS] Loaded {ext} file ({len(file_data)} bytes)") - - if expected_extensions.issubset(found_files.keys()): - print("[COMPLETE] All expected ACK files received") - - if expected_extensions == {".csv"}: - return {"csv": found_files[".csv"]} - - return { - "csv": found_files[".csv"], - "json": found_files[".json"], - } + contents = [obj for obj in contents if os.path.splitext(obj["Key"])[1].lower() in expected_extensions] + + if len(contents) < expected_count: + print(f"[WAIT] Not all ACK files arrived yet... ({elapsed}s)") + time.sleep(interval) + elapsed += interval + continue + + contents = sorted(contents, key=lambda x: x["LastModified"], reverse=True) + + if duplicate_inf_files and len(contents) == 1: + print(f"[WAIT] Waiting for more INF files... ({elapsed}s)") + time.sleep(interval) + elapsed += interval + continue + + if duplicate_bus_files: + if len(contents) > expected_count: + print(f"[ERROR] Unexpected extra BUS files detected: {contents}") + return "Unexpected duplicate BUS file found" + + print("[INFO] BUS mode: expected files already exist — skipping processing") + return None + + for obj in contents: + key = obj["Key"] + ext = os.path.splitext(key)[1].lower() + + if ext in expected_extensions and ext not in found_files: + print(f"[FOUND] {ext} file located: {key}") + s3_obj = s3.get_object(Bucket=destination_bucket, Key=key) + file_data = s3_obj["Body"].read().decode("utf-8") + found_files[ext] = {"key": key, "content": file_data} + + if expected_extensions.issubset(found_files.keys()): + print("[COMPLETE] All expected ACK files received") + + if expected_extensions == {".csv"}: + return {"csv": found_files[".csv"]} + + return { + "csv": found_files[".csv"], + "json": found_files[".json"], + } except ClientError as e: print(f"[ERROR] S3 access failed: {e}")