From d4e6f515b824498ece5a49a9deb279059e7e2c43 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 29 Oct 2025 10:25:55 -0500 Subject: [PATCH 01/18] Prevent repeat SSNs in post license endpoint --- .../provider-data-v1/handlers/licenses.py | 12 ++++++++ .../function/test_handlers/test_licenses.py | 29 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py index 17c80b9d8..97567d484 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py @@ -64,6 +64,18 @@ def post_licenses(event: dict, context: LambdaContext): # noqa: ARG001 unused-a } ) + # verify that none of the SSNs are repeats within the same batch + license_ssns = [license_record['ssn'] for license_record in licenses] + if len(set(license_ssns)) < len(license_ssns): + raise CCInvalidRequestCustomResponseException( + response_body={ + 'message': 'Invalid license records in request. See errors for more detail.', + 'errors': { + 'SSN': 'Same SSN detected on multiple rows. Every record must have a unique SSN within the same request.' + }, + } + ) + event_time = config.current_standard_datetime logger.info('Sending license records to preprocessing queue', compact=compact, jurisdiction=jurisdiction) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_licenses.py b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_licenses.py index 143b0bfcc..f4e3120c4 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_licenses.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_licenses.py @@ -365,6 +365,35 @@ def test_post_licenses_null_field_returns_error(self): json.loads(resp['body']), ) + def test_post_licenses_returns_400_if_repeated_ssns_detected(self): + from handlers.licenses import post_licenses + + with open('../common/tests/resources/api-event.json') as f: + event = json.load(f) + + # The user has write permission for aslp/oh + event['requestContext']['authorizer']['claims']['scope'] = 'openid email aslp/readGeneral oh/aslp.write' + event['pathParameters'] = {'compact': 'aslp', 'jurisdiction': 'oh'} + with open('../common/tests/resources/api/license-post.json') as f: + license_data = json.load(f) + event['body'] = json.dumps([license_data, license_data]) + + # Add signature authentication headers + event = self._create_signed_event(event) + + resp = post_licenses(event, self.mock_context) + + self.assertEqual(400, resp['statusCode']) + self.assertEqual( + { + 'message': 'Invalid license records in request. See errors for more detail.', + 'errors': { + 'SSN': 'Same SSN detected on multiple rows. Every record must have a unique SSN within the same request.', + }, + }, + json.loads(resp['body']), + ) + def test_post_licenses_strips_whitespace_from_string_fields(self): """Test that whitespace is stripped from all string fields in license data.""" from handlers.licenses import post_licenses From 7b650c5b2697ae171152de4e36a1311564dda5ba Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 30 Oct 2025 10:12:48 -0500 Subject: [PATCH 02/18] Prevent repeated SSN updates within same CSV file upload --- .../provider-data-v1/handlers/bulk_upload.py | 12 +++ .../test_handlers/test_bulk_upload.py | 81 +++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py index 1b9118d25..a7fa4b79f 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py @@ -139,6 +139,9 @@ def process_bulk_upload_file( current_batch = [] total_processed = 0 failed_validation_count = 0 + # track which ssns were included in this file to detect duplicates, + # which are not allowed within the same file upload + ssns_in_file_upload = [] with EventBatchWriter(config.events_client) as event_writer: for i, raw_license in enumerate(reader.licenses(stream)): @@ -148,6 +151,15 @@ def process_bulk_upload_file( # dict() here, because it prevents `compact` and `jurisdiction` from being allowed in the # raw_license validated_license = schema.load(dict(compact=compact, jurisdiction=jurisdiction, **raw_license)) + # verify that this ssn has not been used previously in the same batch + license_ssn = validated_license['ssn'] + for index, record_ssn in enumerate(ssns_in_file_upload): + if license_ssn == record_ssn: + raise ValidationError( + message=f'Duplicate License SSN detected. SSN matches with record {index + 1}. ' + f'Every record must have a unique SSN within the same file.' + ) + ssns_in_file_upload.append(license_ssn) except TypeError as e: # This will be raised, if `raw_license` includes compact and/or jurisdiction fields logger.error('License contains unsupported fields', fields=list(raw_license.keys()), exc_info=e) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py index d5a90aa09..67a86652e 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py @@ -267,6 +267,87 @@ def test_bulk_upload_prevents_compact_jurisdiction_overwrites(self): self.assertEqual(expected_entry, call_args) + + def test_bulk_upload_prevents_repeated_ssns_within_the_same_file_upload(self): + """Test that CSV compact/jurisdiction fields cannot overwrite URL path values.""" + from handlers.bulk_upload import parse_bulk_upload_file + + # Create CSV content that includes duplicate SSNs + # Rows that duplicate the same SSN will be considered an error and not processed + csv_content = ( + 'ssn,npi,licenseNumber,givenName,middleName,familyName,suffix,dateOfBirth,dateOfIssuance' + ',dateOfRenewal,dateOfExpiration,licenseStatus,compactEligibility,homeAddressStreet1' + ',homeAddressStreet2,homeAddressCity,homeAddressState,homeAddressPostalCode' + ',emailAddress,phoneNumber,licenseType,licenseStatusName\n' + '123-45-6789,1234567890,LICENSE123,John,Middle,Doe,Jr.,1990-01-01,2020-01-01,2021-01-01,2023-01-01,active,' + 'eligible,123 Main St,Apt 1,Columbus,OH,43215,test@example.com,+15551234567,audiologist,Active\n' + '123-45-6789,1234567890,LICENSE456,Jane,Middle,Smith,,1995-01-01,2023-01-01,2025-01-01,2026-01-01,active,' + 'eligible,123 Main St,Apt 1,Columbus,OH,43215,test@example.com,+15551234567,audiologist,Active' + ) + + # Upload the CSV content directly to the mock S3 bucket + # URL path indicates aslp/oh, but CSV contains malicious_compact/malicious_jurisdiction + object_key = f'aslp/oh/{uuid4().hex}' + self._bucket.put_object(Key=object_key, Body=csv_content) + + # Simulate the s3 bucket event + with open('../common/tests/resources/put-event.json') as f: + event = json.load(f) + + event['Records'][0]['s3']['bucket'] = { + 'name': self._bucket.name, + 'arn': f'arn:aws:s3:::{self._bucket.name}', + 'ownerIdentity': {'principalId': 'ASDFG123'}, + } + event['Records'][0]['s3']['object']['key'] = object_key + + # Mock EventBatchWriter to capture put_event calls + with patch('handlers.bulk_upload.EventBatchWriter') as mock_event_writer_class: + mock_event_writer = mock_event_writer_class.return_value.__enter__.return_value + # Mock the failed_entry_count attribute to return 0 + mock_event_writer.failed_entry_count = 0 + + # Process the file - should not raise an exception + parse_bulk_upload_file(event, self.mock_context) + + # Verify that put_event was called for the validation error + mock_event_writer.put_event.assert_called_once() + + # Get the call arguments to verify the event details + call_args = mock_event_writer.put_event.call_args[1]['Entry'] + + # Verify the complete event structure + expected_entry = { + 'Source': f'org.compactconnect.bulk-ingest.{object_key}', + 'DetailType': 'license.validation-error', + 'Detail': json.dumps( + { + 'eventTime': '1970-01-01T00:00:00+00:00', + 'compact': 'aslp', + 'jurisdiction': 'oh', + 'recordNumber': 2, + 'validData': { + 'licenseType': 'audiologist', + 'licenseStatusName': 'Active', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'npi': '1234567890', + 'licenseNumber': 'LICENSE456', + 'givenName': 'Jane', + 'middleName': 'Middle', + 'familyName': 'Smith', + 'dateOfIssuance': '2023-01-01', + 'dateOfRenewal': '2025-01-01', + 'dateOfExpiration': '2026-01-01', + }, + 'errors': ['Duplicate License SSN detected. SSN matches with record 1. Every record must have a unique SSN within the same file.'], + } + ), + 'EventBusName': 'license-data-events', + } + + self.assertEqual(expected_entry, call_args) + def test_bulk_upload_handles_bom_character(self): """Test that CSV files with BOM characters are handled correctly.""" from handlers.bulk_upload import parse_bulk_upload_file From 9bbb74bb5e79fd15830cf393fcd38b6e123637c1 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 30 Oct 2025 10:19:22 -0500 Subject: [PATCH 03/18] rename common enum update category to denote use in ingest --- .../python/common/cc_common/data_model/schema/common.py | 4 +++- .../lambdas/python/provider-data-v1/handlers/ingest.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/common.py b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/common.py index c2f77a878..27d3fa4d4 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/common.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/common.py @@ -288,7 +288,6 @@ class UpdateCategory(CCEnum): DEACTIVATION = 'deactivation' EXPIRATION = 'expiration' ISSUANCE = 'issuance' - OTHER = 'other' RENEWAL = 'renewal' ENCUMBRANCE = 'encumbrance' HOME_JURISDICTION_CHANGE = 'homeJurisdictionChange' @@ -297,6 +296,9 @@ class UpdateCategory(CCEnum): # this is specific to privileges that are deactivated due to a state license deactivation LICENSE_DEACTIVATION = 'licenseDeactivation' EMAIL_CHANGE = 'emailChange' + # NOTE: this value should explicitly be used for license upload updates, not anywhere else + # it is referenced in the event that an invalid license upload needs to be reverted. + LICENSE_UPLOAD_UPDATE_OTHER = 'other' class ActiveInactiveStatus(CCEnum): diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/ingest.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/ingest.py index 7021d1d5e..e84ace0b0 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/ingest.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/ingest.py @@ -297,7 +297,7 @@ def _populate_update_record(*, existing_license: dict, updated_values: dict, rem update_type = UpdateCategory.DEACTIVATION logger.info('License deactivation detected') if update_type is None: - update_type = UpdateCategory.OTHER + update_type = UpdateCategory.LICENSE_UPLOAD_UPDATE_OTHER logger.info('License update detected') now = config.current_standard_datetime From 72b2de082900b291bb61e212da6c000c2efa8779 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 30 Oct 2025 10:35:31 -0500 Subject: [PATCH 04/18] formatting --- .../lambdas/python/provider-data-v1/handlers/bulk_upload.py | 2 +- .../lambdas/python/provider-data-v1/handlers/licenses.py | 3 ++- .../tests/function/test_handlers/test_bulk_upload.py | 6 ++++-- .../tests/function/test_handlers/test_licenses.py | 3 ++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py index a7fa4b79f..8c04fa829 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py @@ -157,7 +157,7 @@ def process_bulk_upload_file( if license_ssn == record_ssn: raise ValidationError( message=f'Duplicate License SSN detected. SSN matches with record {index + 1}. ' - f'Every record must have a unique SSN within the same file.' + f'Every record must have a unique SSN within the same file.' ) ssns_in_file_upload.append(license_ssn) except TypeError as e: diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py index 97567d484..191176357 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py @@ -71,7 +71,8 @@ def post_licenses(event: dict, context: LambdaContext): # noqa: ARG001 unused-a response_body={ 'message': 'Invalid license records in request. See errors for more detail.', 'errors': { - 'SSN': 'Same SSN detected on multiple rows. Every record must have a unique SSN within the same request.' + 'SSN': 'Same SSN detected on multiple rows. ' + 'Every record must have a unique SSN within the same request.' }, } ) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py index 67a86652e..6f522dec0 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py @@ -267,7 +267,6 @@ def test_bulk_upload_prevents_compact_jurisdiction_overwrites(self): self.assertEqual(expected_entry, call_args) - def test_bulk_upload_prevents_repeated_ssns_within_the_same_file_upload(self): """Test that CSV compact/jurisdiction fields cannot overwrite URL path values.""" from handlers.bulk_upload import parse_bulk_upload_file @@ -340,7 +339,10 @@ def test_bulk_upload_prevents_repeated_ssns_within_the_same_file_upload(self): 'dateOfRenewal': '2025-01-01', 'dateOfExpiration': '2026-01-01', }, - 'errors': ['Duplicate License SSN detected. SSN matches with record 1. Every record must have a unique SSN within the same file.'], + 'errors': [ + 'Duplicate License SSN detected. SSN matches with record 1. ' + 'Every record must have a unique SSN within the same file.' + ], } ), 'EventBusName': 'license-data-events', diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_licenses.py b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_licenses.py index f4e3120c4..49e767af4 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_licenses.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_licenses.py @@ -388,7 +388,8 @@ def test_post_licenses_returns_400_if_repeated_ssns_detected(self): { 'message': 'Invalid license records in request. See errors for more detail.', 'errors': { - 'SSN': 'Same SSN detected on multiple rows. Every record must have a unique SSN within the same request.', + 'SSN': 'Same SSN detected on multiple rows. ' + 'Every record must have a unique SSN within the same request.', }, }, json.loads(resp['body']), From 696521f40d78843c9aaa137e66458926f19a7ec7 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 30 Oct 2025 10:58:32 -0500 Subject: [PATCH 05/18] Use str enum for feature flag names to reduce risk of typos --- .../cc_common/data_model/data_client.py | 8 +++--- .../data_model/provider_record_util.py | 4 +-- .../common/cc_common/feature_flag_client.py | 6 ++--- .../common/cc_common/feature_flag_enum.py | 14 ++++++++++ .../tests/unit/test_feature_flag_client.py | 26 ++++++++++--------- .../provider-data-v1/handlers/encumbrance.py | 4 +-- 6 files changed, 39 insertions(+), 23 deletions(-) create mode 100644 backend/compact-connect/lambdas/python/common/cc_common/feature_flag_enum.py diff --git a/backend/compact-connect/lambdas/python/common/cc_common/data_model/data_client.py b/backend/compact-connect/lambdas/python/common/cc_common/data_model/data_client.py index 84483a716..1156b72ab 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/data_model/data_client.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/data_model/data_client.py @@ -1428,9 +1428,9 @@ def encumber_privilege(self, adverse_action: AdverseActionData) -> None: now = config.current_standard_datetime # TODO - remove the flag as part of https://github.com/csg-org/CompactConnect/issues/1136 # noqa: FIX002 - from cc_common.feature_flag_client import is_feature_enabled + from cc_common.feature_flag_client import FeatureFlagEnum, is_feature_enabled - if is_feature_enabled('encumbrance-multi-category-flag'): + if is_feature_enabled(FeatureFlagEnum.ENCUMBRANCE_MULTI_CATEGORY_FLAG): encumbrance_details = { 'clinicalPrivilegeActionCategories': adverse_action.clinicalPrivilegeActionCategories, 'adverseActionId': adverse_action.adverseActionId, @@ -2674,9 +2674,9 @@ def encumber_home_jurisdiction_license_privileges( 'Found privileges to encumber', privilege_count=len(unencumbered_privileges_associated_with_license) ) # TODO - remove the flag as part of https://github.com/csg-org/CompactConnect/issues/1136 # noqa: FIX002 - from cc_common.feature_flag_client import is_feature_enabled + from cc_common.feature_flag_client import FeatureFlagEnum, is_feature_enabled - if is_feature_enabled('encumbrance-multi-category-flag'): + if is_feature_enabled(FeatureFlagEnum.ENCUMBRANCE_MULTI_CATEGORY_FLAG): encumbrance_details = { 'clinicalPrivilegeActionCategories': adverse_action.clinicalPrivilegeActionCategories, 'licenseJurisdiction': jurisdiction, diff --git a/backend/compact-connect/lambdas/python/common/cc_common/data_model/provider_record_util.py b/backend/compact-connect/lambdas/python/common/cc_common/data_model/provider_record_util.py index ded200d6f..5ce44325a 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/data_model/provider_record_util.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/data_model/provider_record_util.py @@ -375,9 +375,9 @@ def construct_simplified_privilege_history_object( ): # TODO - remove the flag as part of https://github.com/csg-org/CompactConnect/issues/1136 # noqa: FIX002 # as well as check for deprecated field - from cc_common.feature_flag_client import is_feature_enabled + from cc_common.feature_flag_client import FeatureFlagEnum, is_feature_enabled - if is_feature_enabled('encumbrance-multi-category-flag'): + if is_feature_enabled(FeatureFlagEnum.ENCUMBRANCE_MULTI_CATEGORY_FLAG): if 'clinicalPrivilegeActionCategory' in event['encumbranceDetails']: event['note'] = event['encumbranceDetails'].get('clinicalPrivilegeActionCategory') else: diff --git a/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_client.py b/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_client.py index 23e3565f2..a41a82d75 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_client.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_client.py @@ -9,8 +9,8 @@ from typing import Any import requests - from cc_common.config import config, logger +from cc_common.feature_flag_enum import FeatureFlagEnum @dataclass @@ -43,14 +43,14 @@ def to_dict(self) -> dict[str, Any]: return result -def is_feature_enabled(flag_name: str, context: FeatureFlagContext | None = None, fail_default: bool = False) -> bool: +def is_feature_enabled(flag_name: FeatureFlagEnum, context: FeatureFlagContext | None = None, fail_default: bool = False) -> bool: """ Check if a feature flag is enabled. This function calls the internal feature flag API endpoint to determine if a feature flag is enabled for the given context. - :param flag_name: The name of the feature flag to check + :param flag_name: The name of the feature flag to check. :param context: Optional FeatureFlagContext for feature flag evaluation :param fail_default: If True, return True on errors; if False, return False on errors (default: False) :return: True if the feature flag is enabled, False otherwise (or fail_default value on error) diff --git a/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_enum.py b/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_enum.py new file mode 100644 index 000000000..1e49b28d9 --- /dev/null +++ b/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_enum.py @@ -0,0 +1,14 @@ +from enum import StrEnum + + +class FeatureFlagEnum(StrEnum): + """ + Central source for all feature flags currently referenced in the python code of the project. + Flags should be defined here when first added, and removed when the flag + is no longer in use. + """ + # flag used by internal testing + TEST_FLAG = 'test-flag' + # runtime flags + ENCUMBRANCE_MULTI_CATEGORY_FLAG = 'encumbrance-multi-category-flag' + DUPLICATE_SSN_UPLOAD_CHECK_FLAG = 'duplicate-ssn-upload-check-flag' diff --git a/backend/compact-connect/lambdas/python/common/tests/unit/test_feature_flag_client.py b/backend/compact-connect/lambdas/python/common/tests/unit/test_feature_flag_client.py index d244cea68..b95300b13 100644 --- a/backend/compact-connect/lambdas/python/common/tests/unit/test_feature_flag_client.py +++ b/backend/compact-connect/lambdas/python/common/tests/unit/test_feature_flag_client.py @@ -1,5 +1,7 @@ from unittest.mock import MagicMock, patch +from cc_common.feature_flag_enum import FeatureFlagEnum + from tests import TstLambdas @@ -13,7 +15,7 @@ def test_is_feature_enabled_returns_true_when_flag_enabled(self): mock_response.json.return_value = {'enabled': True} with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response) as mock_post: - result = is_feature_enabled('test-flag') + result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG) # Verify the result self.assertTrue(result) @@ -35,7 +37,7 @@ def test_is_feature_enabled_returns_false_when_flag_disabled(self): mock_response.json.return_value = {'enabled': False} with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response): - result = is_feature_enabled('test-flag') + result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG) # Verify the result self.assertFalse(result) @@ -51,7 +53,7 @@ def test_is_feature_enabled_with_context(self): context = FeatureFlagContext(user_id='user123', custom_attributes={'licenseType': 'lpc'}) with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response) as mock_post: - result = is_feature_enabled('test-flag', context=context) + result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, context=context) # Verify the result self.assertTrue(result) @@ -71,7 +73,7 @@ def test_is_feature_enabled_fail_closed_on_timeout(self): from cc_common.feature_flag_client import is_feature_enabled with patch('cc_common.feature_flag_client.requests.post', side_effect=Exception('Timeout')): - result = is_feature_enabled('test-flag', fail_default=False) + result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=False) # Verify it fails closed (returns False) self.assertFalse(result) @@ -81,7 +83,7 @@ def test_is_feature_enabled_fail_open_on_timeout(self): from cc_common.feature_flag_client import is_feature_enabled with patch('cc_common.feature_flag_client.requests.post', side_effect=Exception('Timeout')): - result = is_feature_enabled('test-flag', fail_default=True) + result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=True) # Verify it fails open (returns True) self.assertTrue(result) @@ -95,7 +97,7 @@ def test_is_feature_enabled_fail_closed_on_http_error(self): mock_response.raise_for_status.side_effect = Exception('500 Server Error') with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response): - result = is_feature_enabled('test-flag', fail_default=False) + result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=False) # Verify it fails closed (returns False) self.assertFalse(result) @@ -109,7 +111,7 @@ def test_is_feature_enabled_fail_open_on_http_error(self): mock_response.raise_for_status.side_effect = Exception('500 Server Error') with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response): - result = is_feature_enabled('test-flag', fail_default=True) + result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=True) # Verify it fails open (returns True) self.assertTrue(result) @@ -124,7 +126,7 @@ def test_is_feature_enabled_fail_closed_on_invalid_response(self): mock_response.raise_for_status = MagicMock() with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response): - result = is_feature_enabled('test-flag', fail_default=False) + result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=False) # Verify it fails closed (returns False) self.assertFalse(result) @@ -139,7 +141,7 @@ def test_is_feature_enabled_fail_open_on_invalid_response(self): mock_response.raise_for_status = MagicMock() with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response): - result = is_feature_enabled('test-flag', fail_default=True) + result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=True) # Verify it fails open (returns True) self.assertTrue(result) @@ -154,7 +156,7 @@ def test_is_feature_enabled_fail_closed_on_json_parse_error(self): mock_response.raise_for_status = MagicMock() with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response): - result = is_feature_enabled('test-flag', fail_default=False) + result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=False) # Verify it fails closed (returns False) self.assertFalse(result) @@ -168,7 +170,7 @@ def test_is_feature_enabled_fail_open_on_json_parse_error(self): mock_response.json.side_effect = ValueError('Invalid JSON') with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response): - result = is_feature_enabled('test-flag', fail_default=True) + result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=True) # Verify it fails open (returns True) self.assertTrue(result) @@ -220,7 +222,7 @@ def test_is_feature_enabled_with_context_user_id_only(self): context = FeatureFlagContext(user_id='user789') with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response) as mock_post: - result = is_feature_enabled('test-flag', context=context) + result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, context=context) # Verify the result self.assertTrue(result) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/encumbrance.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/encumbrance.py index 2a394b04f..652a861b4 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/encumbrance.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/encumbrance.py @@ -103,9 +103,9 @@ def _generate_adverse_action_for_record_type( adverse_action.actionAgainst = adverse_action_against_record_type adverse_action.encumbranceType = EncumbranceType(adverse_action_request['encumbranceType']) # TODO - remove the flag conditions as part of https://github.com/csg-org/CompactConnect/issues/1136 # noqa: FIX002 - from cc_common.feature_flag_client import is_feature_enabled + from cc_common.feature_flag_client import FeatureFlagEnum, is_feature_enabled - if is_feature_enabled('encumbrance-multi-category-flag'): + if is_feature_enabled(FeatureFlagEnum.ENCUMBRANCE_MULTI_CATEGORY_FLAG): if 'clinicalPrivilegeActionCategory' in adverse_action_request: # replicate data to both the deprecated and new fields adverse_action.clinicalPrivilegeActionCategory = ClinicalPrivilegeActionCategory( From 4a9458e24a1f1e6ac08d0f9f2cded2f0df7146a2 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 30 Oct 2025 11:33:42 -0500 Subject: [PATCH 06/18] Add feature flag to codebase for checking ssn duplicates --- .../provider-data-v1/handlers/bulk_upload.py | 20 +++++++----- .../provider-data-v1/handlers/licenses.py | 31 +++++++++++-------- .../test_handlers/test_bulk_upload.py | 6 +++- .../function/test_handlers/test_licenses.py | 6 +++- 4 files changed, 41 insertions(+), 22 deletions(-) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py index 8c04fa829..7086f992a 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py @@ -13,6 +13,9 @@ ) from cc_common.event_batch_writer import EventBatchWriter from cc_common.exceptions import CCInternalException + +# initialize flag outside of handler so the flag is cached for the lifecycle of the container +from cc_common.feature_flag_client import FeatureFlagEnum, is_feature_enabled # noqa: E402 from cc_common.utils import ( ResponseEncoder, api_handler, @@ -22,6 +25,8 @@ from license_csv_reader import LicenseCSVReader from marshmallow import ValidationError +duplicate_ssn_check_flag_enabled = is_feature_enabled(FeatureFlagEnum.DUPLICATE_SSN_UPLOAD_CHECK_FLAG) + @api_handler @authorize_compact_jurisdiction(action='write') @@ -153,13 +158,14 @@ def process_bulk_upload_file( validated_license = schema.load(dict(compact=compact, jurisdiction=jurisdiction, **raw_license)) # verify that this ssn has not been used previously in the same batch license_ssn = validated_license['ssn'] - for index, record_ssn in enumerate(ssns_in_file_upload): - if license_ssn == record_ssn: - raise ValidationError( - message=f'Duplicate License SSN detected. SSN matches with record {index + 1}. ' - f'Every record must have a unique SSN within the same file.' - ) - ssns_in_file_upload.append(license_ssn) + if duplicate_ssn_check_flag_enabled: + for index, record_ssn in enumerate(ssns_in_file_upload): + if license_ssn == record_ssn: + raise ValidationError( + message=f'Duplicate License SSN detected. SSN matches with record {index + 1}. ' + f'Every record must have a unique SSN within the same file.' + ) + ssns_in_file_upload.append(license_ssn) except TypeError as e: # This will be raised, if `raw_license` includes compact and/or jurisdiction fields logger.error('License contains unsupported fields', fields=list(raw_license.keys()), exc_info=e) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py index 191176357..f8a22a4cb 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py @@ -10,6 +10,11 @@ schema = LicensePostRequestSchema() +# initialize flag outside of handler so the flag is cached for the lifecycle of the container +from cc_common.feature_flag_client import FeatureFlagEnum, is_feature_enabled # noqa: E402 + +duplicate_ssn_check_flag_enabled = is_feature_enabled(FeatureFlagEnum.DUPLICATE_SSN_UPLOAD_CHECK_FLAG) + @api_handler @optional_signature_auth @@ -63,19 +68,19 @@ def post_licenses(event: dict, context: LambdaContext): # noqa: ARG001 unused-a 'errors': invalid_records, } ) - - # verify that none of the SSNs are repeats within the same batch - license_ssns = [license_record['ssn'] for license_record in licenses] - if len(set(license_ssns)) < len(license_ssns): - raise CCInvalidRequestCustomResponseException( - response_body={ - 'message': 'Invalid license records in request. See errors for more detail.', - 'errors': { - 'SSN': 'Same SSN detected on multiple rows. ' - 'Every record must have a unique SSN within the same request.' - }, - } - ) + if duplicate_ssn_check_flag_enabled: + # verify that none of the SSNs are repeats within the same batch + license_ssns = [license_record['ssn'] for license_record in licenses] + if len(set(license_ssns)) < len(license_ssns): + raise CCInvalidRequestCustomResponseException( + response_body={ + 'message': 'Invalid license records in request. See errors for more detail.', + 'errors': { + 'SSN': 'Same SSN detected on multiple rows. ' + 'Every record must have a unique SSN within the same request.' + }, + } + ) event_time = config.current_standard_datetime diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py index 6f522dec0..7943b7971 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py @@ -1,6 +1,6 @@ import csv import json -from unittest.mock import patch +from unittest.mock import MagicMock, patch from uuid import uuid4 from botocore.exceptions import ClientError @@ -8,8 +8,12 @@ from tests.function import TstFunction +mock_flag_client = MagicMock() +mock_flag_client.return_value = True + @mock_aws +@patch('cc_common.feature_flag_client.is_feature_enabled', mock_flag_client) class TestBulkUpload(TstFunction): def test_get_bulk_upload_url(self): from handlers.bulk_upload import bulk_upload_url_handler diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_licenses.py b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_licenses.py index 49e767af4..f338eb0e6 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_licenses.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_licenses.py @@ -1,6 +1,6 @@ import json from datetime import datetime -from unittest.mock import patch +from unittest.mock import MagicMock, patch from uuid import uuid4 from common_test.sign_request import sign_request @@ -8,8 +8,12 @@ from .. import TstFunction +mock_flag_client = MagicMock() +mock_flag_client.return_value = True + @mock_aws +@patch('cc_common.feature_flag_client.is_feature_enabled', mock_flag_client) @patch('cc_common.config._Config.current_standard_datetime', datetime.fromisoformat('2024-11-08T23:59:59+00:00')) class TestLicenses(TstFunction): def setUp(self): From 39b22800ea358af5afe70cff33d9f4260fe13f2f Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 30 Oct 2025 11:39:14 -0500 Subject: [PATCH 07/18] Add feature flag to CDK stack --- .../stacks/feature_flag_stack/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/backend/compact-connect/stacks/feature_flag_stack/__init__.py b/backend/compact-connect/stacks/feature_flag_stack/__init__.py index 301d9536d..3700c5696 100644 --- a/backend/compact-connect/stacks/feature_flag_stack/__init__.py +++ b/backend/compact-connect/stacks/feature_flag_stack/__init__.py @@ -124,6 +124,20 @@ def __init__( environment_name=environment_name, ) + self.duplicate_ssn_upload_check_flag = FeatureFlagResource( + self, + 'DuplicateSsnUploadCheckFlag', + provider=self.provider, # Shared provider + flag_name='duplicate-ssn-upload-check-flag', + # Low risk update, we will automatically enable for every environment + auto_enable_envs=[ + FeatureFlagEnvironmentName.TEST, + FeatureFlagEnvironmentName.BETA, + FeatureFlagEnvironmentName.PROD, + ], + environment_name=environment_name, + ) + def _create_common_provider(self, environment_name: str) -> Provider: # Create shared Lambda function for managing all feature flags # This function is reused across all FeatureFlagResource instances From b3546866ff40284e7f51ea2e2c247c04f9692e4d Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 30 Oct 2025 11:39:47 -0500 Subject: [PATCH 08/18] formatting --- .../lambdas/python/common/cc_common/feature_flag_client.py | 5 ++++- .../lambdas/python/common/cc_common/feature_flag_enum.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_client.py b/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_client.py index a41a82d75..f435b34d6 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_client.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_client.py @@ -9,6 +9,7 @@ from typing import Any import requests + from cc_common.config import config, logger from cc_common.feature_flag_enum import FeatureFlagEnum @@ -43,7 +44,9 @@ def to_dict(self) -> dict[str, Any]: return result -def is_feature_enabled(flag_name: FeatureFlagEnum, context: FeatureFlagContext | None = None, fail_default: bool = False) -> bool: +def is_feature_enabled( + flag_name: FeatureFlagEnum, context: FeatureFlagContext | None = None, fail_default: bool = False +) -> bool: """ Check if a feature flag is enabled. diff --git a/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_enum.py b/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_enum.py index 1e49b28d9..5180b9199 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_enum.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_enum.py @@ -7,6 +7,7 @@ class FeatureFlagEnum(StrEnum): Flags should be defined here when first added, and removed when the flag is no longer in use. """ + # flag used by internal testing TEST_FLAG = 'test-flag' # runtime flags From f5b125e0e92b406a3f26ae55987876be397add4a Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 30 Oct 2025 12:49:00 -0500 Subject: [PATCH 09/18] default flag to true --- .../lambdas/python/provider-data-v1/handlers/licenses.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py index f8a22a4cb..49cc5b866 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py @@ -13,7 +13,10 @@ # initialize flag outside of handler so the flag is cached for the lifecycle of the container from cc_common.feature_flag_client import FeatureFlagEnum, is_feature_enabled # noqa: E402 -duplicate_ssn_check_flag_enabled = is_feature_enabled(FeatureFlagEnum.DUPLICATE_SSN_UPLOAD_CHECK_FLAG) +# low risk flag, so we default to enabled if failure detected +duplicate_ssn_check_flag_enabled = is_feature_enabled( + FeatureFlagEnum.DUPLICATE_SSN_UPLOAD_CHECK_FLAG, fail_default=True +) @api_handler From c3b53ad6ae31ca6e2c9a16f3f29aa32ddf8f0541 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 30 Oct 2025 12:57:49 -0500 Subject: [PATCH 10/18] Update docs to reflect unique SSN requirements --- backend/compact-connect/docs/README.md | 1 + .../compact-connect/docs/it_staff_onboarding_instructions.md | 2 ++ 2 files changed, 3 insertions(+) diff --git a/backend/compact-connect/docs/README.md b/backend/compact-connect/docs/README.md index ab0fdd159..51eacba3b 100644 --- a/backend/compact-connect/docs/README.md +++ b/backend/compact-connect/docs/README.md @@ -31,6 +31,7 @@ Export your license data to a CSV file, formatted as follows: - String lengths are enforced - exceeding them will cause validation errors - Some fields have a set list of allowed values. For those fields, make sure to enter the value exactly, including spacing and capitalization + - SSNs must be unique within a single CSV upload file. Do not include multiple rows with the same `ssn` in one file #### Field Descriptions diff --git a/backend/compact-connect/docs/it_staff_onboarding_instructions.md b/backend/compact-connect/docs/it_staff_onboarding_instructions.md index 7dfcf6b1c..910ef453a 100644 --- a/backend/compact-connect/docs/it_staff_onboarding_instructions.md +++ b/backend/compact-connect/docs/it_staff_onboarding_instructions.md @@ -240,6 +240,8 @@ For your convenience, use of this feature is included in the [Postman Collection - `licenseType` must match exactly with one of the valid types for the specified compact - All date fields must use the `YYYY-MM-DD` format - The API does not accept `null` values. For optional fields with no value, omit the field or leave it empty in CSV. +- For CSV uploads, SSNs must be unique within a single file. Do not include multiple rows with the same `ssn` in one upload +- For JSON uploads, SSNs must be unique within a single request payload (array). Do not include duplicate `ssn` values in the same batch ## Common Upload Strategies: JSON vs CSV From 03299f5752da29f5e02982da8b9a2c8313e654f0 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 30 Oct 2025 13:33:56 -0500 Subject: [PATCH 11/18] PR feedback --- .../lambdas/python/provider-data-v1/handlers/bulk_upload.py | 4 +++- .../tests/function/test_handlers/test_bulk_upload.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py index 7086f992a..bf4fd8b07 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py @@ -25,7 +25,9 @@ from license_csv_reader import LicenseCSVReader from marshmallow import ValidationError -duplicate_ssn_check_flag_enabled = is_feature_enabled(FeatureFlagEnum.DUPLICATE_SSN_UPLOAD_CHECK_FLAG) +duplicate_ssn_check_flag_enabled = is_feature_enabled( + FeatureFlagEnum.DUPLICATE_SSN_UPLOAD_CHECK_FLAG, fail_default=True +) @api_handler diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py index 7943b7971..2f32ef7c6 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py @@ -13,7 +13,6 @@ @mock_aws -@patch('cc_common.feature_flag_client.is_feature_enabled', mock_flag_client) class TestBulkUpload(TstFunction): def test_get_bulk_upload_url(self): from handlers.bulk_upload import bulk_upload_url_handler @@ -44,6 +43,7 @@ def test_get_bulk_upload_url_forbidden(self): @mock_aws +@patch('cc_common.feature_flag_client.is_feature_enabled', mock_flag_client) class TestProcessObjects(TstFunction): def test_uploaded_csv(self): from handlers.bulk_upload import parse_bulk_upload_file @@ -272,7 +272,7 @@ def test_bulk_upload_prevents_compact_jurisdiction_overwrites(self): self.assertEqual(expected_entry, call_args) def test_bulk_upload_prevents_repeated_ssns_within_the_same_file_upload(self): - """Test that CSV compact/jurisdiction fields cannot overwrite URL path values.""" + """Test that duplicate SSNs within a CSV upload are detected and rejected.""" from handlers.bulk_upload import parse_bulk_upload_file # Create CSV content that includes duplicate SSNs From 9d7e82fb12d39c8d6ba8f2909d47aa2eaf3f58c0 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 30 Oct 2025 13:34:28 -0500 Subject: [PATCH 12/18] PR feedback --- .../tests/function/test_handlers/test_bulk_upload.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py index 2f32ef7c6..865f912a4 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py @@ -289,7 +289,6 @@ def test_bulk_upload_prevents_repeated_ssns_within_the_same_file_upload(self): ) # Upload the CSV content directly to the mock S3 bucket - # URL path indicates aslp/oh, but CSV contains malicious_compact/malicious_jurisdiction object_key = f'aslp/oh/{uuid4().hex}' self._bucket.put_object(Key=object_key, Body=csv_content) From 5e630112c7cae36b3b4108cf84d1f405e2b961fd Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 30 Oct 2025 15:26:32 -0500 Subject: [PATCH 13/18] PR feedback - improve efficiency of ssn check --- .../python/provider-data-v1/handlers/bulk_upload.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py index bf4fd8b07..246814bdc 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py @@ -148,7 +148,7 @@ def process_bulk_upload_file( failed_validation_count = 0 # track which ssns were included in this file to detect duplicates, # which are not allowed within the same file upload - ssns_in_file_upload = [] + ssns_in_file_upload = {} with EventBatchWriter(config.events_client) as event_writer: for i, raw_license in enumerate(reader.licenses(stream)): @@ -161,13 +161,14 @@ def process_bulk_upload_file( # verify that this ssn has not been used previously in the same batch license_ssn = validated_license['ssn'] if duplicate_ssn_check_flag_enabled: - for index, record_ssn in enumerate(ssns_in_file_upload): - if license_ssn == record_ssn: + matched_ssn_index = ssns_in_file_upload.get(license_ssn) + if matched_ssn_index: raise ValidationError( - message=f'Duplicate License SSN detected. SSN matches with record {index + 1}. ' - f'Every record must have a unique SSN within the same file.' + message=f'Duplicate License SSN detected. SSN matches with record ' + f'{matched_ssn_index}. Every record must have a unique SSN within the same ' + f'file.' ) - ssns_in_file_upload.append(license_ssn) + ssns_in_file_upload.update({license_ssn: i + 1}) except TypeError as e: # This will be raised, if `raw_license` includes compact and/or jurisdiction fields logger.error('License contains unsupported fields', fields=list(raw_license.keys()), exc_info=e) From 9f66c2b64e8a6998d991a39cbfa49218c4f66e17 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 30 Oct 2025 16:42:14 -0500 Subject: [PATCH 14/18] Add logging/env var for feature flag checks --- backend/common-cdk/common_constructs/stack.py | 4 ++++ .../lambdas/python/common/cc_common/feature_flag_client.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/common-cdk/common_constructs/stack.py b/backend/common-cdk/common_constructs/stack.py index c38dcd994..652b0383f 100644 --- a/backend/common-cdk/common_constructs/stack.py +++ b/backend/common-cdk/common_constructs/stack.py @@ -97,6 +97,10 @@ def __init__(self, *args, environment_context: dict, environment_name: str, **kw self.environment_context = environment_context self.environment_name = environment_name + # We only set the API_BASE_URL common env var if the API_DOMAIN_NAME is set + # The API_BASE_URL is used by the feature flag client to call the flag check endpoint + if self.api_domain_name: + self.common_env_vars.update({'API_BASE_URL': f'https://{self.api_domain_name}'}) @cached_property def hosted_zone(self) -> IHostedZone | None: diff --git a/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_client.py b/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_client.py index f435b34d6..0b9bcaa21 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_client.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_client.py @@ -79,6 +79,7 @@ def is_feature_enabled( ): """ try: + logger.info("checking status of feature flag", flag_name=flag_name) api_base_url = _get_api_base_url() endpoint_url = f'{api_base_url}/v1/flags/{flag_name}/check' @@ -106,7 +107,8 @@ def is_feature_enabled( # Invalid response format - return fail_default value return fail_default - return bool(response_data['enabled']) + logger.info('Checked flag status successfully', flag_name=flag_name, enabled=response_data['enabled']) + return response_data['enabled'] # We catch all exceptions to prevent a feature flag issue causing the system from operating except Exception as e: # noqa: BLE001 From 8321f3bde21da6e9ccbb86430df60946de2c3567 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 31 Oct 2025 10:49:13 -0500 Subject: [PATCH 15/18] fix comment --- .../lambdas/python/provider-data-v1/handlers/licenses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py index 49cc5b866..405a22b5e 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py @@ -10,7 +10,7 @@ schema = LicensePostRequestSchema() -# initialize flag outside of handler so the flag is cached for the lifecycle of the container +# initialize flag outside of handler so the flag is cached for the lifecycle of the execution environment from cc_common.feature_flag_client import FeatureFlagEnum, is_feature_enabled # noqa: E402 # low risk flag, so we default to enabled if failure detected From 1b3badf56d46161c233b4660c91f56c8b34b237f Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 3 Nov 2025 09:51:36 -0600 Subject: [PATCH 16/18] PR feedback - document behavior --- backend/compact-connect/docs/README.md | 2 +- .../compact-connect/docs/it_staff_onboarding_instructions.md | 4 ++-- .../lambdas/python/provider-data-v1/handlers/bulk_upload.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/compact-connect/docs/README.md b/backend/compact-connect/docs/README.md index 51eacba3b..c78f08e0a 100644 --- a/backend/compact-connect/docs/README.md +++ b/backend/compact-connect/docs/README.md @@ -31,7 +31,7 @@ Export your license data to a CSV file, formatted as follows: - String lengths are enforced - exceeding them will cause validation errors - Some fields have a set list of allowed values. For those fields, make sure to enter the value exactly, including spacing and capitalization - - SSNs must be unique within a single CSV upload file. Do not include multiple rows with the same `ssn` in one file + - SSNs must be unique within a single CSV upload file. Do not include multiple rows with the same `ssn` in one file. If duplicate SSNs are sent within the same file, the first row to be processed, but all other duplicate rows will be rejected. #### Field Descriptions diff --git a/backend/compact-connect/docs/it_staff_onboarding_instructions.md b/backend/compact-connect/docs/it_staff_onboarding_instructions.md index 910ef453a..f386cd213 100644 --- a/backend/compact-connect/docs/it_staff_onboarding_instructions.md +++ b/backend/compact-connect/docs/it_staff_onboarding_instructions.md @@ -240,8 +240,8 @@ For your convenience, use of this feature is included in the [Postman Collection - `licenseType` must match exactly with one of the valid types for the specified compact - All date fields must use the `YYYY-MM-DD` format - The API does not accept `null` values. For optional fields with no value, omit the field or leave it empty in CSV. -- For CSV uploads, SSNs must be unique within a single file. Do not include multiple rows with the same `ssn` in one upload -- For JSON uploads, SSNs must be unique within a single request payload (array). Do not include duplicate `ssn` values in the same batch +- For CSV uploads, SSNs must be unique within a single file. Do not include multiple rows with the same `ssn` in one upload. If duplicate SSNs are sent within the same file, the first row to be processed, but all other duplicate rows will be rejected. +- For JSON uploads, SSNs must be unique within a single request payload (array). Do not include duplicate `ssn` values in the same batch. Attempting to do so will cause the entire request to be rejected. ## Common Upload Strategies: JSON vs CSV diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py index 246814bdc..9aff60105 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py @@ -14,7 +14,7 @@ from cc_common.event_batch_writer import EventBatchWriter from cc_common.exceptions import CCInternalException -# initialize flag outside of handler so the flag is cached for the lifecycle of the container +# initialize flag outside of handler so the flag is cached for the lifecycle of the lambda execution environment from cc_common.feature_flag_client import FeatureFlagEnum, is_feature_enabled # noqa: E402 from cc_common.utils import ( ResponseEncoder, From 95b71cbbef024c67f761f921816f07268ceaf159 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 3 Nov 2025 11:46:06 -0600 Subject: [PATCH 17/18] pin pip version to deal with test runner compatibility issue --- .github/workflows/check-compact-connect.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-compact-connect.yml b/.github/workflows/check-compact-connect.yml index 7902ac5a4..0b69755da 100644 --- a/.github/workflows/check-compact-connect.yml +++ b/.github/workflows/check-compact-connect.yml @@ -101,9 +101,11 @@ jobs: - name: Install dev dependencies run: "pip install -r backend/compact-connect/requirements-dev.txt" - + # Note we are currently pinning the pip version to deal with compatibility issues released with pip 25.3 + # see https://stackoverflow.com/a/79802727 If this issues is addressed in a later version, we can remove the + # extra pip install command so we stop pinning the pip version - name: Install all Python dependencies - run: "cd backend/compact-connect; bin/sync_deps.sh" + run: "cd backend/compact-connect; pip install -U 'pip<25.3'; bin/sync_deps.sh" - name: Test backend run: "cd backend/compact-connect; bin/run_tests.sh -l all -no" From 7698b358a14f6a1165524caf9881c2826b10804a Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 3 Nov 2025 11:52:10 -0600 Subject: [PATCH 18/18] PR feedback - fix grammar --- backend/compact-connect/docs/README.md | 2 +- .../compact-connect/docs/it_staff_onboarding_instructions.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/docs/README.md b/backend/compact-connect/docs/README.md index c78f08e0a..ca34259a2 100644 --- a/backend/compact-connect/docs/README.md +++ b/backend/compact-connect/docs/README.md @@ -31,7 +31,7 @@ Export your license data to a CSV file, formatted as follows: - String lengths are enforced - exceeding them will cause validation errors - Some fields have a set list of allowed values. For those fields, make sure to enter the value exactly, including spacing and capitalization - - SSNs must be unique within a single CSV upload file. Do not include multiple rows with the same `ssn` in one file. If duplicate SSNs are sent within the same file, the first row to be processed, but all other duplicate rows will be rejected. + - SSNs must be unique within a single CSV upload file. Do not include multiple rows with the same `ssn` in one file. If duplicate SSNs are sent within the same file, the first row will be processed, but all other duplicate rows will be rejected. #### Field Descriptions diff --git a/backend/compact-connect/docs/it_staff_onboarding_instructions.md b/backend/compact-connect/docs/it_staff_onboarding_instructions.md index f386cd213..45fe9642d 100644 --- a/backend/compact-connect/docs/it_staff_onboarding_instructions.md +++ b/backend/compact-connect/docs/it_staff_onboarding_instructions.md @@ -240,7 +240,7 @@ For your convenience, use of this feature is included in the [Postman Collection - `licenseType` must match exactly with one of the valid types for the specified compact - All date fields must use the `YYYY-MM-DD` format - The API does not accept `null` values. For optional fields with no value, omit the field or leave it empty in CSV. -- For CSV uploads, SSNs must be unique within a single file. Do not include multiple rows with the same `ssn` in one upload. If duplicate SSNs are sent within the same file, the first row to be processed, but all other duplicate rows will be rejected. +- For CSV uploads, SSNs must be unique within a single file. Do not include multiple rows with the same `ssn` in one upload. If duplicate SSNs are sent within the same file, the first row will be processed, but all other duplicate rows will be rejected. - For JSON uploads, SSNs must be unique within a single request payload (array). Do not include duplicate `ssn` values in the same batch. Attempting to do so will cause the entire request to be rejected. ## Common Upload Strategies: JSON vs CSV