From 7c5b8066ee712c1264046404e1657a1b8cf4f8d4 Mon Sep 17 00:00:00 2001 From: Rohit Kumar Date: Thu, 6 Aug 2026 15:57:55 +0000 Subject: [PATCH 1/2] Support date-only version folders in list_import_summaries Add regex matching for YYYY-MM-DD version folders and increase the scan limit to 1000 to support imports with large run histories. --- .../recipes/gcp/gcs/list-import-summaries.md | 2 +- .../common/scripts/list_import_summaries.py | 6 +++- .../scripts/list_import_summaries_test.py | 28 +++++++++++++++---- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/agents/common/recipes/gcp/gcs/list-import-summaries.md b/agents/common/recipes/gcp/gcs/list-import-summaries.md index 9b3ec92cab..baa4850a9a 100644 --- a/agents/common/recipes/gcp/gcs/list-import-summaries.md +++ b/agents/common/recipes/gcp/gcs/list-import-summaries.md @@ -45,7 +45,7 @@ the version URI only when the exact summary is needed. ## Required bounds -Scan up to 100 matching summary object names plus one overflow sentinel (101 +Scan up to 1000 matching summary object names plus one overflow sentinel (1001 names maximum). Return at most five timestamp-named versions and download at most those five summaries. diff --git a/agents/common/scripts/list_import_summaries.py b/agents/common/scripts/list_import_summaries.py index 814a0e4bc1..b62e4b05b2 100644 --- a/agents/common/scripts/list_import_summaries.py +++ b/agents/common/scripts/list_import_summaries.py @@ -33,9 +33,11 @@ _VERSION_PATTERN = re.compile( r'^(?P\d{4})_(?P\d{2})_(?P\d{2})T' r'\d{2}_\d{2}_\d{2}(?:_\d{1,6})?_\d{2}_\d{2}$') +_DATE_VERSION_PATTERN = re.compile( + r'^(?P\d{4})-(?P\d{2})-(?P\d{2})$') _SUMMARY_FILENAME = 'import_summary.json' _MAX_RESULT_LIMIT = 5 -_SCAN_LIMIT = 100 +_SCAN_LIMIT = 1000 def _define_flags() -> None: @@ -79,6 +81,8 @@ def normalize_import_name(absolute_import_name: str) -> dict[str, str]: def _version_date(version: str) -> str | None: match = _VERSION_PATTERN.fullmatch(version) + if not match: + match = _DATE_VERSION_PATTERN.fullmatch(version) if not match: return None try: diff --git a/agents/common/scripts/list_import_summaries_test.py b/agents/common/scripts/list_import_summaries_test.py index 7b3b38e3c1..f5083d2b1c 100644 --- a/agents/common/scripts/list_import_summaries_test.py +++ b/agents/common/scripts/list_import_summaries_test.py @@ -78,15 +78,15 @@ def test_derives_exact_prefix_and_bounded_glob(self): client=client) self.assertEqual('scripts/a:Import', result['absolute_import_name']) - self.assertEqual(100, result['scan_limit']) + self.assertEqual(1000, result['scan_limit']) self.assertEqual(1, len(client.calls)) bucket, kwargs = client.calls[0] self.assertEqual('bucket', bucket) self.assertEqual('scripts/a/Import/', kwargs['prefix']) self.assertEqual('scripts/a/Import/*/import_summary.json', kwargs['match_glob']) - self.assertEqual(101, kwargs['max_results']) - self.assertEqual(101, kwargs['page_size']) + self.assertEqual(1001, kwargs['max_results']) + self.assertEqual(1001, kwargs['page_size']) self.assertEqual('items(name),nextPageToken', kwargs['fields']) def test_returns_newest_five_with_date_and_batch_job_id(self): @@ -184,10 +184,28 @@ def test_reports_invalid_or_mismatched_selected_summaries(self): 'summary_job_id_missing', 'summary_missing' ], [issue['code'] for issue in result['issues']]) + def test_returns_date_only_versions(self): + versions = [f'2026-08-0{day}' for day in (3, 1, 7, 2, 6, 4, 5)] + blobs = [_blob(version) for version in versions] + + result = list_import_summaries('scripts/a:Import', + 'project', + 'bucket', + client=_StorageClient(blobs)) + + self.assertEqual([ + '2026-08-07', + '2026-08-06', + '2026-08-05', + '2026-08-04', + '2026-08-03', + ], [item['version'] for item in result['results']]) + self.assertEqual('2026-08-07', result['results'][0]['date']) + def test_returns_no_history_when_scan_limit_is_exceeded(self): blobs = [ _blob(f'2026_07_{(index % 28) + 1:02d}T01_02_03_{index:06d}_07_00') - for index in range(101) + for index in range(1001) ] result = list_import_summaries('scripts/a:Import', @@ -196,7 +214,7 @@ def test_returns_no_history_when_scan_limit_is_exceeded(self): client=_StorageClient(blobs)) self.assertTrue(result['scan_truncated']) - self.assertEqual(101, result['scanned_summary_count']) + self.assertEqual(1001, result['scanned_summary_count']) self.assertEqual([], result['results']) self.assertEqual(0, sum(blob.download_count for blob in blobs)) self.assertEqual('summary_scan_limit_exceeded', From 78024818c98688fa8f448a412b54ed14cd9caf1e Mon Sep 17 00:00:00 2001 From: Rohit Kumar Date: Thu, 6 Aug 2026 22:05:59 +0530 Subject: [PATCH 2/2] Sort mixed import versions by date --- agents/common/scripts/list_import_summaries.py | 2 +- .../scripts/list_import_summaries_test.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/agents/common/scripts/list_import_summaries.py b/agents/common/scripts/list_import_summaries.py index b62e4b05b2..aaa6013557 100644 --- a/agents/common/scripts/list_import_summaries.py +++ b/agents/common/scripts/list_import_summaries.py @@ -185,7 +185,7 @@ def list_import_summaries(absolute_import_name: str, continue candidates.append((version, version_date, blob)) - candidates.sort(key=lambda item: item[0], reverse=True) + candidates.sort(key=lambda item: item[0].replace('_', '-'), reverse=True) for version, version_date, blob in candidates[:limit]: batch_job_id, issue = _read_batch_job_id(blob, version, identity['simple_import_name']) diff --git a/agents/common/scripts/list_import_summaries_test.py b/agents/common/scripts/list_import_summaries_test.py index f5083d2b1c..a699ae7b52 100644 --- a/agents/common/scripts/list_import_summaries_test.py +++ b/agents/common/scripts/list_import_summaries_test.py @@ -202,6 +202,23 @@ def test_returns_date_only_versions(self): ], [item['version'] for item in result['results']]) self.assertEqual('2026-08-07', result['results'][0]['date']) + def test_sorts_mixed_version_formats_by_date(self): + versions = [ + '2026_08_03T01_02_03_123456_07_00', + '2026-08-04', + ] + + result = list_import_summaries( + 'scripts/a:Import', + 'project', + 'bucket', + client=_StorageClient([_blob(version) for version in versions])) + + self.assertEqual([ + '2026-08-04', + '2026_08_03T01_02_03_123456_07_00', + ], [item['version'] for item in result['results']]) + def test_returns_no_history_when_scan_limit_is_exceeded(self): blobs = [ _blob(f'2026_07_{(index % 28) + 1:02d}T01_02_03_{index:06d}_07_00')