From 3b14645e6e8b32f193545bca6b8fb741949bf936 Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Tue, 28 Jul 2026 09:37:58 +0000 Subject: [PATCH] fix(benchmark): make prompt cache instantiation atomic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit __instantiate_prompts reset the five prompt caches and appended to them in place while paginating. Any failure partway through — a 503 on a later page, a schema-drift parse error on one item — left the caches partially populated, and every cache getter treats a non-empty list as a complete, authoritative set. The result was a silently truncated benchmark: e.g. .prompts raising, then .identifiers returning a single entry instead of re-fetching or raising, which later surfaced as a misleading "all identifiers must be registered" error rather than the real fetch failure. Accumulate into locals and assign to self only after every page has been fetched and parsed, so the caches are all-or-nothing. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: mads <37117211+Makuh17@users.noreply.github.com> --- .../benchmark/rapidata_benchmark.py | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py index 4e9d61fbe..a495551a9 100644 --- a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py +++ b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py @@ -69,11 +69,11 @@ def __instantiate_prompts(self) -> None: ) with tracer.start_as_current_span("RapidataBenchmark.__instantiate_prompts"): - self.__prompts = [] - self.__english_prompts = [] - self.__identifiers = [] - self.__prompt_assets = [] - self.__tags = [] + prompts: list[str | None] = [] + english_prompts: list[str | None] = [] + identifiers: list[str] = [] + prompt_assets: list[str | None] = [] + tags: list[list[str]] = [] current_page = 1 total_pages = None @@ -93,11 +93,11 @@ def __instantiate_prompts(self) -> None: total_pages = prompts_result.total_pages for prompt in prompts_result.items: - self.__prompts.append(prompt.original_prompt) - self.__english_prompts.append(prompt.english_prompt) - self.__identifiers.append(prompt.identifier) + prompts.append(prompt.original_prompt) + english_prompts.append(prompt.english_prompt) + identifiers.append(prompt.identifier) if prompt.prompt_asset is None: - self.__prompt_assets.append(None) + prompt_assets.append(None) else: file_asset = prompt.prompt_asset.actual_instance assert isinstance(file_asset, IAssetModelFileAssetModel) @@ -108,22 +108,31 @@ def __instantiate_prompts(self) -> None: assert isinstance( instance, IMetadataModelSourceUrlMetadataModel ) - self.__prompt_assets.append(instance.url) + prompt_assets.append(instance.url) elif original_filename is not None: instance = original_filename.actual_instance assert isinstance( instance, IMetadataModelOriginalFilenameMetadataModel ) - self.__prompt_assets.append(instance.original_filename) + prompt_assets.append(instance.original_filename) else: - self.__prompt_assets.append(None) + prompt_assets.append(None) - self.__tags.append([tag.value for tag in prompt.tags]) + tags.append([tag.value for tag in prompt.tags]) if current_page >= total_pages: break current_page += 1 + # Commit the caches only once every page fetched and parsed cleanly. + # A mid-pagination failure would otherwise leave them partially filled, + # and the property getters treat any non-empty cache as complete. + self.__prompts = prompts + self.__english_prompts = english_prompts + self.__identifiers = identifiers + self.__prompt_assets = prompt_assets + self.__tags = tags + # http / https in any case — same detection the asset uploader uses to tell # a remote URL from a local path. __URL_SCHEME_RE = re.compile(r"^https?://", re.IGNORECASE)