From 6f686fa64c1f0b2ea5865cb0dd97f9d12ddd1f0b Mon Sep 17 00:00:00 2001 From: VANDRANKI Date: Sun, 8 Nov 2026 15:02:49 -0500 Subject: [PATCH] Python: key Chroma records by storage name when serializing and deserializing (#14755) Dict and dataclass records reach the Chroma serializer keyed by storage name and pydantic models keyed by field name. The serializer read the key by declared name and the vector by storage name, so a dict or dataclass record with a storage_name on the key field failed to upsert and a pydantic model with a storage_name on the vector field failed too. The deserializer returned the key and the vector under the declared names, which the data model step does not read for dict and dataclass records, so a vector with a storage_name came back as None. Pydantic metadata was stored under the declared names, so filters on storage names did not match it. Closes #14755 Co-authored-by: VANDRANKI --- python/semantic_kernel/connectors/chroma.py | 35 ++++++---- .../unit/connectors/memory/test_chroma.py | 67 +++++++++++++++++++ 2 files changed, 90 insertions(+), 12 deletions(-) diff --git a/python/semantic_kernel/connectors/chroma.py b/python/semantic_kernel/connectors/chroma.py index b5c821b40b25..abb1a4699cbd 100644 --- a/python/semantic_kernel/connectors/chroma.py +++ b/python/semantic_kernel/connectors/chroma.py @@ -209,24 +209,33 @@ def _validate_data_model(self): f"Chroma only supports one vector field, but {len(self.definition.vector_fields)} were provided." ) + def _storage_keyed(self, record: dict[str, Any]) -> dict[str, Any]: + """Return the record keyed by storage names. + + Dict and dataclass records arrive keyed by storage name, pydantic models keyed by field name. + """ + keyed = dict(record) + for field in self.definition.fields: + storage_name = field.storage_name or field.name + if storage_name != field.name and storage_name not in keyed and field.name in keyed: + keyed[storage_name] = keyed.pop(field.name) + return keyed + @override def _serialize_dicts_to_store_models(self, records: Sequence[dict[str, Any]], **kwargs: Any) -> Sequence[Any]: vector_field = self.definition.vector_fields[0] - id_field_name = self.definition.key_name + id_field_name = self.definition.key_field_storage_name + vector_field_name = vector_field.storage_name or vector_field.name store_models = [] - for record in records: + for record in map(self._storage_keyed, records): store_model = { "id": record[id_field_name], - "metadata": { - k: v - for k, v in record.items() - if k not in [id_field_name, vector_field.storage_name or vector_field.name] - }, + "metadata": {k: v for k, v in record.items() if k not in [id_field_name, vector_field_name]}, } if self.embedding_func: - store_model["document"] = (record[vector_field.storage_name or vector_field.name],) + store_model["document"] = (record[vector_field_name],) else: - store_model["embedding"] = record[vector_field.storage_name or vector_field.name] + store_model["embedding"] = record[vector_field_name] if store_model["metadata"] == {}: store_model.pop("metadata") store_models.append(store_model) @@ -235,10 +244,12 @@ def _serialize_dicts_to_store_models(self, records: Sequence[dict[str, Any]], ** @override def _deserialize_store_models_to_dicts(self, records: Sequence[Any], **kwargs: Any) -> Sequence[dict[str, Any]]: vector_field = self.definition.vector_fields[0] - # replace back the name of the vector, content and id fields + # replace back the storage name of the vector, content and id fields for record in records: - record[self.definition.key_name] = record.pop("id") - record[vector_field.name] = record.pop("document", None) or record.pop("embedding", None) + record[self.definition.key_field_storage_name] = record.pop("id") + record[vector_field.storage_name or vector_field.name] = record.pop("document", None) or record.pop( + "embedding", None + ) return records @override diff --git a/python/tests/unit/connectors/memory/test_chroma.py b/python/tests/unit/connectors/memory/test_chroma.py index 1d944b8abb0a..c582c5dea0fb 100644 --- a/python/tests/unit/connectors/memory/test_chroma.py +++ b/python/tests/unit/connectors/memory/test_chroma.py @@ -1,13 +1,18 @@ # Copyright (c) Microsoft. All rights reserved. +import uuid +from dataclasses import dataclass from unittest.mock import MagicMock +import chromadb import pytest from chromadb.api import ClientAPI from chromadb.api.models.Collection import Collection from chromadb.errors import NotFoundError +from pydantic import BaseModel from semantic_kernel.connectors.chroma import ChromaCollection, ChromaStore +from semantic_kernel.data.vector import VectorStoreCollectionDefinition, VectorStoreField from semantic_kernel.exceptions.vector_store_exceptions import VectorStoreOperationException @@ -180,3 +185,65 @@ def test_chroma_collection_build_filter_keeps_apostrophes(chroma_collection): """Chroma filters are dicts, not SQL, so the value must reach Chroma unchanged.""" assert chroma_collection._build_filter('lambda x: x.content == "O\'Brien"') == {"content": "O'Brien"} assert chroma_collection._build_filter('lambda x: x.content != "O\'Brien"') == {"content": {"$ne": "O'Brien"}} + + +@dataclass +class AliasedDataclass: + id: str + text: str + vector: list[float] | None = None + + +class AliasedModel(BaseModel): + id: str + text: str + vector: list[float] | None = None + + +@pytest.mark.parametrize("key_storage_name", [None, "point_id"], ids=["plain_key", "aliased_key"]) +@pytest.mark.parametrize("record_type", [dict, AliasedDataclass, AliasedModel]) +async def test_storage_names_round_trip(record_type, key_storage_name): + definition = VectorStoreCollectionDefinition( + fields=[ + VectorStoreField("key", name="id", type="str", storage_name=key_storage_name), + VectorStoreField("data", name="text", type="str", storage_name="doc_text", is_indexed=True), + VectorStoreField("vector", name="vector", dimensions=2, type="float", storage_name="doc_vector"), + ] + ) + + def make(record_id: str, text: str, vector: list[float]): + values = {"id": record_id, "text": text, "vector": vector} + return values if record_type is dict else record_type(**values) + + def as_dict(record): + if isinstance(record, dict): + values = dict(record) + else: + values = dict(vars(record)) if record_type is AliasedDataclass else record.model_dump() + if values.get("vector") is not None: + values["vector"] = [float(x) for x in values["vector"]] + return values + + collection = ChromaCollection( + record_type=record_type, + definition=definition, + collection_name=f"test-{uuid.uuid4().hex}", + client=chromadb.EphemeralClient(), + ) + await collection.ensure_collection_exists() + try: + await collection.upsert([make("k1", "a", [1.0, 0.0]), make("k2", "b", [0.0, 1.0])]) + + stored = as_dict(await collection.get("k1", include_vectors=True)) + assert stored == {"id": "k1", "text": "a", "vector": [1.0, 0.0]} + + results = await collection.search(vector=[0.0, 1.0], include_vectors=True) + found = [as_dict(r.record) async for r in results.results] + assert [r["id"] for r in found] == ["k2", "k1"] + assert found[0] == {"id": "k2", "text": "b", "vector": [0.0, 1.0]} + + # filters use the storage names, so the metadata must be stored under them for every record type + results = await collection.search(vector=[0.0, 1.0], filter="lambda x: x.doc_text == 'a'") + assert [as_dict(r.record)["id"] async for r in results.results] == ["k1"] + finally: + await collection.ensure_collection_deleted()