Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 23 additions & 12 deletions python/semantic_kernel/connectors/chroma.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,24 +209,33 @@ def _validate_data_model(self):
f"Chroma only supports one vector field, but {len(self.definition.vector_fields)} were provided."
)

def _storage_keyed(self, record: dict[str, Any]) -> dict[str, Any]:
"""Return the record keyed by storage names.

Dict and dataclass records arrive keyed by storage name, pydantic models keyed by field name.
"""
keyed = dict(record)
for field in self.definition.fields:
storage_name = field.storage_name or field.name
if storage_name != field.name and storage_name not in keyed and field.name in keyed:
keyed[storage_name] = keyed.pop(field.name)
return keyed

@override
def _serialize_dicts_to_store_models(self, records: Sequence[dict[str, Any]], **kwargs: Any) -> Sequence[Any]:
vector_field = self.definition.vector_fields[0]
id_field_name = self.definition.key_name
id_field_name = self.definition.key_field_storage_name
vector_field_name = vector_field.storage_name or vector_field.name
store_models = []
for record in records:
for record in map(self._storage_keyed, records):
store_model = {
"id": record[id_field_name],
"metadata": {
k: v
for k, v in record.items()
if k not in [id_field_name, vector_field.storage_name or vector_field.name]
},
"metadata": {k: v for k, v in record.items() if k not in [id_field_name, vector_field_name]},
}
if self.embedding_func:
store_model["document"] = (record[vector_field.storage_name or vector_field.name],)
store_model["document"] = (record[vector_field_name],)
else:
store_model["embedding"] = record[vector_field.storage_name or vector_field.name]
store_model["embedding"] = record[vector_field_name]
if store_model["metadata"] == {}:
store_model.pop("metadata")
store_models.append(store_model)
Expand All @@ -235,10 +244,12 @@ def _serialize_dicts_to_store_models(self, records: Sequence[dict[str, Any]], **
@override
def _deserialize_store_models_to_dicts(self, records: Sequence[Any], **kwargs: Any) -> Sequence[dict[str, Any]]:
vector_field = self.definition.vector_fields[0]
# replace back the name of the vector, content and id fields
# replace back the storage name of the vector, content and id fields
for record in records:
record[self.definition.key_name] = record.pop("id")
record[vector_field.name] = record.pop("document", None) or record.pop("embedding", None)
record[self.definition.key_field_storage_name] = record.pop("id")
record[vector_field.storage_name or vector_field.name] = record.pop("document", None) or record.pop(
"embedding", None
)
return records

@override
Expand Down
67 changes: 67 additions & 0 deletions python/tests/unit/connectors/memory/test_chroma.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
# Copyright (c) Microsoft. All rights reserved.

import uuid
from dataclasses import dataclass
from unittest.mock import MagicMock

import chromadb
import pytest
from chromadb.api import ClientAPI
from chromadb.api.models.Collection import Collection
from chromadb.errors import NotFoundError
from pydantic import BaseModel

from semantic_kernel.connectors.chroma import ChromaCollection, ChromaStore
from semantic_kernel.data.vector import VectorStoreCollectionDefinition, VectorStoreField
from semantic_kernel.exceptions.vector_store_exceptions import VectorStoreOperationException


Expand Down Expand Up @@ -180,3 +185,65 @@ def test_chroma_collection_build_filter_keeps_apostrophes(chroma_collection):
"""Chroma filters are dicts, not SQL, so the value must reach Chroma unchanged."""
assert chroma_collection._build_filter('lambda x: x.content == "O\'Brien"') == {"content": "O'Brien"}
assert chroma_collection._build_filter('lambda x: x.content != "O\'Brien"') == {"content": {"$ne": "O'Brien"}}


@dataclass
class AliasedDataclass:
id: str
text: str
vector: list[float] | None = None


class AliasedModel(BaseModel):
id: str
text: str
vector: list[float] | None = None


@pytest.mark.parametrize("key_storage_name", [None, "point_id"], ids=["plain_key", "aliased_key"])
@pytest.mark.parametrize("record_type", [dict, AliasedDataclass, AliasedModel])
async def test_storage_names_round_trip(record_type, key_storage_name):
definition = VectorStoreCollectionDefinition(
fields=[
VectorStoreField("key", name="id", type="str", storage_name=key_storage_name),
VectorStoreField("data", name="text", type="str", storage_name="doc_text", is_indexed=True),
VectorStoreField("vector", name="vector", dimensions=2, type="float", storage_name="doc_vector"),
]
)

def make(record_id: str, text: str, vector: list[float]):
values = {"id": record_id, "text": text, "vector": vector}
return values if record_type is dict else record_type(**values)

def as_dict(record):
if isinstance(record, dict):
values = dict(record)
else:
values = dict(vars(record)) if record_type is AliasedDataclass else record.model_dump()
if values.get("vector") is not None:
values["vector"] = [float(x) for x in values["vector"]]
return values

collection = ChromaCollection(
record_type=record_type,
definition=definition,
collection_name=f"test-{uuid.uuid4().hex}",
client=chromadb.EphemeralClient(),
)
await collection.ensure_collection_exists()
try:
await collection.upsert([make("k1", "a", [1.0, 0.0]), make("k2", "b", [0.0, 1.0])])

stored = as_dict(await collection.get("k1", include_vectors=True))
assert stored == {"id": "k1", "text": "a", "vector": [1.0, 0.0]}

results = await collection.search(vector=[0.0, 1.0], include_vectors=True)
found = [as_dict(r.record) async for r in results.results]
assert [r["id"] for r in found] == ["k2", "k1"]
assert found[0] == {"id": "k2", "text": "b", "vector": [0.0, 1.0]}

# filters use the storage names, so the metadata must be stored under them for every record type
results = await collection.search(vector=[0.0, 1.0], filter="lambda x: x.doc_text == 'a'")
assert [as_dict(r.record)["id"] async for r in results.results] == ["k1"]
finally:
await collection.ensure_collection_deleted()