Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 82 additions & 1 deletion python/semantic_kernel/data/vector.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,32 @@
import logging
import operator
from abc import abstractmethod
from ast import AST, Constant, Lambda, NodeTransformer, NodeVisitor, UnaryOp, USub, copy_location, expr, parse
from ast import (
AST,
And,
Attribute,
BoolOp,
Compare,
Constant,
Eq,
Gt,
GtE,
Is,
IsNot,
Lambda,
Lt,
LtE,
Name,
NodeTransformer,
NodeVisitor,
NotEq,
UnaryOp,
USub,
cmpop,
copy_location,
expr,
parse,
)
from collections.abc import AsyncIterable, Callable, Mapping, Sequence
from copy import deepcopy
from dataclasses import dataclass
Expand Down Expand Up @@ -742,6 +767,59 @@ def visit_UnaryOp(self, node: UnaryOp) -> AST:
return node


_OP_FLIP: dict[type[cmpop], type[cmpop]] = {
Eq: Eq,
NotEq: NotEq,
Lt: Gt,
LtE: GtE,
Gt: Lt,
GtE: LtE,
Is: Is,
IsNot: IsNot,
}


class _NormalizeComparisonFilters(NodeTransformer):
"""Normalize comparison filter expressions before connector parsers run.

Vector-store connector ``_lambda_parser`` implementations expect the field reference
(e.g., ``x.num``) on the left-hand side and the constant/literal on the right-hand side.
When a filter is written with the constant on the left (e.g., ``2 < x.num`` or ``3 == x.num``),
or uses Python's chained comparison syntax (e.g., ``1 <= x.num < 4``), connectors either
fail with validation errors or misinterpret the field name (#14779).

This transformer:
1. Splits chained comparisons (``len(ops) > 1``) into an explicit ``BoolOp(And, ...)``.
2. Rewrites ``Constant op Field`` into ``Field op' Constant`` (e.g., ``<`` to ``>``).
"""

def visit_Compare(self, node: Compare) -> AST:
"""Normalize chained comparisons and comparisons with constant on the left."""
self.generic_visit(node)
if len(node.ops) > 1:
pairs: list[expr] = []
for i in range(len(node.ops)):
left = node.left if i == 0 else node.comparators[i - 1]
right = node.comparators[i]
op = node.ops[i]
sub_compare = copy_location(Compare(left=left, ops=[op], comparators=[right]), node)
transformed = self.visit_Compare(sub_compare)
pairs.append(transformed if isinstance(transformed, expr) else sub_compare)
return copy_location(BoolOp(op=And(), values=pairs), node)

op_type = type(node.ops[0])
if op_type in _OP_FLIP:
left = node.left
right = node.comparators[0]
if not isinstance(left, (Attribute, Name)) and isinstance(right, (Attribute, Name)):
flipped_op = _OP_FLIP[op_type]()
return copy_location(
Compare(left=right, ops=[flipped_op], comparators=[left]),
node,
)
return node


class LambdaVisitor(NodeVisitor, Generic[TFilters]):
"""Visitor class to visit the AST nodes."""

Expand Down Expand Up @@ -2016,11 +2094,14 @@ def _build_filter(self, search_filter: OptionalOneOrMany[Callable | str] | None)

visitor = LambdaVisitor(self._lambda_parser)
folder = _FoldUnaryNumericConstants()
normalizer = _NormalizeComparisonFilters()
for filter_ in filters:
# parse lambda expression with AST
tree = parse(filter_ if isinstance(filter_, str) else getsource(filter_).strip())
# Fold negative literals so connector parsers see a numeric Constant (#14571).
tree = folder.visit(tree)
# Normalize chained comparisons and constant-on-left comparisons (#14779).
tree = normalizer.visit(tree)
visitor.visit(tree)
created_filters = visitor.output_filters
if len(created_filters) == 0:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
# Copyright (c) Microsoft. All rights reserved.

"""Regression: Vector store filters fail when constant is on the left or chained (#14779)."""

import ast
from unittest.mock import MagicMock

import pytest
from chromadb.api import ClientAPI
from qdrant_client.async_qdrant_client import AsyncQdrantClient
from qdrant_client.models import FieldCondition, Filter, MatchValue, Range

from semantic_kernel.connectors.chroma import ChromaCollection
from semantic_kernel.connectors.in_memory import InMemoryCollection
from semantic_kernel.connectors.qdrant import QdrantCollection
from semantic_kernel.data.vector import (
VectorStoreCollectionDefinition,
VectorStoreField,
_FoldUnaryNumericConstants,
_NormalizeComparisonFilters,
)


@pytest.fixture
def numeric_definition() -> VectorStoreCollectionDefinition:
return VectorStoreCollectionDefinition(
fields=[
VectorStoreField("key", name="id", type="str"),
VectorStoreField("data", name="num", type="int"),
VectorStoreField("data", name="status", type="str"),
VectorStoreField("vector", name="vector", dimensions=2, type="float"),
]
)


@pytest.mark.parametrize(
"expression,expected_unparsed",
[
# Constant on left flipped to field on left
("lambda x: 2 < x.num", "lambda x: x.num > 2"),
("lambda x: 2 <= x.num", "lambda x: x.num >= 2"),
("lambda x: 2 > x.num", "lambda x: x.num < 2"),
("lambda x: 2 >= x.num", "lambda x: x.num <= 2"),
("lambda x: 3 == x.num", "lambda x: x.num == 3"),
("lambda x: 3 != x.num", "lambda x: x.num != 3"),
("lambda x: 'active' == x.status", "lambda x: x.status == 'active'"),
("lambda x: True == x.status", "lambda x: x.status == True"),
("lambda x: None == x.status", "lambda x: x.status == None"),
("lambda num: 2 < num", "lambda num: num > 2"),
("lambda num: 3 == num", "lambda num: num == 3"),
# Field on left preserved as-is
("lambda x: x.num > 2", "lambda x: x.num > 2"),
("lambda x: x.num == 3", "lambda x: x.num == 3"),
("lambda x: x.status == 'active'", "lambda x: x.status == 'active'"),
# Chained comparisons expanded and flipped
("lambda x: 1 <= x.num < 4", "lambda x: x.num >= 1 and x.num < 4"),
("lambda x: 1 < x.num < 10", "lambda x: x.num > 1 and x.num < 10"),
("lambda x: 4 > x.num >= 1", "lambda x: x.num < 4 and x.num >= 1"),
("lambda x: 10 >= x.num > 2", "lambda x: x.num <= 10 and x.num > 2"),
# In / NotIn preserved
("lambda x: 2 in x.num", "lambda x: 2 in x.num"),
("lambda x: 'tag' not in x.status", "lambda x: 'tag' not in x.status"),
# Two fields or two constants preserved
("lambda x: x.num < x.status", "lambda x: x.num < x.status"),
("lambda x: 1 < 2", "lambda x: 1 < 2"),
],
)
def test_normalize_comparison_filters_ast(expression, expected_unparsed):
tree = ast.parse(expression)
normalized = _NormalizeComparisonFilters().visit(tree)
assert ast.unparse(normalized) == expected_unparsed


@pytest.mark.parametrize(
"expression,expected_unparsed",
[
("lambda x: -5 < x.num", "lambda x: x.num > -5"),
("lambda x: -5 <= x.num < 10", "lambda x: x.num >= -5 and x.num < 10"),
("lambda x: 10 >= x.num > -5", "lambda x: x.num <= 10 and x.num > -5"),
],
)
def test_normalize_comparison_filters_with_negative_literals(expression, expected_unparsed):
tree = ast.parse(expression)
folded = _FoldUnaryNumericConstants().visit(tree)
normalized = _NormalizeComparisonFilters().visit(folded)
assert ast.unparse(normalized) == expected_unparsed


def test_chroma_build_filter_constant_on_left(numeric_definition):
collection = ChromaCollection(
collection_name="test",
record_type=dict,
definition=numeric_definition,
client=MagicMock(spec=ClientAPI),
)
assert collection._build_filter("lambda x: 2 < x.num") == {"num": {"$gt": 2}}
assert collection._build_filter("lambda x: 3 == x.num") == {"num": 3}
assert collection._build_filter("lambda x: 1 <= x.num < 4") == {"$and": [{"num": {"$gte": 1}}, {"num": {"$lt": 4}}]}
assert collection._build_filter("lambda x: -5 < x.num") == {"num": {"$gt": -5}}


def test_qdrant_build_filter_constant_on_left(numeric_definition):
collection = QdrantCollection(
collection_name="test",
record_type=dict,
definition=numeric_definition,
client=MagicMock(spec=AsyncQdrantClient),
)
assert collection._build_filter("lambda x: 2 < x.num") == FieldCondition(key="num", range=Range(gt=2.0))
assert collection._build_filter("lambda x: 3 == x.num") == FieldCondition(key="num", match=MatchValue(value=3))
assert collection._build_filter("lambda x: 1 <= x.num < 4") == Filter(
must=[
FieldCondition(key="num", range=Range(gte=1.0)),
FieldCondition(key="num", range=Range(lt=4.0)),
]
)
assert collection._build_filter("lambda x: -5 < x.num") == FieldCondition(key="num", range=Range(gt=-5.0))


@pytest.mark.asyncio
async def test_in_memory_collection_searches_with_normalized_filters(numeric_definition):
collection = InMemoryCollection(
collection_name="test",
record_type=dict,
definition=numeric_definition,
)
records = [{"id": f"rec_{i}", "num": i, "status": "active", "vector": [1.0, 0.0]} for i in range(6)]
await collection.upsert(records)

# lambda x: 2 < x.num -> expected records 3, 4, 5
res = await collection.search(vector=[1.0, 0.0], filter="lambda x: 2 < x.num", top=10)
hits = [hit.record["num"] async for hit in res.results]
assert sorted(hits) == [3, 4, 5]

# lambda x: 3 == x.num -> expected record 3
res = await collection.search(vector=[1.0, 0.0], filter="lambda x: 3 == x.num", top=10)
hits = [hit.record["num"] async for hit in res.results]
assert hits == [3]

# lambda x: 1 <= x.num < 4 -> expected records 1, 2, 3
res = await collection.search(vector=[1.0, 0.0], filter="lambda x: 1 <= x.num < 4", top=10)
hits = [hit.record["num"] async for hit in res.results]
assert sorted(hits) == [1, 2, 3]
Loading