From a89b4825b5bf61fe060ea95a88beb838487e2e55 Mon Sep 17 00:00:00 2001 From: Hardik Kaurani Date: Wed, 11 Nov 2026 11:25:49 +0530 Subject: [PATCH 1/5] feat(data): add AST comparison operator inversion mapping Signed-off-by: Hardik Kaurani --- python/semantic_kernel/data/vector.py | 30 ++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/python/semantic_kernel/data/vector.py b/python/semantic_kernel/data/vector.py index 69bdc78a938e..ac655128e7bf 100644 --- a/python/semantic_kernel/data/vector.py +++ b/python/semantic_kernel/data/vector.py @@ -4,7 +4,25 @@ import logging import operator from abc import abstractmethod -from ast import AST, Constant, Lambda, NodeTransformer, NodeVisitor, UnaryOp, USub, copy_location, expr, parse +from ast import ( + AST, + Constant, + Eq, + Gt, + GtE, + Lambda, + Lt, + LtE, + NodeTransformer, + NodeVisitor, + NotEq, + UnaryOp, + USub, + cmpop, + copy_location, + expr, + parse, +) from collections.abc import AsyncIterable, Callable, Mapping, Sequence from copy import deepcopy from dataclasses import dataclass @@ -742,6 +760,16 @@ def visit_UnaryOp(self, node: UnaryOp) -> AST: return node +_OP_INVERSION: Final[dict[type[cmpop], type[cmpop]]] = { + Lt: Gt, + LtE: GtE, + Gt: Lt, + GtE: LtE, + Eq: Eq, + NotEq: NotEq, +} + + class LambdaVisitor(NodeVisitor, Generic[TFilters]): """Visitor class to visit the AST nodes.""" From f4654cbbf127be0978a22460894d346a79bb0d80 Mon Sep 17 00:00:00 2001 From: Hardik Kaurani Date: Wed, 11 Nov 2026 11:26:33 +0530 Subject: [PATCH 2/5] feat(data): add _NormalizeComparisonConstants node transformer for filter lambdas Signed-off-by: Hardik Kaurani --- python/semantic_kernel/data/vector.py | 55 +++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/python/semantic_kernel/data/vector.py b/python/semantic_kernel/data/vector.py index ac655128e7bf..0f0649e13403 100644 --- a/python/semantic_kernel/data/vector.py +++ b/python/semantic_kernel/data/vector.py @@ -6,6 +6,10 @@ from abc import abstractmethod from ast import ( AST, + And, + Attribute, + BoolOp, + Compare, Constant, Eq, Gt, @@ -13,6 +17,7 @@ Lambda, Lt, LtE, + Name, NodeTransformer, NodeVisitor, NotEq, @@ -770,6 +775,56 @@ def visit_UnaryOp(self, node: UnaryOp) -> AST: } +class _NormalizeComparisonConstants(NodeTransformer): + """Normalize constant-on-left comparisons and chained comparisons in filter lambdas. + + Vector-store ``_lambda_parser`` implementations assume comparison operations have the + record property on the left and the constant value on the right (e.g. ``x.price > 5``). + When a user writes a comparison with the constant on the left (e.g. ``5 < x.price`` or + ``10 == x.id``) or writes a chained range comparison (e.g. ``0 <= x.price < 5``), + connectors either produce malformed query filters (e.g. Chroma/MongoDB using the constant + as field key), fail schema validation (e.g. Qdrant FieldCondition), or raise + NotImplementedError (e.g. Weaviate). + + Normalizing these comparisons into canonical ``property op constant`` comparisons and + decomposing chained comparisons into logical ANDs ensures consistent behavior across + all vector database connectors. + """ + + def _normalize_single(self, node: Compare) -> Compare: + """Invert a single comparison if the constant is on the left and property on the right.""" + if ( + len(node.ops) == 1 + and type(node.ops[0]) in _OP_INVERSION + and isinstance(node.left, Constant) + and isinstance(node.comparators[0], (Attribute, Name)) + ): + inv_op_cls = _OP_INVERSION[type(node.ops[0])] + inv_op = copy_location(inv_op_cls(), node.ops[0]) + return copy_location( + Compare(left=node.comparators[0], ops=[inv_op], comparators=[node.left]), + node, + ) + return node + + def visit_Compare(self, node: Compare) -> AST: + """Normalize chained comparisons into BoolOp(And) and flip constant-on-left operands.""" + self.generic_visit(node) + if len(node.ops) > 1: + values: list[expr] = [] + for idx in range(len(node.ops)): + left = node.left if idx == 0 else node.comparators[idx - 1] + right = node.comparators[idx] + op = node.ops[idx] + sub = copy_location( + Compare(left=left, ops=[op], comparators=[right]), + node, + ) + values.append(self._normalize_single(sub)) + return copy_location(BoolOp(op=And(), values=values), node) + return self._normalize_single(node) + + class LambdaVisitor(NodeVisitor, Generic[TFilters]): """Visitor class to visit the AST nodes.""" From 92737c21cc404c45f4a762927133b2c21c9b3259 Mon Sep 17 00:00:00 2001 From: Hardik Kaurani Date: Wed, 11 Nov 2026 11:27:04 +0530 Subject: [PATCH 3/5] fix(data): normalize constant-on-left and chained comparisons in VectorSearch._build_filter Signed-off-by: Hardik Kaurani --- python/semantic_kernel/data/vector.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/semantic_kernel/data/vector.py b/python/semantic_kernel/data/vector.py index 0f0649e13403..4cee606e29df 100644 --- a/python/semantic_kernel/data/vector.py +++ b/python/semantic_kernel/data/vector.py @@ -2099,11 +2099,14 @@ def _build_filter(self, search_filter: OptionalOneOrMany[Callable | str] | None) visitor = LambdaVisitor(self._lambda_parser) folder = _FoldUnaryNumericConstants() + normalizer = _NormalizeComparisonConstants() for filter_ in filters: # parse lambda expression with AST tree = parse(filter_ if isinstance(filter_, str) else getsource(filter_).strip()) # Fold negative literals so connector parsers see a numeric Constant (#14571). tree = folder.visit(tree) + # Normalize constant-on-left comparisons and decompose chained comparisons. + tree = normalizer.visit(tree) visitor.visit(tree) created_filters = visitor.output_filters if len(created_filters) == 0: From 67a9bbdd4bfb2fbf3306c409c0a69488af008eb6 Mon Sep 17 00:00:00 2001 From: Hardik Kaurani Date: Wed, 11 Nov 2026 11:27:37 +0530 Subject: [PATCH 4/5] test(data): add unit tests for _NormalizeComparisonConstants AST transformation Signed-off-by: Hardik Kaurani --- .../test_vector_comparison_normalization.py | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 python/tests/unit/data/test_vector_comparison_normalization.py diff --git a/python/tests/unit/data/test_vector_comparison_normalization.py b/python/tests/unit/data/test_vector_comparison_normalization.py new file mode 100644 index 000000000000..3a71189a10b4 --- /dev/null +++ b/python/tests/unit/data/test_vector_comparison_normalization.py @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Unit and integration tests for comparison normalization in vector store filter lambdas.""" + +import ast +from ast import Attribute, BoolOp, Compare, Constant, Eq, Gt, GtE, Lt, LtE, Name, NotEq, parse + +import pytest + +from semantic_kernel.data.vector import ( + _FoldUnaryNumericConstants, + _NormalizeComparisonConstants, +) + + +@pytest.mark.parametrize( + "expr_str,expected_op,expected_constant", + [ + ("lambda x: 5 < x.price", Gt, 5), + ("lambda x: 5 <= x.price", GtE, 5), + ("lambda x: 5 > x.price", Lt, 5), + ("lambda x: 5 >= x.price", LtE, 5), + ('lambda x: "active" == x.status', Eq, "active"), + ("lambda x: 10 != x.id", NotEq, 10), + ], +) +def test_normalize_single_comparison_flips_constant_on_left(expr_str, expected_op, expected_constant): + tree = parse(expr_str) + transformed = _NormalizeComparisonConstants().visit(tree) + compare = transformed.body[0].value.body + + assert isinstance(compare, Compare) + assert len(compare.ops) == 1 + assert isinstance(compare.ops[0], expected_op) + assert isinstance(compare.left, Attribute) + assert isinstance(compare.comparators[0], Constant) + assert compare.comparators[0].value == expected_constant + + +def test_normalize_single_comparison_works_with_bare_name(): + tree = parse("lambda price: 5 > price") + transformed = _NormalizeComparisonConstants().visit(tree) + compare = transformed.body[0].value.body + + assert isinstance(compare, Compare) + assert isinstance(compare.ops[0], Lt) + assert isinstance(compare.left, Name) + assert compare.left.id == "price" + assert isinstance(compare.comparators[0], Constant) + assert compare.comparators[0].value == 5 + + +def test_normalize_folded_negative_constant_on_left(): + tree = parse("lambda x: -5 > x.price") + tree = _FoldUnaryNumericConstants().visit(tree) + transformed = _NormalizeComparisonConstants().visit(tree) + compare = transformed.body[0].value.body + + assert isinstance(compare, Compare) + assert isinstance(compare.ops[0], Lt) + assert isinstance(compare.left, Attribute) + assert isinstance(compare.comparators[0], Constant) + assert compare.comparators[0].value == -5 + + +@pytest.mark.parametrize( + "expr_str", + [ + "lambda x: x.price > 5", + "lambda x: x.price <= 5", + 'lambda x: x.status == "active"', + "lambda x: x.id != 10", + ], +) +def test_normalize_preserves_constant_on_right(expr_str): + tree = parse(expr_str) + original_dump = ast.dump(tree) + transformed = _NormalizeComparisonConstants().visit(tree) + assert ast.dump(transformed) == original_dump + + +@pytest.mark.parametrize( + "expr_str", + [ + 'lambda x: x.tag in ["a", "b"]', + 'lambda x: "a" in x.tags', + "lambda x: x.price == x.discount", + "lambda x: 5 == 5", + ], +) +def test_normalize_preserves_non_invertible_comparisons(expr_str): + tree = parse(expr_str) + original_dump = ast.dump(tree) + transformed = _NormalizeComparisonConstants().visit(tree) + assert ast.dump(transformed) == original_dump + + +def test_normalize_chained_comparisons_decomposes_into_and(): + tree = parse("lambda x: 0 <= x.price < 5") + transformed = _NormalizeComparisonConstants().visit(tree) + body = transformed.body[0].value.body + + assert isinstance(body, BoolOp) + assert isinstance(body.op, ast.And) + assert len(body.values) == 2 + + # First sub-comparison: 0 <= x.price -> x.price >= 0 + first = body.values[0] + assert isinstance(first, Compare) + assert isinstance(first.ops[0], GtE) + assert isinstance(first.left, Attribute) + assert first.left.attr == "price" + assert isinstance(first.comparators[0], Constant) + assert first.comparators[0].value == 0 + + # Second sub-comparison: x.price < 5 + second = body.values[1] + assert isinstance(second, Compare) + assert isinstance(second.ops[0], Lt) + assert isinstance(second.left, Attribute) + assert second.left.attr == "price" + assert isinstance(second.comparators[0], Constant) + assert second.comparators[0].value == 5 + + +def test_normalize_chained_comparisons_with_folded_negative_literal(): + tree = parse("lambda x: -10 < x.temperature <= 20") + tree = _FoldUnaryNumericConstants().visit(tree) + transformed = _NormalizeComparisonConstants().visit(tree) + body = transformed.body[0].value.body + + assert isinstance(body, BoolOp) + assert isinstance(body.op, ast.And) + assert len(body.values) == 2 + + first = body.values[0] + assert isinstance(first, Compare) + assert isinstance(first.ops[0], Gt) + assert isinstance(first.left, Attribute) + assert first.left.attr == "temperature" + assert first.comparators[0].value == -10 + + second = body.values[1] + assert isinstance(second, Compare) + assert isinstance(second.ops[0], LtE) + assert isinstance(second.left, Attribute) + assert second.left.attr == "temperature" + assert second.comparators[0].value == 20 From f236bfff8195b71add3404b488a405b3d1f0f27f Mon Sep 17 00:00:00 2001 From: Hardik Kaurani Date: Wed, 11 Nov 2026 11:30:21 +0530 Subject: [PATCH 5/5] test(data): add connector integration tests for normalized comparisons across Chroma, Qdrant, and Weaviate Signed-off-by: Hardik Kaurani --- .../test_vector_comparison_normalization.py | 73 ++++++++++++++++++- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/python/tests/unit/data/test_vector_comparison_normalization.py b/python/tests/unit/data/test_vector_comparison_normalization.py index 3a71189a10b4..d876735c80f3 100644 --- a/python/tests/unit/data/test_vector_comparison_normalization.py +++ b/python/tests/unit/data/test_vector_comparison_normalization.py @@ -4,15 +4,39 @@ import ast from ast import Attribute, BoolOp, Compare, Constant, Eq, Gt, GtE, Lt, LtE, Name, NotEq, parse +from unittest.mock import AsyncMock, MagicMock import pytest - +from chromadb.api import ClientAPI +from qdrant_client.async_qdrant_client import AsyncQdrantClient +from qdrant_client.models import FieldCondition, Range +from qdrant_client.models import Filter as QdrantFilter +from weaviate import WeaviateAsyncClient +from weaviate.classes.query import Filter as WeaviateFilter + +from semantic_kernel.connectors.chroma import ChromaCollection +from semantic_kernel.connectors.qdrant import QdrantCollection +from semantic_kernel.connectors.weaviate import WeaviateCollection from semantic_kernel.data.vector import ( + VectorStoreCollectionDefinition, + VectorStoreField, _FoldUnaryNumericConstants, _NormalizeComparisonConstants, ) +@pytest.fixture +def numeric_definition() -> VectorStoreCollectionDefinition: + return VectorStoreCollectionDefinition( + fields=[ + VectorStoreField("key", name="id", type="str"), + VectorStoreField("data", name="price", type="float"), + VectorStoreField("data", name="temperature", type="float"), + VectorStoreField("vector", name="vector", dimensions=2, type="float"), + ] + ) + + @pytest.mark.parametrize( "expr_str,expected_op,expected_constant", [ @@ -104,7 +128,6 @@ def test_normalize_chained_comparisons_decomposes_into_and(): assert isinstance(body.op, ast.And) assert len(body.values) == 2 - # First sub-comparison: 0 <= x.price -> x.price >= 0 first = body.values[0] assert isinstance(first, Compare) assert isinstance(first.ops[0], GtE) @@ -113,7 +136,6 @@ def test_normalize_chained_comparisons_decomposes_into_and(): assert isinstance(first.comparators[0], Constant) assert first.comparators[0].value == 0 - # Second sub-comparison: x.price < 5 second = body.values[1] assert isinstance(second, Compare) assert isinstance(second.ops[0], Lt) @@ -146,3 +168,48 @@ def test_normalize_chained_comparisons_with_folded_negative_literal(): assert isinstance(second.left, Attribute) assert second.left.attr == "temperature" assert second.comparators[0].value == 20 + + +def test_chroma_build_filter_normalized_comparisons(numeric_definition): + collection = ChromaCollection( + collection_name="temps", + record_type=dict, + definition=numeric_definition, + client=MagicMock(spec=ClientAPI), + ) + assert collection._build_filter("lambda x: 5 > x.price") == {"price": {"$lt": 5}} + assert collection._build_filter("lambda x: 0 <= x.price < 5") == { + "$and": [{"price": {"$gte": 0}}, {"price": {"$lt": 5}}] + } + + +def test_qdrant_build_filter_normalized_comparisons(numeric_definition): + collection = QdrantCollection( + collection_name="temps", + record_type=dict, + definition=numeric_definition, + client=MagicMock(spec=AsyncQdrantClient), + ) + assert collection._build_filter("lambda x: 5 > x.price") == FieldCondition(key="price", range=Range(lt=5.0)) + assert collection._build_filter("lambda x: 0 <= x.price < 5") == QdrantFilter( + must=[ + FieldCondition(key="price", range=Range(gte=0.0)), + FieldCondition(key="price", range=Range(lt=5.0)), + ] + ) + + +def test_weaviate_build_filter_normalized_comparisons(numeric_definition): + collection = WeaviateCollection( + collection_name="temps", + record_type=dict, + definition=numeric_definition, + async_client=AsyncMock(spec=WeaviateAsyncClient), + ) + assert collection._build_filter("lambda x: 5 > x.price") == WeaviateFilter.by_property("price").less_than(5) + parsed = collection._build_filter("lambda x: 0 <= x.price < 5") + expected = WeaviateFilter.all_of([ + WeaviateFilter.by_property("price").greater_or_equal(0), + WeaviateFilter.by_property("price").less_than(5), + ]) + assert parsed.filters == expected.filters