From c54f8f7a60a90223ef4ae558dd42146c09e9d896 Mon Sep 17 00:00:00 2001 From: Wasim Date: Wed, 11 Nov 2026 13:29:03 +0530 Subject: [PATCH] Python: normalize comparison filters with constant on left and chained comparisons (#14779) --- python/semantic_kernel/data/vector.py | 83 +++++++++- ..._vector_filter_comparison_normalization.py | 143 ++++++++++++++++++ 2 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 python/tests/unit/data/test_vector_filter_comparison_normalization.py diff --git a/python/semantic_kernel/data/vector.py b/python/semantic_kernel/data/vector.py index 69bdc78a938e..e0a4786235b0 100644 --- a/python/semantic_kernel/data/vector.py +++ b/python/semantic_kernel/data/vector.py @@ -4,7 +4,32 @@ import logging import operator from abc import abstractmethod -from ast import AST, Constant, Lambda, NodeTransformer, NodeVisitor, UnaryOp, USub, copy_location, expr, parse +from ast import ( + AST, + And, + Attribute, + BoolOp, + Compare, + Constant, + Eq, + Gt, + GtE, + Is, + IsNot, + Lambda, + Lt, + LtE, + Name, + NodeTransformer, + NodeVisitor, + NotEq, + UnaryOp, + USub, + cmpop, + copy_location, + expr, + parse, +) from collections.abc import AsyncIterable, Callable, Mapping, Sequence from copy import deepcopy from dataclasses import dataclass @@ -742,6 +767,59 @@ def visit_UnaryOp(self, node: UnaryOp) -> AST: return node +_OP_FLIP: dict[type[cmpop], type[cmpop]] = { + Eq: Eq, + NotEq: NotEq, + Lt: Gt, + LtE: GtE, + Gt: Lt, + GtE: LtE, + Is: Is, + IsNot: IsNot, +} + + +class _NormalizeComparisonFilters(NodeTransformer): + """Normalize comparison filter expressions before connector parsers run. + + Vector-store connector ``_lambda_parser`` implementations expect the field reference + (e.g., ``x.num``) on the left-hand side and the constant/literal on the right-hand side. + When a filter is written with the constant on the left (e.g., ``2 < x.num`` or ``3 == x.num``), + or uses Python's chained comparison syntax (e.g., ``1 <= x.num < 4``), connectors either + fail with validation errors or misinterpret the field name (#14779). + + This transformer: + 1. Splits chained comparisons (``len(ops) > 1``) into an explicit ``BoolOp(And, ...)``. + 2. Rewrites ``Constant op Field`` into ``Field op' Constant`` (e.g., ``<`` to ``>``). + """ + + def visit_Compare(self, node: Compare) -> AST: + """Normalize chained comparisons and comparisons with constant on the left.""" + self.generic_visit(node) + if len(node.ops) > 1: + pairs: list[expr] = [] + for i in range(len(node.ops)): + left = node.left if i == 0 else node.comparators[i - 1] + right = node.comparators[i] + op = node.ops[i] + sub_compare = copy_location(Compare(left=left, ops=[op], comparators=[right]), node) + transformed = self.visit_Compare(sub_compare) + pairs.append(transformed if isinstance(transformed, expr) else sub_compare) + return copy_location(BoolOp(op=And(), values=pairs), node) + + op_type = type(node.ops[0]) + if op_type in _OP_FLIP: + left = node.left + right = node.comparators[0] + if not isinstance(left, (Attribute, Name)) and isinstance(right, (Attribute, Name)): + flipped_op = _OP_FLIP[op_type]() + return copy_location( + Compare(left=right, ops=[flipped_op], comparators=[left]), + node, + ) + return node + + class LambdaVisitor(NodeVisitor, Generic[TFilters]): """Visitor class to visit the AST nodes.""" @@ -2016,11 +2094,14 @@ def _build_filter(self, search_filter: OptionalOneOrMany[Callable | str] | None) visitor = LambdaVisitor(self._lambda_parser) folder = _FoldUnaryNumericConstants() + normalizer = _NormalizeComparisonFilters() for filter_ in filters: # parse lambda expression with AST tree = parse(filter_ if isinstance(filter_, str) else getsource(filter_).strip()) # Fold negative literals so connector parsers see a numeric Constant (#14571). tree = folder.visit(tree) + # Normalize chained comparisons and constant-on-left comparisons (#14779). + tree = normalizer.visit(tree) visitor.visit(tree) created_filters = visitor.output_filters if len(created_filters) == 0: diff --git a/python/tests/unit/data/test_vector_filter_comparison_normalization.py b/python/tests/unit/data/test_vector_filter_comparison_normalization.py new file mode 100644 index 000000000000..5d69faf614b9 --- /dev/null +++ b/python/tests/unit/data/test_vector_filter_comparison_normalization.py @@ -0,0 +1,143 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Regression: Vector store filters fail when constant is on the left or chained (#14779).""" + +import ast +from unittest.mock import MagicMock + +import pytest +from chromadb.api import ClientAPI +from qdrant_client.async_qdrant_client import AsyncQdrantClient +from qdrant_client.models import FieldCondition, Filter, MatchValue, Range + +from semantic_kernel.connectors.chroma import ChromaCollection +from semantic_kernel.connectors.in_memory import InMemoryCollection +from semantic_kernel.connectors.qdrant import QdrantCollection +from semantic_kernel.data.vector import ( + VectorStoreCollectionDefinition, + VectorStoreField, + _FoldUnaryNumericConstants, + _NormalizeComparisonFilters, +) + + +@pytest.fixture +def numeric_definition() -> VectorStoreCollectionDefinition: + return VectorStoreCollectionDefinition( + fields=[ + VectorStoreField("key", name="id", type="str"), + VectorStoreField("data", name="num", type="int"), + VectorStoreField("data", name="status", type="str"), + VectorStoreField("vector", name="vector", dimensions=2, type="float"), + ] + ) + + +@pytest.mark.parametrize( + "expression,expected_unparsed", + [ + # Constant on left flipped to field on left + ("lambda x: 2 < x.num", "lambda x: x.num > 2"), + ("lambda x: 2 <= x.num", "lambda x: x.num >= 2"), + ("lambda x: 2 > x.num", "lambda x: x.num < 2"), + ("lambda x: 2 >= x.num", "lambda x: x.num <= 2"), + ("lambda x: 3 == x.num", "lambda x: x.num == 3"), + ("lambda x: 3 != x.num", "lambda x: x.num != 3"), + ("lambda x: 'active' == x.status", "lambda x: x.status == 'active'"), + ("lambda x: True == x.status", "lambda x: x.status == True"), + ("lambda x: None == x.status", "lambda x: x.status == None"), + ("lambda num: 2 < num", "lambda num: num > 2"), + ("lambda num: 3 == num", "lambda num: num == 3"), + # Field on left preserved as-is + ("lambda x: x.num > 2", "lambda x: x.num > 2"), + ("lambda x: x.num == 3", "lambda x: x.num == 3"), + ("lambda x: x.status == 'active'", "lambda x: x.status == 'active'"), + # Chained comparisons expanded and flipped + ("lambda x: 1 <= x.num < 4", "lambda x: x.num >= 1 and x.num < 4"), + ("lambda x: 1 < x.num < 10", "lambda x: x.num > 1 and x.num < 10"), + ("lambda x: 4 > x.num >= 1", "lambda x: x.num < 4 and x.num >= 1"), + ("lambda x: 10 >= x.num > 2", "lambda x: x.num <= 10 and x.num > 2"), + # In / NotIn preserved + ("lambda x: 2 in x.num", "lambda x: 2 in x.num"), + ("lambda x: 'tag' not in x.status", "lambda x: 'tag' not in x.status"), + # Two fields or two constants preserved + ("lambda x: x.num < x.status", "lambda x: x.num < x.status"), + ("lambda x: 1 < 2", "lambda x: 1 < 2"), + ], +) +def test_normalize_comparison_filters_ast(expression, expected_unparsed): + tree = ast.parse(expression) + normalized = _NormalizeComparisonFilters().visit(tree) + assert ast.unparse(normalized) == expected_unparsed + + +@pytest.mark.parametrize( + "expression,expected_unparsed", + [ + ("lambda x: -5 < x.num", "lambda x: x.num > -5"), + ("lambda x: -5 <= x.num < 10", "lambda x: x.num >= -5 and x.num < 10"), + ("lambda x: 10 >= x.num > -5", "lambda x: x.num <= 10 and x.num > -5"), + ], +) +def test_normalize_comparison_filters_with_negative_literals(expression, expected_unparsed): + tree = ast.parse(expression) + folded = _FoldUnaryNumericConstants().visit(tree) + normalized = _NormalizeComparisonFilters().visit(folded) + assert ast.unparse(normalized) == expected_unparsed + + +def test_chroma_build_filter_constant_on_left(numeric_definition): + collection = ChromaCollection( + collection_name="test", + record_type=dict, + definition=numeric_definition, + client=MagicMock(spec=ClientAPI), + ) + assert collection._build_filter("lambda x: 2 < x.num") == {"num": {"$gt": 2}} + assert collection._build_filter("lambda x: 3 == x.num") == {"num": 3} + assert collection._build_filter("lambda x: 1 <= x.num < 4") == {"$and": [{"num": {"$gte": 1}}, {"num": {"$lt": 4}}]} + assert collection._build_filter("lambda x: -5 < x.num") == {"num": {"$gt": -5}} + + +def test_qdrant_build_filter_constant_on_left(numeric_definition): + collection = QdrantCollection( + collection_name="test", + record_type=dict, + definition=numeric_definition, + client=MagicMock(spec=AsyncQdrantClient), + ) + assert collection._build_filter("lambda x: 2 < x.num") == FieldCondition(key="num", range=Range(gt=2.0)) + assert collection._build_filter("lambda x: 3 == x.num") == FieldCondition(key="num", match=MatchValue(value=3)) + assert collection._build_filter("lambda x: 1 <= x.num < 4") == Filter( + must=[ + FieldCondition(key="num", range=Range(gte=1.0)), + FieldCondition(key="num", range=Range(lt=4.0)), + ] + ) + assert collection._build_filter("lambda x: -5 < x.num") == FieldCondition(key="num", range=Range(gt=-5.0)) + + +@pytest.mark.asyncio +async def test_in_memory_collection_searches_with_normalized_filters(numeric_definition): + collection = InMemoryCollection( + collection_name="test", + record_type=dict, + definition=numeric_definition, + ) + records = [{"id": f"rec_{i}", "num": i, "status": "active", "vector": [1.0, 0.0]} for i in range(6)] + await collection.upsert(records) + + # lambda x: 2 < x.num -> expected records 3, 4, 5 + res = await collection.search(vector=[1.0, 0.0], filter="lambda x: 2 < x.num", top=10) + hits = [hit.record["num"] async for hit in res.results] + assert sorted(hits) == [3, 4, 5] + + # lambda x: 3 == x.num -> expected record 3 + res = await collection.search(vector=[1.0, 0.0], filter="lambda x: 3 == x.num", top=10) + hits = [hit.record["num"] async for hit in res.results] + assert hits == [3] + + # lambda x: 1 <= x.num < 4 -> expected records 1, 2, 3 + res = await collection.search(vector=[1.0, 0.0], filter="lambda x: 1 <= x.num < 4", top=10) + hits = [hit.record["num"] async for hit in res.results] + assert sorted(hits) == [1, 2, 3]