From abdbad645d6dc4b4c6d1400a540e2f9426174d75 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Fri, 6 Mar 2026 11:04:47 +0200 Subject: [PATCH] Remove legacy filtering from MEVD providers Closes #10456 --- .../AzureAISearch/AzureAISearchCollection.cs | 15 +- .../AzureAISearchCollectionSearchMapping.cs | 78 -------- .../CosmosMongoDB/CosmosMongoCollection.cs | 12 +- .../CosmosMongoCollectionSearchMapping.cs | 72 ------- .../CosmosNoSql/CosmosNoSqlCollection.cs | 6 - .../CosmosNoSqlCollectionQueryBuilder.cs | 77 +------- .../VectorData/InMemory/InMemoryCollection.cs | 12 +- .../InMemoryCollectionSearchMapping.cs | 134 ------------- .../src/VectorData/MongoDB/MongoCollection.cs | 24 +-- .../MongoDB/MongoCollectionSearchMapping.cs | 73 -------- .../VectorData/PgVector/PostgresCollection.cs | 6 - .../VectorData/PgVector/PostgresSqlBuilder.cs | 108 ++--------- .../VectorData/Pinecone/PineconeCollection.cs | 12 +- .../PineconeCollectionSearchMapping.cs | 63 ------- .../src/VectorData/Qdrant/QdrantCollection.cs | 25 +-- .../Qdrant/QdrantCollectionSearchMapping.cs | 72 ------- .../Redis/RedisCollectionSearchMapping.cs | 71 +------ .../SqlServer/SqlServerCollection.cs | 12 -- .../VectorData/SqliteVec/SqliteCollection.cs | 58 +----- .../VectorSearch/HybridSearchOptions.cs | 8 - .../VectorSearch/RecordSearchOptions.cs | 11 +- .../VectorSearch/VectorSearchFilter.cs | 81 -------- .../Weaviate/WeaviateQueryBuilder.cs | 112 +---------- .../AzureAISearchCollectionTests.cs | 12 +- .../CosmosMongoFilterTests.cs | 9 - ...CosmosMongoCollectionSearchMappingTests.cs | 109 ----------- .../CosmosNoSqlCollectionQueryBuilderTests.cs | 176 ------------------ .../MongoFilterTests.cs | 9 - .../MongoCollectionSearchMappingTests.cs | 75 -------- .../PostgresFilterTests.cs | 4 - .../PineconeFilterTests.cs | 9 - .../QdrantCollectionSearchMappingTests.cs | 90 --------- .../Qdrant.UnitTests/QdrantCollectionTests.cs | 5 +- .../RedisFilterTests.cs | 8 - .../RedisCollectionSearchMappingTests.cs | 91 --------- .../RedisHashSetCollectionTests.cs | 8 +- .../RedisJsonCollectionTests.cs | 8 +- .../SqlServerFilterTests.cs | 16 -- .../SqliteFilterTests.cs | 9 - .../FilterTests.cs | 66 ------- .../WeaviateQueryBuilderTests.cs | 78 -------- 41 files changed, 66 insertions(+), 1858 deletions(-) delete mode 100644 dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionSearchMapping.cs delete mode 100644 dotnet/src/VectorData/Pinecone/PineconeCollectionSearchMapping.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchFilter.cs delete mode 100644 dotnet/test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoCollectionSearchMappingTests.cs delete mode 100644 dotnet/test/VectorData/MongoDB.UnitTests/MongoCollectionSearchMappingTests.cs diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollection.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollection.cs index 1eb213862345..16ed75d6c6ee 100644 --- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollection.cs +++ b/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollection.cs @@ -442,9 +442,6 @@ public async IAsyncEnumerable> HybridSearchAsync throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"), - { OldFilter: VectorSearchFilter legacyFilter } => AzureAISearchCollectionSearchMapping.BuildLegacyFilterString(legacyFilter, model), - { Filter: Expression> newFilter } => new AzureAISearchFilterTranslator().Translate(newFilter, model), - _ => null - }; -#pragma warning restore CS0618 + var filter = options.Filter is not null + ? new AzureAISearchFilterTranslator().Translate(options.Filter, model) + : null; // Build search options. var searchOptions = new SearchOptions diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionSearchMapping.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionSearchMapping.cs deleted file mode 100644 index 4a049796c552..000000000000 --- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionSearchMapping.cs +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Linq; -using Microsoft.Extensions.VectorData; -using Microsoft.Extensions.VectorData.ProviderServices; - -namespace Microsoft.SemanticKernel.Connectors.AzureAISearch; - -/// -/// Contains mapping helpers to use when searching for documents using Azure AI Search. -/// -internal static class AzureAISearchCollectionSearchMapping -{ -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - /// - /// Build an OData filter string from the provided . - /// - /// The to build an OData filter string from. - /// The model. - /// The OData filter string. - /// Thrown when a provided filter value is not supported. - public static string BuildLegacyFilterString(VectorSearchFilter basicVectorSearchFilter, CollectionModel model) - { - var filterString = string.Empty; - if (basicVectorSearchFilter.FilterClauses is not null) - { - // Map Equality clauses. - var filterStrings = basicVectorSearchFilter?.FilterClauses.OfType().Select(x => - { - string storageFieldName = GetStoragePropertyName(model, x.FieldName); - - return x.Value switch - { - string stringValue => $"{storageFieldName} eq '{stringValue}'", -#pragma warning disable CA1308 // Normalize strings to uppercase - OData filter strings use lowercase boolean literals. See https://docs.oasis-open.org/odata/odata/v4.01/cs01/part2-url-conventions/odata-v4.01-cs01-part2-url-conventions.html#sec_PrimitiveLiterals - bool boolValue => $"{storageFieldName} eq {boolValue.ToString().ToLowerInvariant()}", -#pragma warning restore CA1308 // Normalize strings to uppercase - int intValue => $"{storageFieldName} eq {intValue}", - long longValue => $"{storageFieldName} eq {longValue}", - float floatValue => $"{storageFieldName} eq {floatValue}", - double doubleValue => $"{storageFieldName} eq {doubleValue}", - DateTimeOffset dateTimeOffsetValue => $"{storageFieldName} eq {dateTimeOffsetValue.UtcDateTime:O}", - null => $"{storageFieldName} eq null", - _ => throw new InvalidOperationException($"Unsupported filter value type '{x.Value.GetType().Name}'.") - }; - }); - - // Map tag contains clauses. - var tagListContainsStrings = basicVectorSearchFilter?.FilterClauses - .OfType() - .Select(x => $"{GetStoragePropertyName(model, x.FieldName)}/any(t: t eq '{x.Value}')"); - - // Combine clauses. - filterString = string.Join(" and ", filterStrings!.Concat(tagListContainsStrings!)); - } - - return filterString; - } -#pragma warning restore CS0618 // VectorSearchFilter is obsolete - - /// - /// Gets the name of the name under which the property with the given name is stored. - /// - /// The model. - /// The name of the property in the data model. - /// The name that the property os stored under. - /// Thrown when the property name is not found. - private static string GetStoragePropertyName(CollectionModel model, string fieldName) - { - if (!model.PropertyMap.TryGetValue(fieldName, out var property)) - { - throw new InvalidOperationException($"Property name '{fieldName}' provided as part of the filter clause is not a valid property name."); - } - - return property.StorageName; - } -} diff --git a/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollection.cs b/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollection.cs index 04dee834cd05..52cc3ded9a2d 100644 --- a/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollection.cs +++ b/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollection.cs @@ -373,15 +373,9 @@ _ when vectorProperty.EmbeddingGenerationDispatcher is not null : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType())) }; -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - var filter = options switch - { - { OldFilter: not null, Filter: not null } => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"), - { OldFilter: VectorSearchFilter legacyFilter } => CosmosMongoCollectionSearchMapping.BuildFilter(legacyFilter, this._model), - { Filter: Expression> newFilter } => new CosmosMongoFilterTranslator().Translate(newFilter, this._model), - _ => null - }; -#pragma warning restore CS0618 + var filter = options.Filter is not null + ? new CosmosMongoFilterTranslator().Translate(options.Filter, this._model) + : null; // Constructing a query to fetch "skip + top" total items // to perform skip logic locally, since skip option is not part of API. diff --git a/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollectionSearchMapping.cs b/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollectionSearchMapping.cs index 53ef2fe29322..91aae21f8bb8 100644 --- a/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollectionSearchMapping.cs +++ b/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollectionSearchMapping.cs @@ -1,9 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Linq; using Microsoft.Extensions.VectorData; -using Microsoft.Extensions.VectorData.ProviderServices; using Microsoft.SemanticKernel.Connectors.MongoDB; using MongoDB.Bson; @@ -20,75 +17,6 @@ internal static class CosmosMongoCollectionSearchMapping /// Returns distance function specified on vector property or default . public static string GetVectorPropertyDistanceFunction(string? distanceFunction) => !string.IsNullOrWhiteSpace(distanceFunction) ? distanceFunction! : MongoConstants.DefaultDistanceFunction; -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - /// - /// Build Azure CosmosDB MongoDB filter from the provided . - /// - /// The to build Azure CosmosDB MongoDB filter from. - /// The model. - /// Thrown when the provided filter type is unsupported. - /// Thrown when property name specified in filter doesn't exist. - public static BsonDocument? BuildFilter(VectorSearchFilter? vectorSearchFilter, CollectionModel model) - { - const string EqualOperator = "$eq"; - - var filterClauses = vectorSearchFilter?.FilterClauses.ToList(); - - if (filterClauses is not { Count: > 0 }) - { - return null; - } - - var filter = new BsonDocument(); - - foreach (var filterClause in filterClauses) - { - string propertyName; - BsonValue propertyValue; - string filterOperator; - - if (filterClause is EqualToFilterClause equalToFilterClause) - { - propertyName = equalToFilterClause.FieldName; - propertyValue = BsonValueFactory.Create(equalToFilterClause.Value); - filterOperator = EqualOperator; - } - else - { - throw new NotSupportedException( - $"Unsupported filter clause type '{filterClause.GetType().Name}'. " + - $"Supported filter clause types are: {string.Join(", ", [ - nameof(EqualToFilterClause)])}"); - } - - if (!model.PropertyMap.TryGetValue(propertyName, out var property)) - { - throw new InvalidOperationException($"Property name '{propertyName}' provided as part of the filter clause is not a valid property name."); - } - - var storageName = property.StorageName; - - if (filter.Contains(storageName)) - { - if (filter[storageName] is BsonDocument document && document.Contains(filterOperator)) - { - throw new NotSupportedException( - $"Filter with operator '{filterOperator}' is already added to '{propertyName}' property. " + - "Multiple filters of the same type in the same property are not supported."); - } - - filter[storageName][filterOperator] = propertyValue; - } - else - { - filter[storageName] = new BsonDocument() { [filterOperator] = propertyValue }; - } - } - - return filter; - } -#pragma warning restore CS0618 // VectorSearchFilter is obsolete - /// Returns search part of the search query for index kind. public static BsonDocument GetSearchQueryForHnswIndex( TVector vector, diff --git a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollection.cs b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollection.cs index 484f55a2d61b..d535a16e6d31 100644 --- a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollection.cs +++ b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollection.cs @@ -541,7 +541,6 @@ public override async IAsyncEnumerable> SearchAsync< var vectorProperty = this._model.GetVectorPropertyOrSingle(options); object vector = await GetSearchVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false); -#pragma warning disable CS0618 // Type or member is obsolete var queryDefinition = CosmosNoSqlCollectionQueryBuilder.BuildSearchQuery( vector, null, @@ -550,13 +549,11 @@ public override async IAsyncEnumerable> SearchAsync< vectorProperty.DistanceFunction, textPropertyName: null, ScorePropertyName, - options.OldFilter, options.Filter, options.ScoreThreshold, top, options.Skip, options.IncludeVectors); -#pragma warning restore CS0618 // Type or member is obsolete var searchResults = this.GetItemsAsync(queryDefinition, OperationName, cancellationToken); @@ -645,7 +642,6 @@ public async IAsyncEnumerable> HybridSearchAsync( vector, keywords, @@ -654,13 +650,11 @@ public async IAsyncEnumerable> HybridSearchAsync(queryDefinition, OperationName, cancellationToken); diff --git a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollectionQueryBuilder.cs b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollectionQueryBuilder.cs index 5a80e0ce8d91..0a07992b582b 100644 --- a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollectionQueryBuilder.cs +++ b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollectionQueryBuilder.cs @@ -27,9 +27,6 @@ public static QueryDefinition BuildSearchQuery( string? distanceFunction, string? textPropertyName, string scorePropertyName, -#pragma warning disable CS0618 // Type or member is obsolete - VectorSearchFilter? oldFilter, -#pragma warning restore CS0618 // Type or member is obsolete Expression>? filter, double? scoreThreshold, int top, @@ -54,16 +51,10 @@ public static QueryDefinition BuildSearchQuery( var selectClauseArguments = string.Join(",", [.. fieldsArgument, vectorDistanceArgumentWithAlias]); -#pragma warning disable CS0618 // VectorSearchFilter is obsolete // Build filter object. - var (filterClause, filterParameters) = (OldFilter: oldFilter, Filter: filter) switch - { - { OldFilter: not null, Filter: not null } => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"), - { OldFilter: VectorSearchFilter legacyFilter } => BuildSearchFilter(legacyFilter, model), - { Filter: Expression> newFilter } => new CosmosNoSqlFilterTranslator().Translate(newFilter, model), - _ => (null, []) - }; -#pragma warning restore CS0618 // VectorSearchFilter is obsolete + var (filterClause, filterParameters) = filter is not null + ? new CosmosNoSqlFilterTranslator().Translate(filter, model) + : ((string?)null, new Dictionary()); var queryParameters = new Dictionary { @@ -229,68 +220,6 @@ internal static QueryDefinition BuildSearchQuery( #region private -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - private static (string WhereClause, Dictionary Parameters) BuildSearchFilter( - VectorSearchFilter filter, - CollectionModel model) - { - const string ArrayContainsOperator = "ARRAY_CONTAINS"; - const string ConditionValueVariableName = "@cv"; - - var tableVariableName = CosmosNoSqlConstants.ContainerAlias; - - var filterClauses = filter.FilterClauses.ToList(); - - var whereClauseBuilder = new StringBuilder(); - var queryParameters = new Dictionary(); - - for (var i = 0; i < filterClauses.Count; i++) - { - if (i > 0) - { - whereClauseBuilder.Append(" AND "); - } - var filterClause = filterClauses[i]; - - string queryParameterName = $"{ConditionValueVariableName}{i}"; - object queryParameterValue; - - if (filterClause is EqualToFilterClause equalToFilterClause) - { - var propertyName = GetStoragePropertyName(equalToFilterClause.FieldName, model); - whereClauseBuilder - .Append(GeneratePropertyAccess(tableVariableName, propertyName)) - .Append(" = ") - .Append(queryParameterName); - queryParameterValue = equalToFilterClause.Value; - } - else if (filterClause is AnyTagEqualToFilterClause anyTagEqualToFilterClause) - { - var propertyName = GetStoragePropertyName(anyTagEqualToFilterClause.FieldName, model); - whereClauseBuilder.Append(ArrayContainsOperator) - .Append('(') - .Append(GeneratePropertyAccess(tableVariableName, propertyName)) - .Append(", ") - .Append(queryParameterName) - .Append(')'); - queryParameterValue = anyTagEqualToFilterClause.Value; - } - else - { - throw new NotSupportedException( - $"Unsupported filter clause type '{filterClause.GetType().Name}'. " + - $"Supported filter clause types are: {string.Join(", ", [ - nameof(EqualToFilterClause), - nameof(AnyTagEqualToFilterClause)])}"); - } - - queryParameters.Add(queryParameterName, queryParameterValue); - } - - return (whereClauseBuilder.ToString(), queryParameters); - } -#pragma warning restore CS0618 // VectorSearchFilter is obsolete - private static string GetStoragePropertyName(string propertyName, CollectionModel model) { if (!model.PropertyMap.TryGetValue(propertyName, out var property)) diff --git a/dotnet/src/VectorData/InMemory/InMemoryCollection.cs b/dotnet/src/VectorData/InMemory/InMemoryCollection.cs index b0932c621af4..aa2c53d9ad2e 100644 --- a/dotnet/src/VectorData/InMemory/InMemoryCollection.cs +++ b/dotnet/src/VectorData/InMemory/InMemoryCollection.cs @@ -282,17 +282,11 @@ _ when vectorProperty.EmbeddingGenerationDispatcher is not null : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType())) }; -#pragma warning disable CS0618 // VectorSearchFilter is obsolete // Filter records using the provided filter before doing the vector comparison. var allValues = this.GetCollectionDictionary().Values.Cast>(); - var filteredRecords = options switch - { - { OldFilter: not null, Filter: not null } => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"), - { OldFilter: VectorSearchFilter legacyFilter } => InMemoryCollectionSearchMapping.FilterRecords(legacyFilter, allValues), - { Filter: Expression> newFilter } => allValues.AsQueryable().Where(this.ConvertFilter(newFilter)), - _ => allValues - }; -#pragma warning restore CS0618 // VectorSearchFilter is obsolete + var filteredRecords = options.Filter is not null + ? allValues.AsQueryable().Where(this.ConvertFilter(options.Filter)) + : allValues; // Compare each vector in the filtered results with the provided vector. var results = filteredRecords.Select, (TRecord record, float score)?>(wrapper => diff --git a/dotnet/src/VectorData/InMemory/InMemoryCollectionSearchMapping.cs b/dotnet/src/VectorData/InMemory/InMemoryCollectionSearchMapping.cs index 371060cab08a..e763b934cfb3 100644 --- a/dotnet/src/VectorData/InMemory/InMemoryCollectionSearchMapping.cs +++ b/dotnet/src/VectorData/InMemory/InMemoryCollectionSearchMapping.cs @@ -1,11 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; using System.Numerics.Tensors; -using System.Reflection; using Microsoft.Extensions.VectorData; namespace Microsoft.SemanticKernel.Connectors.InMemory; @@ -87,134 +83,4 @@ public static float ConvertScore(float score, string? distanceFunction) throw new NotSupportedException($"The distance function '{distanceFunction}' is not supported by the InMemory connector."); } } - -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - /// - /// Filter the provided records using the provided filter definition. - /// - /// The filter definition to filter the with. - /// The records to filter. - /// The filtered records. - /// Thrown when an unsupported filter clause is encountered. - public static IEnumerable> FilterRecords(VectorSearchFilter filter, IEnumerable> recordWrappers) - { - return recordWrappers.Where(wrapper => - { - if (wrapper.Record is null) - { - return false; - } - - var result = true; - - // Run each filter clause against the record, and AND the results together. - // Break if any clause returns false, since we are doing an AND and no need - // to check any further clauses. - foreach (var clause in filter.FilterClauses) - { - if (clause is EqualToFilterClause equalToFilter) - { - result = result && CheckEqualTo(wrapper.Record, equalToFilter); - - if (result == false) - { - break; - } - } - else if (clause is AnyTagEqualToFilterClause anyTagEqualToFilter) - { - result = result && CheckAnyTagEqualTo(wrapper.Record, anyTagEqualToFilter); - - if (result == false) - { - break; - } - } - else - { - throw new InvalidOperationException($"Unsupported filter clause type {clause.GetType().Name}"); - } - } - - return result; - }); - } - - /// - /// Check if the required property on the record is equal to the required value form the filter. - /// - /// The record to check against the filter. - /// The filter containing the property and value to check. - /// if the property equals the required value, otherwise. - private static bool CheckEqualTo(object record, EqualToFilterClause equalToFilter) - { - var propertyInfo = GetPropertyInfo(record, equalToFilter.FieldName); - var propertyValue = propertyInfo.GetValue(record); - if (propertyValue == null) - { - return propertyValue == equalToFilter.Value; - } - - return propertyValue.Equals(equalToFilter.Value); - } - - /// - /// Check if the required tag list on the record is equal to the required value form the filter. - /// - /// The record to check against the filter. - /// The filter containing the property and value to check. - /// if the tag list contains the required value, otherwise. - /// - private static bool CheckAnyTagEqualTo(object record, AnyTagEqualToFilterClause anyTagEqualToFilter) - { - var propertyInfo = GetPropertyInfo(record, anyTagEqualToFilter.FieldName); - - // Check that the property is actually a list of values. - if (!typeof(IEnumerable).IsAssignableFrom(propertyInfo.PropertyType)) - { - throw new InvalidOperationException($"Property {anyTagEqualToFilter.FieldName} is not a list property on record type {record.GetType().Name}"); - } - - // Check that the tag list contains any values. If not, return false, since the required value cannot be in an empty list. - var propertyValue = propertyInfo.GetValue(record) as IEnumerable; - if (propertyValue == null) - { - return false; - } - - // Check each value in the tag list against the required value. - foreach (var value in propertyValue) - { - if (value == null && anyTagEqualToFilter.Value == null) - { - return true; - } - - if (value != null && value.Equals(anyTagEqualToFilter.Value)) - { - return true; - } - } - - return false; - } -#pragma warning restore CS0618 // VectorSearchFilter is obsolete - - /// - /// Get the property info for the provided property name on the record. - /// - /// The record to find the property on. - /// The name of the property to find. - /// The property info for the required property. - /// Thrown if the required property does not exist on the record. - private static PropertyInfo GetPropertyInfo(object record, string propertyName) - { - var propertyInfo = record.GetType().GetProperty(propertyName); - if (propertyInfo == null) - { - throw new InvalidOperationException($"Property {propertyName} not found on record type {record.GetType().Name}"); - } - - return propertyInfo; - } } diff --git a/dotnet/src/VectorData/MongoDB/MongoCollection.cs b/dotnet/src/VectorData/MongoDB/MongoCollection.cs index d024e2974b08..7cb8f234ec16 100644 --- a/dotnet/src/VectorData/MongoDB/MongoCollection.cs +++ b/dotnet/src/VectorData/MongoDB/MongoCollection.cs @@ -379,15 +379,9 @@ public override async IAsyncEnumerable> SearchAsync< var vectorProperty = this._model.GetVectorPropertyOrSingle(options); var vectorArray = await GetSearchVectorArrayAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false); -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - var filter = options switch - { - { OldFilter: not null, Filter: not null } => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"), - { OldFilter: VectorSearchFilter legacyFilter } => MongoCollectionSearchMapping.BuildLegacyFilter(legacyFilter, this._model), - { Filter: Expression> newFilter } => new MongoFilterTranslator().Translate(newFilter, this._model), - _ => null - }; -#pragma warning restore CS0618 + var filter = options.Filter is not null + ? new MongoFilterTranslator().Translate(options.Filter, this._model) + : null; // Constructing a query to fetch "skip + top" total items // to perform skip logic locally, since skip option is not part of API. @@ -518,15 +512,9 @@ public async IAsyncEnumerable> HybridSearchAsync throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"), - { OldFilter: VectorSearchFilter legacyFilter } => MongoCollectionSearchMapping.BuildLegacyFilter(legacyFilter, this._model), - { Filter: Expression> newFilter } => new MongoFilterTranslator().Translate(newFilter, this._model), - _ => null - }; -#pragma warning restore CS0618 + var filter = options.Filter is not null + ? new MongoFilterTranslator().Translate(options.Filter, this._model) + : null; // Constructing a query to fetch "skip + top" total items // to perform skip logic locally, since skip option is not part of API. diff --git a/dotnet/src/VectorData/MongoDB/MongoCollectionSearchMapping.cs b/dotnet/src/VectorData/MongoDB/MongoCollectionSearchMapping.cs index 515adcf4bccb..90b9ca56e4d1 100644 --- a/dotnet/src/VectorData/MongoDB/MongoCollectionSearchMapping.cs +++ b/dotnet/src/VectorData/MongoDB/MongoCollectionSearchMapping.cs @@ -1,10 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -using System; using System.Collections.Generic; -using System.Linq; -using Microsoft.Extensions.VectorData; -using Microsoft.Extensions.VectorData.ProviderServices; using MongoDB.Bson; namespace Microsoft.SemanticKernel.Connectors.MongoDB; @@ -14,75 +10,6 @@ namespace Microsoft.SemanticKernel.Connectors.MongoDB; /// internal static class MongoCollectionSearchMapping { -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - /// - /// Build MongoDB filter from the provided . - /// - /// The to build MongoDB filter from. - /// The model. - /// Thrown when the provided filter type is unsupported. - /// Thrown when property name specified in filter doesn't exist. - public static BsonDocument? BuildLegacyFilter( - VectorSearchFilter vectorSearchFilter, - CollectionModel model) - { - const string EqualOperator = "$eq"; - - var filterClauses = vectorSearchFilter.FilterClauses.ToList(); - - if (filterClauses is not { Count: > 0 }) - { - return null; - } - - var filter = new BsonDocument(); - - foreach (var filterClause in filterClauses) - { - string propertyName; - BsonValue propertyValue; - string filterOperator; - - if (filterClause is EqualToFilterClause equalToFilterClause) - { - propertyName = equalToFilterClause.FieldName; - propertyValue = BsonValueFactory.Create(equalToFilterClause.Value); - filterOperator = EqualOperator; - } - else - { - throw new NotSupportedException( - $"Unsupported filter clause type '{filterClause.GetType().Name}'. " + - $"Supported filter clause types are: {string.Join(", ", [ - nameof(EqualToFilterClause)])}"); - } - - if (!model.PropertyMap.TryGetValue(propertyName, out var property)) - { - throw new InvalidOperationException($"Property name '{propertyName}' provided as part of the filter clause is not a valid property name."); - } - - if (filter.Contains(property.StorageName)) - { - if (filter[property.StorageName] is BsonDocument document && document.Contains(filterOperator)) - { - throw new NotSupportedException( - $"Filter with operator '{filterOperator}' is already added to '{propertyName}' property. " + - "Multiple filters of the same type in the same property are not supported."); - } - - filter[property.StorageName][filterOperator] = propertyValue; - } - else - { - filter[property.StorageName] = new BsonDocument() { [filterOperator] = propertyValue }; - } - } - - return filter; - } -#pragma warning restore CS0618 - /// Returns search part of the search query. public static BsonDocument GetSearchQuery( TVector vector, diff --git a/dotnet/src/VectorData/PgVector/PostgresCollection.cs b/dotnet/src/VectorData/PgVector/PostgresCollection.cs index 89b61c7a7c63..79d47b7f794c 100644 --- a/dotnet/src/VectorData/PgVector/PostgresCollection.cs +++ b/dotnet/src/VectorData/PgVector/PostgresCollection.cs @@ -388,9 +388,6 @@ public override async IAsyncEnumerable> SearchAsync< using var connection = await this._dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); using var command = connection.CreateCommand(); PostgresSqlBuilder.BuildGetNearestMatchCommand(command, this._schema, this.Name, this._model, vectorProperty, pgVector, -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - options.OldFilter, -#pragma warning restore CS0618 // VectorSearchFilter is obsolete options.Filter, options.Skip, options.IncludeVectors, top, options.ScoreThreshold); using var reader = await connection.ExecuteWithErrorHandlingAsync( @@ -433,9 +430,6 @@ public async IAsyncEnumerable> HybridSearchAsync - /// Generates filter clause from either legacy or new filter, returning condition and parameters. + /// Generates filter clause from the provided filter, returning condition and parameters. /// -#pragma warning disable CS0618 // VectorSearchFilter is obsolete private static (string Clause, List Parameters) GenerateFilterClause( CollectionModel model, - VectorSearchFilter? legacyFilter, - Expression>? newFilter, + Expression>? filter, int startParamIndex) - => (oldFilter: legacyFilter, newFilter) switch - { - (not null, not null) => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"), - (not null, null) => GenerateLegacyFilterWhereClause(model, legacyFilter, startParamIndex), - (null, not null) => GenerateNewFilterWhereClause(model, newFilter, startParamIndex), - _ => (Clause: string.Empty, Parameters: new List()) - }; + => filter is not null + ? TranslateFilterWhereClause(model, filter, startParamIndex) + : (Clause: string.Empty, Parameters: new List()); /// - /// Generates filter condition (without WHERE keyword) from either legacy or new filter. + /// Generates filter condition (without WHERE keyword) from the provided filter. /// private static (string Condition, List Parameters) GenerateFilterCondition( CollectionModel model, - VectorSearchFilter? legacyFilter, - Expression>? newFilter, + Expression>? filter, int startParamIndex) - => (oldFilter: legacyFilter, newFilter) switch - { - (not null, not null) => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"), - (not null, null) => GenerateLegacyFilterCondition(model, legacyFilter, startParamIndex), - (null, not null) => GenerateNewFilterCondition(model, newFilter, startParamIndex), - _ => (Condition: string.Empty, Parameters: new List()) - }; -#pragma warning restore CS0618 // VectorSearchFilter is obsolete + => filter is not null + ? TranslateFilterCondition(model, filter, startParamIndex) + : (Condition: string.Empty, Parameters: new List()); -#pragma warning disable CS0618 // VectorSearchFilter is obsolete /// internal static void BuildGetNearestMatchCommand( NpgsqlCommand command, string? schema, string tableName, CollectionModel model, VectorPropertyModel vectorProperty, object vectorValue, - VectorSearchFilter? legacyFilter, Expression>? newFilter, int? skip, bool includeVectors, int limit, + Expression>? filter, int? skip, bool includeVectors, int limit, double? scoreThreshold = null) { // Build column list with proper escaping @@ -524,7 +510,7 @@ internal static void BuildGetNearestMatchCommand( var distanceOp = GetDistanceOperator(distanceFunction); // Start where clause params at 2, vector takes param 1. - var (where, parameters) = GenerateFilterClause(model, legacyFilter, newFilter, startParamIndex: 2); + var (where, parameters) = GenerateFilterClause(model, filter, startParamIndex: 2); StringBuilder sql = new(); sql.Append("SELECT ").Append(columns).Append(", ").AppendIdentifier(vectorProperty.StorageName) @@ -666,15 +652,15 @@ internal static void BuildSelectWhereCommand( } } - internal static (string Clause, List Parameters) GenerateNewFilterWhereClause(CollectionModel model, LambdaExpression newFilter, int startParamIndex) + private static (string Clause, List Parameters) TranslateFilterWhereClause(CollectionModel model, LambdaExpression filter, int startParamIndex) { - var (condition, parameters) = GenerateNewFilterCondition(model, newFilter, startParamIndex); + var (condition, parameters) = TranslateFilterCondition(model, filter, startParamIndex); return (string.IsNullOrEmpty(condition) ? string.Empty : $"WHERE {condition}", parameters); } - internal static (string Condition, List Parameters) GenerateNewFilterCondition(CollectionModel model, LambdaExpression newFilter, int startParamIndex) + private static (string Condition, List Parameters) TranslateFilterCondition(CollectionModel model, LambdaExpression filter, int startParamIndex) { - PostgresFilterTranslator translator = new(model, newFilter, startParamIndex); + PostgresFilterTranslator translator = new(model, filter, startParamIndex); translator.Translate(appendWhere: false); return (translator.Clause.ToString(), translator.ParameterValues); } @@ -692,63 +678,6 @@ private static StringBuilder AppendTableName(this StringBuilder sb, string? sche return sb.AppendIdentifier(tableName); } -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - internal static (string Clause, List Parameters) GenerateLegacyFilterWhereClause(CollectionModel model, VectorSearchFilter legacyFilter, int startParamIndex) - { - var (condition, parameters) = GenerateLegacyFilterCondition(model, legacyFilter, startParamIndex); - return ($"WHERE {condition}", parameters); - } - - internal static (string Condition, List Parameters) GenerateLegacyFilterCondition(CollectionModel model, VectorSearchFilter legacyFilter, int startParamIndex) - { - StringBuilder condition = new(); - var parameters = new List(); - - var paramIndex = startParamIndex; - var first = true; - - foreach (var filterClause in legacyFilter.FilterClauses) - { - if (!first) - { - condition.Append(" AND "); - } - first = false; - - if (filterClause is EqualToFilterClause equalTo) - { - var property = model.Properties.FirstOrDefault(p => p.ModelName == equalTo.FieldName); - if (property == null) { throw new ArgumentException($"Property {equalTo.FieldName} not found in record definition."); } - - condition.AppendIdentifier(property.StorageName).Append(" = $").Append(paramIndex); - parameters.Add(equalTo.Value); - paramIndex++; - } - else if (filterClause is AnyTagEqualToFilterClause anyTagEqualTo) - { - var property = model.Properties.FirstOrDefault(p => p.ModelName == anyTagEqualTo.FieldName); - if (property == null) { throw new ArgumentException($"Property {anyTagEqualTo.FieldName} not found in record definition."); } - - if (property.Type != typeof(List)) - { - throw new ArgumentException($"Property {anyTagEqualTo.FieldName} must be of type List to use AnyTagEqualTo filter."); - } - - condition.AppendIdentifier(property.StorageName).Append(" @> ARRAY[$").Append(paramIndex).Append("::TEXT]"); - parameters.Add(anyTagEqualTo.Value); - paramIndex++; - } - else - { - throw new NotSupportedException($"Filter clause type {filterClause.GetType().Name} is not supported."); - } - } - - return (condition.ToString(), parameters); - } -#pragma warning restore CS0618 // VectorSearchFilter is obsolete - -#pragma warning disable CS0618 // VectorSearchFilter is obsolete /// /// Builds a hybrid search command that combines vector similarity search with full-text keyword search using RRF (Reciprocal Rank Fusion). /// @@ -756,7 +685,7 @@ internal static void BuildHybridSearchCommand( NpgsqlCommand command, string? schema, string tableName, CollectionModel model, VectorPropertyModel vectorProperty, DataPropertyModel textProperty, object vectorValue, ICollection keywords, - VectorSearchFilter? legacyFilter, Expression>? newFilter, + Expression>? filter, int? skip, bool includeVectors, int top, double? scoreThreshold = null) { // RRF constant - higher values give more weight to lower-ranked results @@ -783,7 +712,7 @@ internal static void BuildHybridSearchCommand( // Parameters: $1 = keywords, $2 = vector, $3 = RRF constant // Additional parameters start at $4 for filters - var (filterCondition, filterParameters) = GenerateFilterCondition(model, legacyFilter, newFilter, startParamIndex: 4); + var (filterCondition, filterParameters) = GenerateFilterCondition(model, filter, startParamIndex: 4); // Build the full table name var fullTableName = new StringBuilder() @@ -883,5 +812,4 @@ internal static void BuildHybridSearchCommand( command.Parameters.Add(new NpgsqlParameter { Value = scoreThreshold.Value }); } } -#pragma warning restore CS0618 // VectorSearchFilter is obsolete } diff --git a/dotnet/src/VectorData/Pinecone/PineconeCollection.cs b/dotnet/src/VectorData/Pinecone/PineconeCollection.cs index cdf0b6efeb09..66a303936849 100644 --- a/dotnet/src/VectorData/Pinecone/PineconeCollection.cs +++ b/dotnet/src/VectorData/Pinecone/PineconeCollection.cs @@ -437,15 +437,9 @@ _ when vectorProperty.EmbeddingGenerationDispatcher is not null : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType())) }; -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - var filter = options switch - { - { OldFilter: not null, Filter: not null } => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"), - { OldFilter: VectorSearchFilter legacyFilter } => PineconeCollectionSearchMapping.BuildSearchFilter(options.OldFilter?.FilterClauses, this._model), - { Filter: Expression> newFilter } => new PineconeFilterTranslator().Translate(newFilter, this._model), - _ => null - }; -#pragma warning restore CS0618 + var filter = options.Filter is not null + ? new PineconeFilterTranslator().Translate(options.Filter, this._model) + : null; QueryRequest request = new() { diff --git a/dotnet/src/VectorData/Pinecone/PineconeCollectionSearchMapping.cs b/dotnet/src/VectorData/Pinecone/PineconeCollectionSearchMapping.cs deleted file mode 100644 index ef70cb37c9d9..000000000000 --- a/dotnet/src/VectorData/Pinecone/PineconeCollectionSearchMapping.cs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using Microsoft.Extensions.VectorData; -using Pinecone; - -namespace Microsoft.SemanticKernel.Connectors.Pinecone; - -/// -/// Contains mapping helpers to use when searching a Pinecone vector collection. -/// -internal static class PineconeCollectionSearchMapping -{ -#pragma warning disable CS0618 // FilterClause is obsolete - /// - /// Build a Pinecone from a set of filter clauses. - /// - /// The filter clauses to build the Pinecone from. - /// The model. - /// The Pinecone . - /// Thrown for invalid property names, value types or filter clause types. - public static Metadata BuildSearchFilter(IEnumerable? filterClauses, Extensions.VectorData.ProviderServices.CollectionModel model) - { - var metadataMap = new Metadata(); - - if (filterClauses is null) - { - return metadataMap; - } - - foreach (var filterClause in filterClauses) - { - if (filterClause is EqualToFilterClause equalToFilterClause) - { - if (!model.PropertyMap.TryGetValue(equalToFilterClause.FieldName, out var property)) - { - throw new InvalidOperationException($"Property '{equalToFilterClause.FieldName}' is not a valid property name."); - } - - var metadataValue = equalToFilterClause.Value switch - { - string stringValue => (MetadataValue)stringValue, - int intValue => (MetadataValue)intValue, - long longValue => (MetadataValue)longValue, - bool boolValue => (MetadataValue)boolValue, - float floatValue => (MetadataValue)floatValue, - double doubleValue => (MetadataValue)doubleValue, - _ => throw new NotSupportedException($"Unsupported filter value type '{equalToFilterClause.Value.GetType().Name}'.") - }; - - metadataMap.Add(property.StorageName, metadataValue); - } - else - { - throw new NotSupportedException($"Unsupported filter clause type '{filterClause.GetType().Name}'."); - } - } - - return metadataMap; - } -#pragma warning restore CS0618 // FilterClause is obsolete -} diff --git a/dotnet/src/VectorData/Qdrant/QdrantCollection.cs b/dotnet/src/VectorData/Qdrant/QdrantCollection.cs index 2d034cca3c30..d37f90a60d76 100644 --- a/dotnet/src/VectorData/Qdrant/QdrantCollection.cs +++ b/dotnet/src/VectorData/Qdrant/QdrantCollection.cs @@ -541,16 +541,10 @@ public override async IAsyncEnumerable> SearchAsync< var vectorProperty = this._model.GetVectorPropertyOrSingle(options); var vectorArray = await GetSearchVectorArrayAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false); -#pragma warning disable CS0618 // Type or member is obsolete // Build filter object. - var filter = options switch - { - { OldFilter: not null, Filter: not null } => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"), - { OldFilter: VectorSearchFilter legacyFilter } => QdrantCollectionSearchMapping.BuildFromLegacyFilter(legacyFilter, this._model), - { Filter: Expression> newFilter } => new QdrantFilterTranslator().Translate(newFilter, this._model), - _ => new Filter() - }; -#pragma warning restore CS0618 // Type or member is obsolete + var filter = options.Filter is not null + ? new QdrantFilterTranslator().Translate(options.Filter, this._model) + : new Filter(); // Specify whether to include vectors in the search results. var vectorsSelector = new WithVectorsSelector { Enable = options.IncludeVectors }; @@ -683,16 +677,9 @@ public async IAsyncEnumerable> HybridSearchAsync throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"), - { OldFilter: VectorSearchFilter legacyFilter } => QdrantCollectionSearchMapping.BuildFromLegacyFilter(legacyFilter, this._model), - { Filter: Expression> newFilter } => new QdrantFilterTranslator().Translate(newFilter, this._model), - _ => new Filter() - }; -#pragma warning restore CS0618 // Type or member is obsolete + var filter = options.Filter is not null + ? new QdrantFilterTranslator().Translate(options.Filter, this._model) + : new Filter(); // Specify whether to include vectors in the search results. var vectorsSelector = new WithVectorsSelector { Enable = options.IncludeVectors }; diff --git a/dotnet/src/VectorData/Qdrant/QdrantCollectionSearchMapping.cs b/dotnet/src/VectorData/Qdrant/QdrantCollectionSearchMapping.cs index f25441e8e53e..c9eb7809ed75 100644 --- a/dotnet/src/VectorData/Qdrant/QdrantCollectionSearchMapping.cs +++ b/dotnet/src/VectorData/Qdrant/QdrantCollectionSearchMapping.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -using System; using Microsoft.Extensions.VectorData; -using Microsoft.Extensions.VectorData.ProviderServices; using Qdrant.Client.Grpc; namespace Microsoft.SemanticKernel.Connectors.Qdrant; @@ -12,76 +10,6 @@ namespace Microsoft.SemanticKernel.Connectors.Qdrant; /// internal static class QdrantCollectionSearchMapping { -#pragma warning disable CS0618 // Type or member is obsolete - /// - /// Build a Qdrant from the provided . - /// - /// The to build a Qdrant from. - /// The model. - /// The Qdrant . - /// Thrown when the provided filter contains unsupported types, values or unknown properties. - public static Filter BuildFromLegacyFilter(VectorSearchFilter basicVectorSearchFilter, CollectionModel model) - { - var filter = new Filter(); - - foreach (var filterClause in basicVectorSearchFilter.FilterClauses) - { - string fieldName; - object filterValue; - - // In Qdrant, tag list contains is handled using a keyword match, which is the same as a string equality check. - // We can therefore just extract the field name and value from each clause and handle them the same. - if (filterClause is EqualToFilterClause equalityFilterClause) - { - fieldName = equalityFilterClause.FieldName; - filterValue = equalityFilterClause.Value; - } - else if (filterClause is AnyTagEqualToFilterClause tagListContainsClause) - { - fieldName = tagListContainsClause.FieldName; - filterValue = tagListContainsClause.Value; - } - else - { - throw new InvalidOperationException($"Unsupported filter clause type '{filterClause.GetType().Name}'."); - } - - // Get the storage name for the field. - if (!model.PropertyMap.TryGetValue(fieldName, out var property)) - { - throw new InvalidOperationException($"Property name '{fieldName}' provided as part of the filter clause is not a valid property name."); - } - - // Map DateTimeOffset equality. - if (filterValue is DateTimeOffset dateTimeOffset) - { - var range = new global::Qdrant.Client.Grpc.DatetimeRange - { - Gte = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTimeOffset(dateTimeOffset), - Lte = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTimeOffset(dateTimeOffset), - }; - - filter.Must.Add(new Condition() { Field = new FieldCondition() { Key = property.StorageName, DatetimeRange = range } }); - continue; - } - - // Map each type of filter value to the appropriate Qdrant match type. - var match = filterValue switch - { - string stringValue => new Match { Keyword = stringValue }, - int intValue => new Match { Integer = intValue }, - long longValue => new Match { Integer = longValue }, - bool boolValue => new Match { Boolean = boolValue }, - _ => throw new InvalidOperationException($"Unsupported filter value type '{filterValue.GetType().Name}'.") - }; - - filter.Must.Add(new Condition() { Field = new FieldCondition() { Key = property.StorageName, Match = match } }); - } - - return filter; - } -#pragma warning restore CS0618 // Type or member is obsolete - /// /// Map the given to a . /// diff --git a/dotnet/src/VectorData/Redis/RedisCollectionSearchMapping.cs b/dotnet/src/VectorData/Redis/RedisCollectionSearchMapping.cs index 442fb7e3557d..e9c36fd87605 100644 --- a/dotnet/src/VectorData/Redis/RedisCollectionSearchMapping.cs +++ b/dotnet/src/VectorData/Redis/RedisCollectionSearchMapping.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Linq; using System.Linq.Expressions; using System.Runtime.InteropServices; using Microsoft.Extensions.AI; @@ -53,15 +52,9 @@ public static Query BuildQuery(byte[] vectorBytes, int top, VectorSearc // Build search query. var redisLimit = top + options.Skip; -#pragma warning disable CS0618 // Type or member is obsolete - var filter = options switch - { - { OldFilter: not null, Filter: not null } => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"), - { OldFilter: VectorSearchFilter legacyFilter } => BuildLegacyFilter(legacyFilter, model), - { Filter: Expression> newFilter } => new RedisFilterTranslator().Translate(newFilter, model), - _ => "*" - }; -#pragma warning restore CS0618 // Type or member is obsolete + var filter = options.Filter is not null + ? new RedisFilterTranslator().Translate(options.Filter, model) + : "*"; var query = new Query($"{filter}=>[KNN {redisLimit} @{vectorProperty.StorageName} $embedding AS vector_score]") .AddParam("embedding", vectorBytes) @@ -102,47 +95,6 @@ internal static Query BuildQuery(Expression> filter return query; } - /// - /// Build a redis filter string from the provided . - /// - /// The to build the Redis filter string from. - /// The model. - /// The Redis filter string. - /// Thrown when a provided filter value is not supported. -#pragma warning disable CS0618 // Type or member is obsolete - public static string BuildLegacyFilter(VectorSearchFilter basicVectorSearchFilter, CollectionModel model) - { - var filterClauses = basicVectorSearchFilter.FilterClauses.Select(clause => - { - if (clause is EqualToFilterClause equalityFilterClause) - { - var storagePropertyName = GetStoragePropertyName(model, equalityFilterClause.FieldName); - - return equalityFilterClause.Value switch - { - string stringValue => $"@{storagePropertyName}:{{{stringValue}}}", - int intValue => $"@{storagePropertyName}:[{intValue} {intValue}]", - long longValue => $"@{storagePropertyName}:[{longValue} {longValue}]", - float floatValue => $"@{storagePropertyName}:[{floatValue} {floatValue}]", - double doubleValue => $"@{storagePropertyName}:[{doubleValue} {doubleValue}]", - _ => throw new InvalidOperationException($"Unsupported filter value type '{equalityFilterClause.Value.GetType().Name}'.") - }; - } - else if (clause is AnyTagEqualToFilterClause tagListContainsClause) - { - var storagePropertyName = GetStoragePropertyName(model, tagListContainsClause.FieldName); - return $"@{storagePropertyName}:{{{tagListContainsClause.Value}}}"; - } - else - { - throw new InvalidOperationException($"Unsupported filter clause type '{clause.GetType().Name}'."); - } - }); - - return $"({string.Join(" ", filterClauses)})"; - } -#pragma warning restore CS0618 // Type or member is obsolete - /// /// Resolve the distance function to use for a search by checking the distance function of the vector property specified in options /// or by falling back to the distance function of the first vector property, or by falling back to the default distance function. @@ -176,21 +128,4 @@ public static string ResolveDistanceFunction(VectorPropertyModel vectorProperty) _ => throw new InvalidOperationException($"The distance function '{distanceFunction}' is not supported."), }; } - - /// - /// Gets the name of the name under which the property with the given name is stored. - /// - /// The model. - /// The name of the property in the data model. - /// The name that the property os stored under. - /// Thrown when the property name is not found. - private static string GetStoragePropertyName(CollectionModel model, string fieldName) - { - if (!model.PropertyMap.TryGetValue(fieldName, out var property)) - { - throw new InvalidOperationException($"Property name '{fieldName}' provided as part of the filter clause is not a valid property name."); - } - - return property.StorageName; - } } diff --git a/dotnet/src/VectorData/SqlServer/SqlServerCollection.cs b/dotnet/src/VectorData/SqlServer/SqlServerCollection.cs index 93708f30a202..470be57ab494 100644 --- a/dotnet/src/VectorData/SqlServer/SqlServerCollection.cs +++ b/dotnet/src/VectorData/SqlServer/SqlServerCollection.cs @@ -582,12 +582,6 @@ public override async IAsyncEnumerable> SearchAsync< { throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); } -#pragma warning disable CS0618 // Type or member is obsolete - if (options.OldFilter is not null) - { - throw new NotSupportedException("The obsolete Filter is not supported by the SQL Server connector, use NewFilter instead."); - } -#pragma warning restore CS0618 // Type or member is obsolete var vectorProperty = this._model.GetVectorPropertyOrSingle(options); @@ -645,12 +639,6 @@ public async IAsyncEnumerable> HybridSearchAsync { VectorProperty = options.VectorProperty }); var textDataProperty = this._model.GetFullTextDataPropertyOrSingle(options.AdditionalProperty); diff --git a/dotnet/src/VectorData/SqliteVec/SqliteCollection.cs b/dotnet/src/VectorData/SqliteVec/SqliteCollection.cs index e9d667690301..03d7849c1ddf 100644 --- a/dotnet/src/VectorData/SqliteVec/SqliteCollection.cs +++ b/dotnet/src/VectorData/SqliteVec/SqliteCollection.cs @@ -207,33 +207,16 @@ _ when vectorProperty.EmbeddingGenerationDispatcher is not null new SqliteWhereEqualsCondition(LimitPropertyName, limit) }; -#pragma warning disable CS0618 // VectorSearchFilter is obsolete string? extraWhereFilter = null; Dictionary? extraParameters = null; - if (options.OldFilter is not null) - { - if (options.Filter is not null) - { - throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"); - } - - // Old filter, we translate it to a list of SqliteWhereCondition, and merge these into the conditions we already have - var filterConditions = this.GetFilterConditions(options.OldFilter, this._dataTableName); - - if (filterConditions is { Count: > 0 }) - { - conditions.AddRange(filterConditions); - } - } - else if (options.Filter is not null) + if (options.Filter is not null) { SqliteFilterTranslator translator = new(this._model, options.Filter); translator.Translate(appendWhere: false); extraWhereFilter = translator.Clause.ToString(); extraParameters = translator.Parameters; } -#pragma warning restore CS0618 // VectorSearchFilter is obsolete await foreach (var record in this.EnumerateAndMapSearchResultsAsync( conditions, @@ -696,45 +679,6 @@ private Task InternalDeleteBatchAsync(SqliteConnection connection, SqliteWhereCo return Task.WhenAll(tasks); } -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - private List? GetFilterConditions(VectorSearchFilter? filter, string? tableName = null) - { - var filterClauses = filter?.FilterClauses.ToList(); - - if (filterClauses is not { Count: > 0 }) - { - return null; - } - - var conditions = new List(); - - foreach (var filterClause in filterClauses) - { - if (filterClause is EqualToFilterClause equalToFilterClause) - { - if (!this._model.PropertyMap.TryGetValue(equalToFilterClause.FieldName, out var property)) - { - throw new InvalidOperationException($"Property name '{equalToFilterClause.FieldName}' provided as part of the filter clause is not a valid property name."); - } - - conditions.Add(new SqliteWhereEqualsCondition(property.StorageName, equalToFilterClause.Value) - { - TableName = tableName - }); - } - else - { - throw new NotSupportedException( - $"Unsupported filter clause type '{filterClause.GetType().Name}'. " + - $"Supported filter clause types are: {string.Join(", ", [ - nameof(EqualToFilterClause)])}"); - } - } - - return conditions; - } -#pragma warning restore CS0618 // VectorSearchFilter is obsolete - /// /// Gets vector table name. /// diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/HybridSearchOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/HybridSearchOptions.cs index d63dce6867e1..e3ca2cc0b430 100644 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/HybridSearchOptions.cs +++ b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/HybridSearchOptions.cs @@ -15,14 +15,6 @@ public class HybridSearchOptions /// /// Gets or sets a search filter to use before doing the hybrid search. /// -#pragma warning disable CS0618 // Type or member is obsolete - [Obsolete("Use Filter instead")] - public VectorSearchFilter? OldFilter { get; set; } -#pragma warning restore CS0618 // Type or member is obsolete - - /// - /// Gets or sets a search filter to use before doing the vector search. - /// public Expression>? Filter { get; set; } /// diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/RecordSearchOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/RecordSearchOptions.cs index 23eea309be67..207034bd95c9 100644 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/RecordSearchOptions.cs +++ b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/RecordSearchOptions.cs @@ -2,23 +2,16 @@ using System; using System.Linq.Expressions; -using System.Threading; namespace Microsoft.Extensions.VectorData; /// -/// Defines options for vector search via . +/// Defines options for vector search via . /// public class VectorSearchOptions { private int _skip = 0; - /// - /// Gets or sets a search filter to use before doing the vector search. - /// - [Obsolete("Use Filter instead")] - public VectorSearchFilter? OldFilter { get; set; } - /// /// Gets or sets a search filter to use before doing the vector search. /// @@ -29,7 +22,7 @@ public class VectorSearchOptions /// Only needs to be set when the collection has multiple vector properties. /// /// - /// If this property isn't set provided, checks if there is a vector property to use by default, and + /// If this property isn't set provided, checks if there is a vector property to use by default, and /// throws an exception if either none or multiple exist. /// public Expression>? VectorProperty { get; set; } diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchFilter.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchFilter.cs deleted file mode 100644 index 3142b6db343f..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchFilter.cs +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Provides filtering when doing vector searches. -/// Contains configuration for doing basic vector search filtering. -/// -/// -/// A filter has a collection of instances that can be used -/// to request that the underlying service filter the search results. -/// All clauses are combined with 'and'. -/// -[Obsolete("Use VectorSearchOptions.Filter instead of VectorSearchOptions.OldFilter")] -public sealed class VectorSearchFilter -{ - /// The filter clauses to 'and' together. - private readonly List _filterClauses = []; - - /// Gets the default search filter. - public static VectorSearchFilter Default { get; } = new VectorSearchFilter(); - - /// - /// Gets the filter clauses to 'and' together. - /// - public IEnumerable FilterClauses => this._filterClauses; - - /// - /// Creates a new instance of - /// - public VectorSearchFilter() - { - } - - /// - /// Creates a new instance of with the provided instances. - /// - /// The instances to use. - public VectorSearchFilter(IEnumerable filterClauses) - { - if (filterClauses == null) - { - throw new ArgumentNullException(nameof(filterClauses)); - } - - this._filterClauses.AddRange(filterClauses); - } - - /// - /// Adds an 'equal to' clause to the filter options. - /// - /// The name of the property to check against. Use the name of the property from your data model or as provided in the record definition. - /// The value that the property should match. - /// A instance to allow fluent configuration. - /// - /// This clause checks if a property is equal to a specific value. - /// - public VectorSearchFilter EqualTo(string propertyName, object value) - { - this._filterClauses.Add(new EqualToFilterClause(propertyName, value)); - return this; - } - - /// - /// Adds an 'any tag equal to' clause to the filter options. - /// - /// The name of the property consisting of a list of values to check against. Use the name of the property from your data model or as provided in the record definition. - /// The value that the list should contain. - /// A instance to allow fluent configuration. - /// - /// This clause checks if a property consisting of a list of values contains a specific value. - /// - public VectorSearchFilter AnyTagEqualTo(string propertyName, string value) - { - this._filterClauses.Add(new AnyTagEqualToFilterClause(propertyName, value)); - return this; - } -} diff --git a/dotnet/src/VectorData/Weaviate/WeaviateQueryBuilder.cs b/dotnet/src/VectorData/Weaviate/WeaviateQueryBuilder.cs index 48312ecf12a7..58aee493a149 100644 --- a/dotnet/src/VectorData/Weaviate/WeaviateQueryBuilder.cs +++ b/dotnet/src/VectorData/Weaviate/WeaviateQueryBuilder.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; using System.Text.Json; @@ -31,15 +30,9 @@ public static string BuildSearchQuery( { var vectorsQuery = GetVectorsPropertyQuery(searchOptions.IncludeVectors, hasNamedVectors, model); -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - var filter = searchOptions switch - { - { OldFilter: not null, Filter: not null } => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"), - { OldFilter: VectorSearchFilter legacyFilter } => BuildLegacyFilter(legacyFilter, jsonSerializerOptions, model), - { Filter: Expression> newFilter } => new WeaviateFilterTranslator().Translate(newFilter, model), - _ => null - }; -#pragma warning restore CS0618 + var filter = searchOptions.Filter is not null + ? new WeaviateFilterTranslator().Translate(searchOptions.Filter, model) + : null; var vectorArray = JsonSerializer.Serialize(vector, jsonSerializerOptions); @@ -138,15 +131,9 @@ public static string BuildHybridSearchQuery( // https://docs.weaviate.io/weaviate/api/graphql/search-operators#hybrid var vectorsQuery = GetVectorsPropertyQuery(searchOptions.IncludeVectors, hasNamedVectors, model); -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - var filter = searchOptions switch - { - { OldFilter: not null, Filter: not null } => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"), - { OldFilter: VectorSearchFilter legacyFilter } => BuildLegacyFilter(legacyFilter, jsonSerializerOptions, model), - { Filter: Expression> newFilter } => new WeaviateFilterTranslator().Translate(newFilter, model), - _ => null - }; -#pragma warning restore CS0618 + var filter = searchOptions.Filter is not null + ? new WeaviateFilterTranslator().Translate(searchOptions.Filter, model) + : null; var vectorArray = JsonSerializer.Serialize(vector, jsonSerializerOptions); var sanitizedKeywords = keywords.Replace("\\", "\\\\").Replace("\"", "\\\""); @@ -197,92 +184,5 @@ private static string GetVectorsPropertyQuery( : string.Empty; } -#pragma warning disable CS0618 // Type or member is obsolete - /// - /// Builds filter for Weaviate search query. - /// More information here: . - /// - private static string BuildLegacyFilter( - VectorSearchFilter? vectorSearchFilter, - JsonSerializerOptions jsonSerializerOptions, - CollectionModel model) - { - const string EqualOperator = "Equal"; - const string ContainsAnyOperator = "ContainsAny"; - - var filterClauses = vectorSearchFilter?.FilterClauses.ToList(); - - if (filterClauses is not { Count: > 0 }) - { - return string.Empty; - } - - var operands = new List(); - - foreach (var filterClause in filterClauses) - { - string filterValueType; - string propertyName; - object propertyValue; - string filterOperator; - - if (filterClause is EqualToFilterClause equalToFilterClause) - { - filterValueType = GetFilterValueType(equalToFilterClause.Value.GetType()); - propertyName = equalToFilterClause.FieldName; - propertyValue = JsonSerializer.Serialize(equalToFilterClause.Value, jsonSerializerOptions); - filterOperator = EqualOperator; - } - else if (filterClause is AnyTagEqualToFilterClause anyTagEqualToFilterClause) - { - filterValueType = GetFilterValueType(anyTagEqualToFilterClause.Value.GetType()); - propertyName = anyTagEqualToFilterClause.FieldName; - propertyValue = JsonSerializer.Serialize(new string[] { anyTagEqualToFilterClause.Value }, jsonSerializerOptions); - filterOperator = ContainsAnyOperator; - } - else - { - throw new NotSupportedException( - $"Unsupported filter clause type '{filterClause.GetType().Name}'. " + - $"Supported filter clause types are: {string.Join(", ", [ - nameof(EqualToFilterClause), - nameof(AnyTagEqualToFilterClause)])}"); - } - - if (!model.PropertyMap.TryGetValue(propertyName, out var property)) - { - throw new InvalidOperationException($"Property name '{propertyName}' provided as part of the filter clause is not a valid property name."); - } - - var storageName = property is KeyPropertyModel ? WeaviateConstants.ReservedKeyPropertyName : property.StorageName; - - var operand = $$"""{ path: ["{{JsonEncodedText.Encode(storageName)}}"], operator: {{filterOperator}}, {{filterValueType}}: {{propertyValue}} }"""; - - operands.Add(operand); - } - - return $$"""{ operator: And, operands: [{{string.Join(", ", operands)}}] }"""; - } -#pragma warning restore CS0618 // Type or member is obsolete - - /// - /// Gets filter value type. - /// More information here: . - /// - private static string GetFilterValueType(Type valueType) - { - return valueType switch - { - Type t when t == typeof(int) || t == typeof(long) || t == typeof(short) || t == typeof(byte) || - t == typeof(int?) || t == typeof(long?) || t == typeof(short?) || t == typeof(byte?) => "valueInt", - Type t when t == typeof(bool) || t == typeof(bool?) => "valueBoolean", - Type t when t == typeof(string) || t == typeof(Guid) || t == typeof(Guid?) => "valueText", - Type t when t == typeof(float) || t == typeof(double) || t == typeof(decimal) || - t == typeof(float?) || t == typeof(double?) || t == typeof(decimal?) => "valueNumber", - Type t when t == typeof(DateTimeOffset) || t == typeof(DateTimeOffset?) => "valueDate", - _ => throw new NotSupportedException($"Unsupported value type {valueType.FullName} in filter.") - }; - } - #endregion } diff --git a/dotnet/test/VectorData/AzureAISearch.UnitTests/AzureAISearchCollectionTests.cs b/dotnet/test/VectorData/AzureAISearch.UnitTests/AzureAISearchCollectionTests.cs index 120fb79efd0b..baf7b2dfa17d 100644 --- a/dotnet/test/VectorData/AzureAISearch.UnitTests/AzureAISearchCollectionTests.cs +++ b/dotnet/test/VectorData/AzureAISearch.UnitTests/AzureAISearchCollectionTests.cs @@ -20,8 +20,6 @@ namespace SemanticKernel.Connectors.AzureAISearch.UnitTests; -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - /// /// Contains tests for the class. /// @@ -441,7 +439,6 @@ public async Task CanSearchWithVectorAndFilterAsync() using var sut = new AzureAISearchCollection( this._searchIndexClientMock.Object, TestCollectionName); - var filter = new VectorSearchFilter().EqualTo(nameof(MultiPropsModel.Data1), "Data1FilterValue"); // Act. var searchResults = await sut.SearchAsync( @@ -450,7 +447,7 @@ public async Task CanSearchWithVectorAndFilterAsync() new() { Skip = 3, - OldFilter = filter, + Filter = r => r.Data1 == "Data1FilterValue", VectorProperty = record => record.Vector1 }, this._testCancellationToken).ToListAsync(); @@ -460,7 +457,7 @@ public async Task CanSearchWithVectorAndFilterAsync() x => x.SearchAsync( null, It.Is(x => - x.Filter == "storage_data1 eq 'Data1FilterValue'" && + x.Filter == "(storage_data1 eq 'Data1FilterValue')" && x.Size == 5 && x.Skip == 3 && x.VectorSearch.Queries.First().GetType() == typeof(VectorizedQuery) && @@ -483,7 +480,6 @@ public async Task CanSearchWithTextAndFilterAsync() using var sut = new AzureAISearchCollection( this._searchIndexClientMock.Object, TestCollectionName); - var filter = new VectorSearchFilter().EqualTo(nameof(MultiPropsModel.Data1), "Data1FilterValue"); // Act. var searchResults = await sut.SearchAsync( @@ -492,7 +488,7 @@ public async Task CanSearchWithTextAndFilterAsync() new() { Skip = 3, - OldFilter = filter, + Filter = r => r.Data1 == "Data1FilterValue", VectorProperty = record => record.Vector1 }, this._testCancellationToken).ToListAsync(); @@ -502,7 +498,7 @@ public async Task CanSearchWithTextAndFilterAsync() x => x.SearchAsync( null, It.Is(x => - x.Filter == "storage_data1 eq 'Data1FilterValue'" && + x.Filter == "(storage_data1 eq 'Data1FilterValue')" && x.Size == 5 && x.Skip == 3 && x.VectorSearch.Queries.First().GetType() == typeof(VectorizableTextQuery) && diff --git a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoFilterTests.cs b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoFilterTests.cs index 520f79ba0d0b..a644f1f1b00d 100644 --- a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoFilterTests.cs +++ b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoFilterTests.cs @@ -56,15 +56,6 @@ public override Task Contains_over_field_string_array() public override Task Contains_over_field_string_List() => Assert.ThrowsAsync(() => base.Contains_over_field_string_List()); - // AnyTagEqualTo not (currently) supported on SQLite - [Obsolete("Legacy filter support")] - public override Task Legacy_AnyTagEqualTo_array() - => Assert.ThrowsAsync(() => base.Legacy_AnyTagEqualTo_array()); - - [Obsolete("Legacy filter support")] - public override Task Legacy_AnyTagEqualTo_List() - => Assert.ThrowsAsync(() => base.Legacy_AnyTagEqualTo_List()); - public new class Fixture : FilterTests.Fixture { public override TestStore TestStore => CosmosMongoTestStore.Instance; diff --git a/dotnet/test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoCollectionSearchMappingTests.cs b/dotnet/test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoCollectionSearchMappingTests.cs deleted file mode 100644 index 165ad088fa5f..000000000000 --- a/dotnet/test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoCollectionSearchMappingTests.cs +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using Microsoft.Extensions.VectorData; -using Microsoft.Extensions.VectorData.ProviderServices; -using Microsoft.SemanticKernel.Connectors.CosmosMongoDB; -using Microsoft.SemanticKernel.Connectors.MongoDB; -using MongoDB.Bson; -using Xunit; - -namespace SemanticKernel.Connectors.CosmosMongoDB.UnitTests; - -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - -/// -/// Unit tests for class. -/// -public sealed class CosmosMongoCollectionSearchMappingTests -{ - private readonly CollectionModel _model = new MongoModelBuilder() - .BuildDynamic( - new() - { - Properties = - [ - new VectorStoreKeyProperty("Property1", typeof(string)) { StorageName = "property_1" }, - new VectorStoreDataProperty("Property2", typeof(string)) { StorageName = "property_2" } - ] - }, - defaultEmbeddingGenerator: null); - - [Fact] - public void BuildFilterWithNullVectorSearchFilterReturnsNull() - { - // Arrange - VectorSearchFilter? vectorSearchFilter = null; - - // Act - var filter = CosmosMongoCollectionSearchMapping.BuildFilter(vectorSearchFilter, this._model); - - // Assert - Assert.Null(filter); - } - - [Fact] - public void BuildFilterWithoutFilterClausesReturnsNull() - { - // Arrange - VectorSearchFilter vectorSearchFilter = new(); - - // Act - var filter = CosmosMongoCollectionSearchMapping.BuildFilter(vectorSearchFilter, this._model); - - // Assert - Assert.Null(filter); - } - - [Fact] - public void BuildFilterThrowsExceptionWithUnsupportedFilterClause() - { - // Arrange - var vectorSearchFilter = new VectorSearchFilter().AnyTagEqualTo("NonExistentProperty", "TestValue"); - - // Act & Assert - Assert.Throws(() => CosmosMongoCollectionSearchMapping.BuildFilter(vectorSearchFilter, this._model)); - } - - [Fact] - public void BuildFilterThrowsExceptionWithNonExistentPropertyName() - { - // Arrange - var vectorSearchFilter = new VectorSearchFilter().EqualTo("NonExistentProperty", "TestValue"); - - // Act & Assert - Assert.Throws(() => CosmosMongoCollectionSearchMapping.BuildFilter(vectorSearchFilter, this._model)); - } - - [Fact] - public void BuildFilterThrowsExceptionWithMultipleFilterClausesOfSameType() - { - // Arrange - var vectorSearchFilter = new VectorSearchFilter() - .EqualTo("Property1", "TestValue1") - .EqualTo("Property1", "TestValue2"); - - // Act & Assert - Assert.Throws(() => CosmosMongoCollectionSearchMapping.BuildFilter(vectorSearchFilter, this._model)); - } - - [Fact] - public void BuilderFilterByDefaultReturnsValidFilter() - { - // Arrange - var expectedFilter = new BsonDocument() { ["property_1"] = new BsonDocument() { ["$eq"] = "TestValue1" } }; - var vectorSearchFilter = new VectorSearchFilter().EqualTo("Property1", "TestValue1"); - - // Act - var filter = CosmosMongoCollectionSearchMapping.BuildFilter(vectorSearchFilter, this._model); - - Assert.Equal(expectedFilter.ToJson(), filter.ToJson()); - } - - private static CollectionModel BuildModel(List properties) - => new MongoModelBuilder() - .BuildDynamic( - new() { Properties = properties }, - defaultEmbeddingGenerator: null); -} diff --git a/dotnet/test/VectorData/CosmosNoSql.UnitTests/CosmosNoSqlCollectionQueryBuilderTests.cs b/dotnet/test/VectorData/CosmosNoSql.UnitTests/CosmosNoSqlCollectionQueryBuilderTests.cs index 8d28b32815a1..be278ab2e7ba 100644 --- a/dotnet/test/VectorData/CosmosNoSql.UnitTests/CosmosNoSqlCollectionQueryBuilderTests.cs +++ b/dotnet/test/VectorData/CosmosNoSql.UnitTests/CosmosNoSqlCollectionQueryBuilderTests.cs @@ -8,8 +8,6 @@ namespace SemanticKernel.Connectors.CosmosNoSql.UnitTests; -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - /// /// Unit tests for class. /// @@ -30,127 +28,6 @@ public sealed class CosmosNoSqlCollectionQueryBuilderTests }, defaultEmbeddingGenerator: null); - [Fact] - public void BuildSearchQueryByDefaultReturnsValidQueryDefinition() - { - // Arrange - var vector = new ReadOnlyMemory([1f, 2f, 3f]); - var vectorPropertyName = "test_property_1"; - - var filter = new VectorSearchFilter() - .EqualTo("TestProperty2", "test-value-2") - .AnyTagEqualTo("TestProperty3", "test-value-3"); - - // Act - var queryDefinition = CosmosNoSqlCollectionQueryBuilder.BuildSearchQuery( - vector, - keywords: null, - this._model, - vectorPropertyName, - distanceFunction: null, - textPropertyName: null, - ScorePropertyName, - oldFilter: filter, - filter: null, - scoreThreshold: null, - 10, - 5, - includeVectors: true); - - var queryText = queryDefinition.QueryText; - var queryParameters = queryDefinition.GetQueryParameters(); - - // Assert - Assert.Contains("SELECT x[\"id\"],x[\"TestProperty1\"],x[\"TestProperty2\"],x[\"TestProperty3\"],VectorDistance(x[\"test_property_1\"], @vector) AS TestScore", queryText); - Assert.Contains("FROM x", queryText); - Assert.Contains("WHERE x[\"TestProperty2\"] = @cv0 AND ARRAY_CONTAINS(x[\"TestProperty3\"], @cv1)", queryText); - Assert.Contains("ORDER BY VectorDistance(x[\"test_property_1\"], @vector)", queryText); - Assert.Contains("OFFSET 5 LIMIT 10", queryText); - - Assert.Equal("@vector", queryParameters[0].Name); - Assert.Equal(vector, queryParameters[0].Value); - - Assert.Equal("@cv0", queryParameters[1].Name); - Assert.Equal("test-value-2", queryParameters[1].Value); - - Assert.Equal("@cv1", queryParameters[2].Name); - Assert.Equal("test-value-3", queryParameters[2].Value); - } - - [Fact] - public void BuildSearchQueryWithoutOffsetReturnsQueryDefinitionWithTopParameter() - { - // Arrange - var vector = new ReadOnlyMemory([1f, 2f, 3f]); - var vectorPropertyName = "test_property_1"; - - var filter = new VectorSearchFilter() - .EqualTo("TestProperty2", "test-value-2") - .AnyTagEqualTo("TestProperty3", "test-value-3"); - - // Act - var queryDefinition = CosmosNoSqlCollectionQueryBuilder.BuildSearchQuery( - vector, - keywords: null, - this._model, - vectorPropertyName, - distanceFunction: null, - textPropertyName: null, - ScorePropertyName, - oldFilter: filter, - filter: null, - scoreThreshold: null, - 10, - 0, - includeVectors: true); - - var queryText = queryDefinition.QueryText; - var queryParameters = queryDefinition.GetQueryParameters(); - - // Assert - Assert.Contains("SELECT TOP 10 x[\"id\"],x[\"TestProperty1\"],x[\"TestProperty2\"],x[\"TestProperty3\"],VectorDistance(x[\"test_property_1\"], @vector) AS TestScore", queryText); - Assert.Contains("FROM x", queryText); - Assert.Contains("WHERE x[\"TestProperty2\"] = @cv0 AND ARRAY_CONTAINS(x[\"TestProperty3\"], @cv1)", queryText); - Assert.Contains("ORDER BY VectorDistance(x[\"test_property_1\"], @vector)", queryText); - - Assert.DoesNotContain("OFFSET 0 LIMIT 10", queryText); - - Assert.Equal("@vector", queryParameters[0].Name); - Assert.Equal(vector, queryParameters[0].Value); - - Assert.Equal("@cv0", queryParameters[1].Name); - Assert.Equal("test-value-2", queryParameters[1].Value); - - Assert.Equal("@cv1", queryParameters[2].Name); - Assert.Equal("test-value-3", queryParameters[2].Value); - } - - [Fact] - public void BuildSearchQueryWithInvalidFilterThrowsException() - { - // Arrange - var vector = new ReadOnlyMemory([1f, 2f, 3f]); - var vectorPropertyName = "test_property_1"; - - var filter = new VectorSearchFilter().EqualTo("non-existent-property", "test-value-2"); - - // Act & Assert - Assert.Throws(() => CosmosNoSqlCollectionQueryBuilder.BuildSearchQuery( - vector, - keywords: null, - this._model, - vectorPropertyName, - distanceFunction: null, - textPropertyName: null, - ScorePropertyName, - oldFilter: filter, - filter: null, - scoreThreshold: null, - 10, - 5, - includeVectors: true)); - } - [Fact] public void BuildSearchQueryWithoutFilterDoesNotContainWhereClause() { @@ -167,7 +44,6 @@ public void BuildSearchQueryWithoutFilterDoesNotContainWhereClause() distanceFunction: null, textPropertyName: null, ScorePropertyName, - oldFilter: null, filter: null, scoreThreshold: null, 10, @@ -185,58 +61,6 @@ public void BuildSearchQueryWithoutFilterDoesNotContainWhereClause() Assert.Equal(vector, queryParameters[0].Value); } - [Fact] - public void BuildSearchQueryWithHybridFieldsReturnsValidHybridQueryDefinition() - { - // Arrange - var vector = new ReadOnlyMemory([1f, 2f, 3f]); - var keywordText = "hybrid"; - var vectorPropertyName = "TestProperty1"; - var textPropertyName = "TestProperty2"; - - var filter = new VectorSearchFilter() - .EqualTo("TestProperty2", "test-value-2") - .AnyTagEqualTo("TestProperty3", "test-value-3"); - - // Act - var queryDefinition = CosmosNoSqlCollectionQueryBuilder.BuildSearchQuery( - vector, - [keywordText], - this._model, - vectorPropertyName, - distanceFunction: null, - textPropertyName, - ScorePropertyName, - oldFilter: filter, - filter: null, - scoreThreshold: null, - 10, - 5, - includeVectors: true); - - var queryText = queryDefinition.QueryText; - var queryParameters = queryDefinition.GetQueryParameters(); - - // Assert - Assert.Contains("SELECT x[\"id\"],x[\"TestProperty1\"],x[\"TestProperty2\"],x[\"TestProperty3\"],VectorDistance(x[\"TestProperty1\"], @vector) AS TestScore", queryText); - Assert.Contains("FROM x", queryText); - Assert.Contains("WHERE x[\"TestProperty2\"] = @cv0 AND ARRAY_CONTAINS(x[\"TestProperty3\"], @cv1)", queryText); - Assert.Contains("ORDER BY RANK RRF(VectorDistance(x[\"TestProperty1\"], @vector), FullTextScore(x[\"TestProperty2\"], @keyword0))", queryText); - Assert.Contains("OFFSET 5 LIMIT 10", queryText); - - Assert.Equal("@vector", queryParameters[0].Name); - Assert.Equal(vector, queryParameters[0].Value); - - Assert.Equal("@keyword0", queryParameters[1].Name); - Assert.Equal("hybrid", queryParameters[1].Value); - - Assert.Equal("@cv0", queryParameters[2].Name); - Assert.Equal("test-value-2", queryParameters[2].Value); - - Assert.Equal("@cv1", queryParameters[3].Name); - Assert.Equal("test-value-3", queryParameters[3].Value); - } - #pragma warning disable CA1812 // An internal class that is apparently never instantiated. If so, remove the code from the assembly. private sealed class DummyType; #pragma warning restore CA1812 diff --git a/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoFilterTests.cs b/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoFilterTests.cs index a37567ecac50..d1541b604db6 100644 --- a/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoFilterTests.cs +++ b/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoFilterTests.cs @@ -83,15 +83,6 @@ public override Task Any_with_Contains_over_captured_string_list() public override Task Any_over_List_with_Contains_over_captured_string_array() => Assert.ThrowsAsync(base.Any_over_List_with_Contains_over_captured_string_array); - // AnyTagEqualTo not (currently) supported on SQLite - [Obsolete("Legacy filter support")] - public override Task Legacy_AnyTagEqualTo_array() - => Assert.ThrowsAsync(base.Legacy_AnyTagEqualTo_array); - - [Obsolete("Legacy filter support")] - public override Task Legacy_AnyTagEqualTo_List() - => Assert.ThrowsAsync(base.Legacy_AnyTagEqualTo_List); - public new class Fixture : FilterTests.Fixture { public override TestStore TestStore => MongoTestStore.Instance; diff --git a/dotnet/test/VectorData/MongoDB.UnitTests/MongoCollectionSearchMappingTests.cs b/dotnet/test/VectorData/MongoDB.UnitTests/MongoCollectionSearchMappingTests.cs deleted file mode 100644 index c4dad5a08bf2..000000000000 --- a/dotnet/test/VectorData/MongoDB.UnitTests/MongoCollectionSearchMappingTests.cs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.Extensions.VectorData; -using Microsoft.Extensions.VectorData.ProviderServices; -using Microsoft.SemanticKernel.Connectors.MongoDB; -using MongoDB.Bson; -using Xunit; - -namespace SemanticKernel.Connectors.MongoDB.UnitTests; - -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - -/// -/// Unit tests for class. -/// -public sealed class MongoCollectionSearchMappingTests -{ - private readonly CollectionModel _model = new MongoModelBuilder() - .BuildDynamic( - new() - { - Properties = - [ - new VectorStoreKeyProperty("Property1", typeof(string)) { StorageName = "property_1" }, - new VectorStoreDataProperty("Property2", typeof(string)) { StorageName = "property_2" }, - ] - }, - defaultEmbeddingGenerator: null); - - [Fact] - public void BuildFilterThrowsExceptionWithUnsupportedFilterClause() - { - // Arrange - var vectorSearchFilter = new VectorSearchFilter().AnyTagEqualTo("NonExistentProperty", "TestValue"); - - // Act & Assert - Assert.Throws(() => MongoCollectionSearchMapping.BuildLegacyFilter(vectorSearchFilter, this._model)); - } - - [Fact] - public void BuildFilterThrowsExceptionWithNonExistentPropertyName() - { - // Arrange - var vectorSearchFilter = new VectorSearchFilter().EqualTo("NonExistentProperty", "TestValue"); - - // Act & Assert - Assert.Throws(() => MongoCollectionSearchMapping.BuildLegacyFilter(vectorSearchFilter, this._model)); - } - - [Fact] - public void BuildFilterThrowsExceptionWithMultipleFilterClausesOfSameType() - { - // Arrange - var vectorSearchFilter = new VectorSearchFilter() - .EqualTo("Property1", "TestValue1") - .EqualTo("Property1", "TestValue2"); - - // Act & Assert - Assert.Throws(() => MongoCollectionSearchMapping.BuildLegacyFilter(vectorSearchFilter, this._model)); - } - - [Fact] - public void BuilderFilterByDefaultReturnsValidFilter() - { - // Arrange - var expectedFilter = new BsonDocument() { ["property_1"] = new BsonDocument() { ["$eq"] = "TestValue1" } }; - var vectorSearchFilter = new VectorSearchFilter().EqualTo("Property1", "TestValue1"); - - // Act - var filter = MongoCollectionSearchMapping.BuildLegacyFilter(vectorSearchFilter, this._model); - - Assert.Equal(expectedFilter.ToJson(), filter.ToJson()); - } -} diff --git a/dotnet/test/VectorData/PgVector.ConformanceTests/PostgresFilterTests.cs b/dotnet/test/VectorData/PgVector.ConformanceTests/PostgresFilterTests.cs index 256f1e37481e..bf034413e2f0 100644 --- a/dotnet/test/VectorData/PgVector.ConformanceTests/PostgresFilterTests.cs +++ b/dotnet/test/VectorData/PgVector.ConformanceTests/PostgresFilterTests.cs @@ -35,10 +35,6 @@ await this.TestFilterAsync( r => r["String"] != null && r["String"] != "foo"); } - [Obsolete("Legacy filter support")] - public override Task Legacy_AnyTagEqualTo_array() - => Assert.ThrowsAsync(() => base.Legacy_AnyTagEqualTo_array()); - public new class Fixture : FilterTests.Fixture { public override TestStore TestStore => PostgresTestStore.Instance; diff --git a/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeFilterTests.cs b/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeFilterTests.cs index d9334580c450..b20a4d4a5b1d 100644 --- a/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeFilterTests.cs +++ b/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeFilterTests.cs @@ -86,15 +86,6 @@ public override Task Any_with_Contains_over_captured_string_list() public override Task Any_over_List_with_Contains_over_captured_string_array() => Assert.ThrowsAsync(base.Any_over_List_with_Contains_over_captured_string_array); - // AnyTagEqualTo not (currently) supported on Pinecone - [Obsolete("Legacy filter support")] - public override Task Legacy_AnyTagEqualTo_array() - => Assert.ThrowsAsync(base.Legacy_AnyTagEqualTo_array); - - [Obsolete("Legacy filter support")] - public override Task Legacy_AnyTagEqualTo_List() - => Assert.ThrowsAsync(base.Legacy_AnyTagEqualTo_List); - public new class Fixture : FilterTests.Fixture { public override TestStore TestStore => PineconeTestStore.Instance; diff --git a/dotnet/test/VectorData/Qdrant.UnitTests/QdrantCollectionSearchMappingTests.cs b/dotnet/test/VectorData/Qdrant.UnitTests/QdrantCollectionSearchMappingTests.cs index bbc5a62e3697..8dd9dbde3982 100644 --- a/dotnet/test/VectorData/Qdrant.UnitTests/QdrantCollectionSearchMappingTests.cs +++ b/dotnet/test/VectorData/Qdrant.UnitTests/QdrantCollectionSearchMappingTests.cs @@ -1,107 +1,17 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Linq; using Microsoft.Extensions.VectorData; -using Microsoft.Extensions.VectorData.ProviderServices; using Qdrant.Client.Grpc; using Xunit; namespace Microsoft.SemanticKernel.Connectors.Qdrant.UnitTests; -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - /// /// Contains tests for the class. /// public class QdrantCollectionSearchMappingTests { - private readonly CollectionModel _model = - new QdrantModelBuilder(hasNamedVectors: false) - .BuildDynamic( - new() - { - Properties = - [ - new VectorStoreKeyProperty("Key", typeof(Guid)) { StorageName = "storage_key" }, - new VectorStoreDataProperty("FieldName", typeof(string)) { StorageName = "storage_FieldName" }, - new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 10) { StorageName = "storage_vector" }, - ] - }, - defaultEmbeddingGenerator: null); - - [Theory] - [InlineData("string")] - [InlineData("int")] - [InlineData("bool")] - [InlineData("long")] - - public void BuildFilterMapsEqualityClause(string type) - { - // Arrange. - object expected = type switch - { - "string" => "Value", - "int" => 1, - "bool" => true, - "long" => 1L, - _ => throw new InvalidOperationException() - }; - var filter = new VectorSearchFilter().EqualTo("FieldName", expected); - - // Act. - var actual = QdrantCollectionSearchMapping.BuildFromLegacyFilter(filter, this._model); - - // Assert. - Assert.Single(actual.Must); - Assert.Equal("storage_FieldName", actual.Must.First().Field.Key); - - var match = actual.Must.First().Field.Match; - object actualSearchValue = type switch - { - "string" => match.Keyword, - "int" => match.Integer, - "bool" => match.Boolean, - "long" => match.Integer, - _ => throw new InvalidOperationException() - }; - - if (type == "int") - { - // Qdrant uses long for integers so we have to cast the expected value to long. - Assert.Equal((long)(int)expected, actualSearchValue); - } - else - { - Assert.Equal(expected, actualSearchValue); - } - } - - [Fact] - public void BuildFilterMapsTagContainsClause() - { - // Arrange. - var filter = new VectorSearchFilter().AnyTagEqualTo("FieldName", "Value"); - - // Act. - var actual = QdrantCollectionSearchMapping.BuildFromLegacyFilter(filter, this._model); - - // Assert. - Assert.Single(actual.Must); - Assert.Equal("storage_FieldName", actual.Must.First().Field.Key); - Assert.Equal("Value", actual.Must.First().Field.Match.Keyword); - } - - [Fact] - public void BuildFilterThrowsForUnknownFieldName() - { - // Arrange. - var filter = new VectorSearchFilter().EqualTo("UnknownFieldName", "Value"); - - // Act and Assert. - Assert.Throws(() => QdrantCollectionSearchMapping.BuildFromLegacyFilter(filter, this._model)); - } - [Fact] public void MapScoredPointToVectorSearchResultMapsResults() { diff --git a/dotnet/test/VectorData/Qdrant.UnitTests/QdrantCollectionTests.cs b/dotnet/test/VectorData/Qdrant.UnitTests/QdrantCollectionTests.cs index 262375d54b9e..13587e0ec3d7 100644 --- a/dotnet/test/VectorData/Qdrant.UnitTests/QdrantCollectionTests.cs +++ b/dotnet/test/VectorData/Qdrant.UnitTests/QdrantCollectionTests.cs @@ -460,7 +460,6 @@ public void CanCreateCollectionWithMismatchedDefinitionAndType() new() { Definition = definition }); } -#pragma warning disable CS0618 // VectorSearchFilter is obsolete [Theory] [MemberData(nameof(TestOptions))] public async Task CanSearchWithVectorAndFilterAsync(bool useDefinition, bool hasNamedVectors, TKey testRecordKey) @@ -471,13 +470,12 @@ public async Task CanSearchWithVectorAndFilterAsync(bool useDefinition, bo // Arrange. var scoredPoint = CreateScoredPoint(hasNamedVectors, testRecordKey); this.SetupQueryMock([scoredPoint]); - var filter = new VectorSearchFilter().EqualTo(nameof(SinglePropsModel.Data), "data 1"); // Act. var results = await sut.SearchAsync( new ReadOnlyMemory(new[] { 1f, 2f, 3f, 4f }), top: 5, - new() { IncludeVectors = true, OldFilter = filter, Skip = 2 }, + new() { IncludeVectors = true, Filter = r => r.Data == "data 1", Skip = 2 }, this._testCancellationToken).ToListAsync(); // Assert. @@ -509,7 +507,6 @@ public async Task CanSearchWithVectorAndFilterAsync(bool useDefinition, bo Assert.Equal(new float[] { 1, 2, 3, 4 }, results.First().Record.Vector!.Value.ToArray()); Assert.Equal(0.5f, results.First().Score); } -#pragma warning restore CS0618 // VectorSearchFilter is obsolete private void SetupRetrieveMock(List retrievedPoints) { diff --git a/dotnet/test/VectorData/Redis.ConformanceTests/RedisFilterTests.cs b/dotnet/test/VectorData/Redis.ConformanceTests/RedisFilterTests.cs index 3e94d68eafda..6bd0eccca629 100644 --- a/dotnet/test/VectorData/Redis.ConformanceTests/RedisFilterTests.cs +++ b/dotnet/test/VectorData/Redis.ConformanceTests/RedisFilterTests.cs @@ -128,14 +128,6 @@ public override Task Contains_with_MemoryExtensions_Contains_with_null_comparer( => Assert.ThrowsAsync(() => base.Contains_with_MemoryExtensions_Contains_with_null_comparer()); #endif - [Obsolete("Legacy filter support")] - public override Task Legacy_AnyTagEqualTo_array() - => Assert.ThrowsAsync(() => base.Legacy_AnyTagEqualTo_array()); - - [Obsolete("Legacy filter support")] - public override Task Legacy_AnyTagEqualTo_List() - => Assert.ThrowsAsync(() => base.Legacy_AnyTagEqualTo_List()); - // Array fields not supported on Redis HashSet - Any tests public override Task Any_with_Contains_over_inline_string_array() => Assert.ThrowsAsync(() => base.Any_with_Contains_over_inline_string_array()); diff --git a/dotnet/test/VectorData/Redis.UnitTests/RedisCollectionSearchMappingTests.cs b/dotnet/test/VectorData/Redis.UnitTests/RedisCollectionSearchMappingTests.cs index 75138404bd71..e25004b8953c 100644 --- a/dotnet/test/VectorData/Redis.UnitTests/RedisCollectionSearchMappingTests.cs +++ b/dotnet/test/VectorData/Redis.UnitTests/RedisCollectionSearchMappingTests.cs @@ -9,8 +9,6 @@ namespace Microsoft.SemanticKernel.Connectors.Redis.UnitTests; -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - /// /// Contains tests for the class. /// @@ -105,95 +103,6 @@ public void BuildQueryBuildsRedisQueryWithCustomVectorName() Assert.Equal("*=>[KNN 8 @storage_Vector $embedding AS vector_score]", query.QueryString); } - [Theory] - [InlineData("stringEquality")] - [InlineData("intEquality")] - [InlineData("longEquality")] - [InlineData("floatEquality")] - [InlineData("doubleEquality")] - [InlineData("tagContains")] - public void BuildFilterBuildsEqualityFilter(string filterType) - { - // Arrange. - var basicVectorSearchFilter = filterType switch - { - "stringEquality" => new VectorSearchFilter().EqualTo("Data1", "my value"), - "intEquality" => new VectorSearchFilter().EqualTo("Data1", 3), - "longEquality" => new VectorSearchFilter().EqualTo("Data1", 3L), - "floatEquality" => new VectorSearchFilter().EqualTo("Data1", 3.3f), - "doubleEquality" => new VectorSearchFilter().EqualTo("Data1", 3.3), - "tagContains" => new VectorSearchFilter().AnyTagEqualTo("Data1", "my value"), - _ => throw new InvalidOperationException(), - }; - - var model = BuildModel( - [ - new VectorStoreKeyProperty("Key", typeof(string)), - new VectorStoreDataProperty("Data1", typeof(string)) { StorageName = "storage_Data1" }, - new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 10) - ]); - - // Act. - var filter = RedisCollectionSearchMapping.BuildLegacyFilter(basicVectorSearchFilter, model); - - // Assert. - switch (filterType) - { - case "stringEquality": - Assert.Equal("(@storage_Data1:{my value})", filter); - break; - case "intEquality": - case "longEquality": - Assert.Equal("(@storage_Data1:[3 3])", filter); - break; - case "floatEquality": - case "doubleEquality": - Assert.Equal("(@storage_Data1:[3.3 3.3])", filter); - break; - case "tagContains": - Assert.Equal("(@storage_Data1:{my value})", filter); - break; - } - } - - [Fact] - public void BuildFilterThrowsForInvalidValueType() - { - // Arrange. - var basicVectorSearchFilter = new VectorSearchFilter().EqualTo("Data1", true); - var model = BuildModel( - [ - new VectorStoreKeyProperty("Key", typeof(string)), - new VectorStoreDataProperty("Data1", typeof(string)) { StorageName = "storage_Data1" }, - new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 10) - ]); - - // Act & Assert. - Assert.Throws(() => - { - var filter = RedisCollectionSearchMapping.BuildLegacyFilter(basicVectorSearchFilter, model); - }); - } - - [Fact] - public void BuildFilterThrowsForUnknownFieldName() - { - // Arrange. - var basicVectorSearchFilter = new VectorSearchFilter().EqualTo("UnknownData", "value"); - var model = BuildModel( - [ - new VectorStoreKeyProperty("Key", typeof(string)), - new VectorStoreDataProperty("Data1", typeof(string)) { StorageName = "storage_Data1" }, - new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 10) - ]); - - // Act & Assert. - Assert.Throws(() => - { - var filter = RedisCollectionSearchMapping.BuildLegacyFilter(basicVectorSearchFilter, model); - }); - } - [Fact] public void ResolveDistanceFunctionReturnsCosineSimilarityIfNoDistanceFunctionSpecified() { diff --git a/dotnet/test/VectorData/Redis.UnitTests/RedisHashSetCollectionTests.cs b/dotnet/test/VectorData/Redis.UnitTests/RedisHashSetCollectionTests.cs index cf8daefc3a2e..59fc50cbf70e 100644 --- a/dotnet/test/VectorData/Redis.UnitTests/RedisHashSetCollectionTests.cs +++ b/dotnet/test/VectorData/Redis.UnitTests/RedisHashSetCollectionTests.cs @@ -323,7 +323,6 @@ public async Task CanUpsertManyRecordsAsync(bool useDefinition) Times.Once); } -#pragma warning disable CS0618 // VectorSearchFilter is obsolete [Theory] [InlineData(true, true)] [InlineData(true, false)] @@ -351,8 +350,6 @@ public async Task CanSearchWithVectorAndFilterAsync(bool useDefinition, bool inc }); using var sut = this.CreateRecordCollection(useDefinition); - var filter = new VectorSearchFilter().EqualTo(nameof(SinglePropsModel.Data), "data 1"); - // Act. var results = await sut.SearchAsync( new ReadOnlyMemory(new[] { 1f, 2f, 3f, 4f }), @@ -360,7 +357,7 @@ public async Task CanSearchWithVectorAndFilterAsync(bool useDefinition, bool inc new() { IncludeVectors = includeVectors, - OldFilter = filter, + Filter = r => r.Data == "data 1", Skip = 2 }).ToListAsync(); @@ -368,7 +365,7 @@ public async Task CanSearchWithVectorAndFilterAsync(bool useDefinition, bool inc var expectedArgsPart1 = new object[] { "testcollection", - "(@data_storage_name:{data 1})=>[KNN 7 @vector_storage_name $embedding AS vector_score]", + "@data_storage_name:{\"data 1\"}=>[KNN 7 @vector_storage_name $embedding AS vector_score]", "WITHSCORES", "SORTBY", "vector_score", @@ -416,7 +413,6 @@ public async Task CanSearchWithVectorAndFilterAsync(bool useDefinition, bool inc Assert.False(results.First().Record.Vector.HasValue); } } -#pragma warning restore CS0618 // VectorSearchFilter is obsolete /// /// Tests that the collection can be created even if the definition and the type do not match. diff --git a/dotnet/test/VectorData/Redis.UnitTests/RedisJsonCollectionTests.cs b/dotnet/test/VectorData/Redis.UnitTests/RedisJsonCollectionTests.cs index 77199de035d8..6441e0ff894b 100644 --- a/dotnet/test/VectorData/Redis.UnitTests/RedisJsonCollectionTests.cs +++ b/dotnet/test/VectorData/Redis.UnitTests/RedisJsonCollectionTests.cs @@ -15,8 +15,6 @@ namespace Microsoft.SemanticKernel.Connectors.Redis.UnitTests; -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - /// /// Contains tests for the class. /// @@ -380,8 +378,6 @@ public async Task CanSearchWithVectorAndFilterAsync(bool useDefinition) }); using var sut = this.CreateRecordCollection(useDefinition); - var filter = new VectorSearchFilter().EqualTo(nameof(MultiPropsModel.Data1), "data 1"); - // Act. var results = await sut.SearchAsync( new ReadOnlyMemory(new[] { 1f, 2f, 3f, 4f }), @@ -389,7 +385,7 @@ public async Task CanSearchWithVectorAndFilterAsync(bool useDefinition) new() { IncludeVectors = true, - OldFilter = filter, + Filter = r => r.Data1 == "data 1", VectorProperty = r => r.Vector1, Skip = 2 }).ToListAsync(); @@ -398,7 +394,7 @@ public async Task CanSearchWithVectorAndFilterAsync(bool useDefinition) var expectedArgs = new object[] { "testcollection", - "(@data1_json_name:{data 1})=>[KNN 7 @vector1_json_name $embedding AS vector_score]", + "@data1_json_name:{\"data 1\"}=>[KNN 7 @vector1_json_name $embedding AS vector_score]", "WITHSCORES", "SORTBY", "vector_score", diff --git a/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerFilterTests.cs b/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerFilterTests.cs index 19247198d7ab..8be860c8d0ad 100644 --- a/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerFilterTests.cs +++ b/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerFilterTests.cs @@ -35,22 +35,6 @@ await this.TestFilterAsync( r => r["String"] != null && r["String"] != "foo"); } - [Fact(Skip = "Not supported")] - [Obsolete("Legacy filters are not supported")] - public override Task Legacy_And() => throw new NotSupportedException(); - - [Fact(Skip = "Not supported")] - [Obsolete("Legacy filters are not supported")] - public override Task Legacy_AnyTagEqualTo_array() => throw new NotSupportedException(); - - [Fact(Skip = "Not supported")] - [Obsolete("Legacy filters are not supported")] - public override Task Legacy_AnyTagEqualTo_List() => throw new NotSupportedException(); - - [Fact(Skip = "Not supported")] - [Obsolete("Legacy filters are not supported")] - public override Task Legacy_equality() => throw new NotSupportedException(); - public new class Fixture : FilterTests.Fixture { private static readonly string s_uniqueName = Guid.NewGuid().ToString(); diff --git a/dotnet/test/VectorData/SqliteVec.ConformanceTests/SqliteFilterTests.cs b/dotnet/test/VectorData/SqliteVec.ConformanceTests/SqliteFilterTests.cs index 0b5b38c0ff4b..c9399be8b937 100644 --- a/dotnet/test/VectorData/SqliteVec.ConformanceTests/SqliteFilterTests.cs +++ b/dotnet/test/VectorData/SqliteVec.ConformanceTests/SqliteFilterTests.cs @@ -72,15 +72,6 @@ public override Task Any_with_Contains_over_captured_string_list() public override Task Any_over_List_with_Contains_over_captured_string_array() => Assert.ThrowsAsync(base.Any_over_List_with_Contains_over_captured_string_array); - // AnyTagEqualTo not (currently) supported on SQLite - [Obsolete("Legacy filter support")] - public override Task Legacy_AnyTagEqualTo_array() - => Assert.ThrowsAsync(base.Legacy_AnyTagEqualTo_array); - - [Obsolete("Legacy filter support")] - public override Task Legacy_AnyTagEqualTo_List() - => Assert.ThrowsAsync(base.Legacy_AnyTagEqualTo_List); - public new class Fixture : FilterTests.Fixture { public override TestStore TestStore => SqliteTestStore.Instance; diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/FilterTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/FilterTests.cs index 1112df1056d0..cdb714eca080 100644 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/FilterTests.cs +++ b/dotnet/test/VectorData/VectorData.ConformanceTests/FilterTests.cs @@ -478,38 +478,6 @@ public virtual Task True() #endregion Miscellaneous - #region Legacy filter support - - [ConditionalFact] - [Obsolete("Legacy filter support")] - public virtual Task Legacy_equality() - => this.TestLegacyFilterAsync( - new VectorSearchFilter().EqualTo("Int", 8), - r => r.Int == 8); - - [ConditionalFact] - [Obsolete("Legacy filter support")] - public virtual Task Legacy_And() - => this.TestLegacyFilterAsync( - new VectorSearchFilter().EqualTo("Int", 8).EqualTo("String", "foo"), - r => r.Int == 8); - - [ConditionalFact] - [Obsolete("Legacy filter support")] - public virtual Task Legacy_AnyTagEqualTo_array() - => this.TestLegacyFilterAsync( - new VectorSearchFilter().AnyTagEqualTo("StringArray", "x"), - r => r.StringArray.Contains("x")); - - [ConditionalFact] - [Obsolete("Legacy filter support")] - public virtual Task Legacy_AnyTagEqualTo_List() - => this.TestLegacyFilterAsync( - new VectorSearchFilter().AnyTagEqualTo("StringList", "x"), - r => r.StringArray.Contains("x")); - - #endregion Legacy filter support - protected virtual async Task TestFilterAsync( Expression> filter, Expression, bool>> dynamicFilter, @@ -578,40 +546,6 @@ protected virtual async Task TestFilterAsync( } } - [Obsolete("Legacy filter support")] - protected virtual async Task TestLegacyFilterAsync( - VectorSearchFilter legacyFilter, - Expression> expectedFilter, - bool expectZeroResults = false, - bool expectAllResults = false) - { - var expected = fixture.TestData.AsQueryable().Where(expectedFilter).OrderBy(r => r.Key).ToList(); - - if (expected.Count == 0 && !expectZeroResults) - { - Assert.Fail("The test returns zero results, and so is unreliable"); - } - - if (expected.Count == fixture.TestData.Count && !expectAllResults) - { - Assert.Fail("The test returns all results, and so is unreliable"); - } - - var actual = await fixture.Collection.SearchAsync( - new ReadOnlyMemory([1, 2, 3]), - top: fixture.TestData.Count, - new() - { - OldFilter = legacyFilter - }) - .Select(r => r.Record).OrderBy(r => r.Key).ToListAsync(); - - foreach (var (e, a) in expected.Zip(actual, (e, a) => (e, a))) - { - fixture.AssertEqualFilterRecord(e, a); - } - } - public class FilterRecord : TestRecord { public ReadOnlyMemory? Vector { get; set; } diff --git a/dotnet/test/VectorData/Weaviate.UnitTests/WeaviateQueryBuilderTests.cs b/dotnet/test/VectorData/Weaviate.UnitTests/WeaviateQueryBuilderTests.cs index 8e6ad94bad1f..3bf911f0cecb 100644 --- a/dotnet/test/VectorData/Weaviate.UnitTests/WeaviateQueryBuilderTests.cs +++ b/dotnet/test/VectorData/Weaviate.UnitTests/WeaviateQueryBuilderTests.cs @@ -10,8 +10,6 @@ namespace SemanticKernel.Connectors.Weaviate.UnitTests; -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - /// /// Unit tests for class. /// @@ -130,81 +128,6 @@ public void BuildSearchQueryWithIncludedVectorsReturnsValidQuery(bool hasNamedVe Assert.Contains(vectorQuery, query); } - [Fact] - public void BuildSearchQueryWithFilterReturnsValidQuery() - { - // Arrange - const string ExpectedFirstSubquery = """{ path: ["HotelName"], operator: Equal, valueText: "Test Name" }"""; - const string ExpectedSecondSubquery = """{ path: ["Tags"], operator: ContainsAny, valueText: ["t1"] }"""; - - var searchOptions = new VectorSearchOptions - { - Skip = 2, - OldFilter = new VectorSearchFilter() - .EqualTo("HotelName", "Test Name") - .AnyTagEqualTo("Tags", "t1") - }; - - // Act - var query = WeaviateQueryBuilder.BuildSearchQuery( - this._vector, - CollectionName, - VectorPropertyName, - s_jsonSerializerOptions, - top: 3, - searchOptions, - this._model, - hasNamedVectors: true); - - // Assert - Assert.Contains(ExpectedFirstSubquery, query); - Assert.Contains(ExpectedSecondSubquery, query); - } - - [Fact] - public void BuildSearchQueryWithInvalidFilterValueThrowsException() - { - // Arrange - var searchOptions = new VectorSearchOptions - { - Skip = 2, - OldFilter = new VectorSearchFilter().EqualTo("HotelName", new TestFilterValue()) - }; - - // Act & Assert - Assert.Throws(() => WeaviateQueryBuilder.BuildSearchQuery( - this._vector, - CollectionName, - VectorPropertyName, - s_jsonSerializerOptions, - top: 3, - searchOptions, - this._model, - hasNamedVectors: true)); - } - - [Fact] - public void BuildSearchQueryWithNonExistentPropertyInFilterThrowsException() - { - // Arrange - var searchOptions = new VectorSearchOptions - { - Skip = 2, - OldFilter = new VectorSearchFilter().EqualTo("NonExistentProperty", "value") - }; - - // Act & Assert - Assert.Throws(() => WeaviateQueryBuilder.BuildSearchQuery( - this._vector, - CollectionName, - VectorPropertyName, - s_jsonSerializerOptions, - top: 3, - searchOptions, - this._model, - hasNamedVectors: true)); - } - [Fact] public void BuildHybridSearchQueryEscapesDoubleQuotesInKeywords() { @@ -285,7 +208,6 @@ public void BuildHybridSearchQueryWithPlainKeywordsWorks() #pragma warning disable CA1812 // An internal class that is apparently never instantiated. If so, remove the code from the assembly. private sealed class DummyType; #pragma warning restore CA1812 - private sealed class TestFilterValue; #endregion }