diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props
index adf5582061dd..75599a2d4751 100644
--- a/dotnet/Directory.Packages.props
+++ b/dotnet/Directory.Packages.props
@@ -27,7 +27,6 @@
-
@@ -105,9 +104,7 @@
-
-
@@ -171,10 +168,8 @@
-
-
@@ -186,20 +181,9 @@
-
-
-
-
-
-
-
-
-
-
-
diff --git a/dotnet/MEVD.slnx b/dotnet/MEVD.slnx
index 9f782e02e062..17ad49e1937e 100644
--- a/dotnet/MEVD.slnx
+++ b/dotnet/MEVD.slnx
@@ -20,47 +20,11 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/dotnet/src/VectorData/AzureAISearch/AssemblyInfo.cs b/dotnet/src/VectorData/AzureAISearch/AssemblyInfo.cs
deleted file mode 100644
index cbb67c1c8afd..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/AssemblyInfo.cs
+++ /dev/null
@@ -1 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearch.csproj b/dotnet/src/VectorData/AzureAISearch/AzureAISearch.csproj
deleted file mode 100644
index 54258372dc2a..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/AzureAISearch.csproj
+++ /dev/null
@@ -1,41 +0,0 @@
-
-
-
- Microsoft.SemanticKernel.Connectors.AzureAISearch
- Microsoft.SemanticKernel.Connectors.AzureAISearch
- net10.0;net8.0;netstandard2.0;net462
- preview
-
-
-
-
-
-
-
-
- Microsoft.SemanticKernel.Connectors.AzureAISearch
- Azure AI Search provider for Microsoft.Extensions.VectorData
- Azure AI Search provider for Microsoft.Extensions.VectorData by Semantic Kernel
- VECTORDATA-CONNECTORS-NUGET.md
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollection.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollection.cs
deleted file mode 100644
index 16ed75d6c6ee..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollection.cs
+++ /dev/null
@@ -1,816 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Diagnostics.CodeAnalysis;
-using System.Linq;
-using System.Linq.Expressions;
-using System.Runtime.CompilerServices;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-using System.Threading;
-using System.Threading.Tasks;
-using Azure;
-using Azure.Search.Documents;
-using Azure.Search.Documents.Indexes;
-using Azure.Search.Documents.Indexes.Models;
-using Azure.Search.Documents.Models;
-using Microsoft.Extensions.AI;
-using Microsoft.Extensions.VectorData;
-using Microsoft.Extensions.VectorData.ProviderServices;
-using MEAI = Microsoft.Extensions.AI;
-
-namespace Microsoft.SemanticKernel.Connectors.AzureAISearch;
-
-///
-/// Service for storing and retrieving vector records, that uses Azure AI Search as the underlying storage.
-///
-/// The data type of the record key. Must be .
-/// The data model to use for adding, updating and retrieving data from storage.
-#pragma warning disable CA1711 // Identifiers should not have incorrect suffix
-public class AzureAISearchCollection : VectorStoreCollection, IKeywordHybridSearchable
- where TKey : notnull
- where TRecord : class
-#pragma warning restore CA1711 // Identifiers should not have incorrect suffix
-{
- /// Metadata about vector store record collection.
- private readonly VectorStoreCollectionMetadata _collectionMetadata;
-
- /// The default options for vector search.
- private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new();
-
- /// The default options for hybrid vector search.
- private static readonly HybridSearchOptions s_defaultKeywordVectorizedHybridSearchOptions = new();
-
- /// Azure AI Search client that can be used to manage the list of indices in an Azure AI Search Service.
- private readonly SearchIndexClient _searchIndexClient;
-
- /// Azure AI Search client that can be used to manage data in an Azure AI Search Service index.
- private readonly SearchClient _searchClient;
-
- /// A mapper to use for converting between the data model and the Azure AI Search record.
- private readonly IAzureAISearchMapper _mappper;
-
- /// The model for this collection.
- private readonly CollectionModel _model;
-
- ///
- /// Initializes a new instance of the class.
- ///
- /// Azure AI Search client that can be used to manage the list of indices in an Azure AI Search Service.
- /// The name of the collection that this will access.
- /// Optional configuration options for this class.
- /// Thrown when is null.
- /// Thrown when options are misconfigured.
- [RequiresUnreferencedCode("The Azure AI Search provider is currently incompatible with trimming.")]
- [RequiresDynamicCode("The Azure AI Search provider is currently incompatible with NativeAOT.")]
- public AzureAISearchCollection(SearchIndexClient searchIndexClient, string name, AzureAISearchCollectionOptions? options = default)
- : this(
- searchIndexClient,
- name,
- static options => typeof(TRecord) == typeof(Dictionary)
- ? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(AzureAISearchDynamicCollection)))
- : new AzureAISearchModelBuilder()
- .Build(typeof(TRecord), typeof(TKey), options.Definition, options.EmbeddingGenerator, options.JsonSerializerOptions ?? JsonSerializerOptions.Default),
- options)
- {
- }
-
- internal AzureAISearchCollection(SearchIndexClient searchIndexClient, string name, Func modelFactory, AzureAISearchCollectionOptions? options)
- {
- // Verify.
- Verify.NotNull(searchIndexClient);
- Verify.NotNullOrWhiteSpace(name);
-
- if (typeof(TKey) != typeof(string) && typeof(TKey) != typeof(Guid) && typeof(TKey) != typeof(object))
- {
- throw new NotSupportedException("Only string and Guid keys are supported.");
- }
-
- options ??= AzureAISearchCollectionOptions.Default;
-
- // Assign.
- this.Name = name;
- this._model = modelFactory(options);
- this._searchIndexClient = searchIndexClient;
- this._searchClient = this._searchIndexClient.GetSearchClient(name);
-
- this._mappper = typeof(TRecord) == typeof(Dictionary) ?
- (IAzureAISearchMapper)(object)new AzureAISearchDynamicMapper(this._model, options.JsonSerializerOptions) :
- new AzureAISearchMapper(this._model, options.JsonSerializerOptions);
-
- this._collectionMetadata = new()
- {
- VectorStoreSystemName = AzureAISearchConstants.VectorStoreSystemName,
- VectorStoreName = searchIndexClient.ServiceName,
- CollectionName = name
- };
- }
-
- ///
- public override string Name { get; }
-
- ///
- public override async Task CollectionExistsAsync(CancellationToken cancellationToken = default)
- {
- try
- {
- await this._searchIndexClient.GetIndexAsync(this.Name, cancellationToken).ConfigureAwait(false);
- return true;
- }
- catch (RequestFailedException ex) when (ex.Status == 404)
- {
- return false;
- }
- catch (RequestFailedException ex)
- {
- throw new VectorStoreException("Call to vector store failed.", ex)
- {
- VectorStoreSystemName = AzureAISearchConstants.VectorStoreSystemName,
- VectorStoreName = this._collectionMetadata.VectorStoreName,
- CollectionName = this.Name,
- OperationName = "GetIndex"
- };
- }
- }
-
- ///
- public override async Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default)
- {
- const string OperationName = "CreateIndex";
-
- // Don't even try to create if the collection already exists.
- if (await this.CollectionExistsAsync(cancellationToken).ConfigureAwait(false))
- {
- return;
- }
-
- var vectorSearchConfig = new VectorSearch();
- var searchFields = new List();
-
- // Loop through all properties and create the search fields.
- foreach (var property in this._model.Properties)
- {
- switch (property)
- {
- case KeyPropertyModel p:
- searchFields.Add(AzureAISearchCollectionCreateMapping.MapKeyField(p));
- break;
-
- case DataPropertyModel p:
- searchFields.Add(AzureAISearchCollectionCreateMapping.MapDataField(p));
- break;
-
- case VectorPropertyModel p:
- (VectorSearchField vectorSearchField, VectorSearchAlgorithmConfiguration algorithmConfiguration, VectorSearchProfile vectorSearchProfile) = AzureAISearchCollectionCreateMapping.MapVectorField(p);
-
- // Add the search field, plus its profile and algorithm configuration to the search config.
- searchFields.Add(vectorSearchField);
- vectorSearchConfig.Algorithms.Add(algorithmConfiguration);
- vectorSearchConfig.Profiles.Add(vectorSearchProfile);
- break;
-
- default:
- throw new UnreachableException();
- }
- }
-
- // Create the index definition.
- var searchIndex = new SearchIndex(this.Name, searchFields);
- searchIndex.VectorSearch = vectorSearchConfig;
-
- try
- {
- await this._searchIndexClient.CreateIndexAsync(searchIndex, cancellationToken).ConfigureAwait(false);
- }
- catch (RequestFailedException ex) when (ex.ErrorCode == "ResourceNameAlreadyInUse")
- {
- // Index already exists, ignore.
- }
- catch (RequestFailedException ex)
- {
- throw new VectorStoreException("Call to vector store failed.", ex)
- {
- VectorStoreSystemName = AzureAISearchConstants.VectorStoreSystemName,
- VectorStoreName = this._collectionMetadata.VectorStoreName,
- CollectionName = this.Name,
- OperationName = OperationName
- };
- }
- catch (AggregateException ex) when (ex.InnerException is RequestFailedException innerEx)
- {
- throw new VectorStoreException("Call to vector store failed.", ex)
- {
- VectorStoreSystemName = AzureAISearchConstants.VectorStoreSystemName,
- VectorStoreName = this._collectionMetadata.VectorStoreName,
- CollectionName = this.Name,
- OperationName = OperationName
- };
- }
- }
-
- ///
- public override Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default)
- {
- return this.RunOperationAsync(
- "DeleteIndex",
- async () =>
- {
- try
- {
- return await this._searchIndexClient.DeleteIndexAsync(this.Name, cancellationToken).ConfigureAwait(false);
- }
- catch (RequestFailedException ex) when (ex.Status == 404)
- {
- return null!;
- }
- });
- }
-
- ///
- public override Task GetAsync(TKey key, RecordRetrievalOptions? options = default, CancellationToken cancellationToken = default)
- {
- // Create Options.
- var innerOptions = this.ConvertGetDocumentOptions(options);
- var includeVectors = options?.IncludeVectors ?? false;
- if (includeVectors && this._model.EmbeddingGenerationRequired)
- {
- throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration);
- }
-
- // Get record.
- return this.GetDocumentAndMapToDataModelAsync(key, includeVectors, innerOptions, cancellationToken);
- }
-
- ///
- public override async IAsyncEnumerable GetAsync(IEnumerable keys, RecordRetrievalOptions? options = default, [EnumeratorCancellation] CancellationToken cancellationToken = default)
- {
- Verify.NotNull(keys);
-
- // Create Options
- var innerOptions = this.ConvertGetDocumentOptions(options);
- var includeVectors = options?.IncludeVectors ?? false;
- if (includeVectors && this._model.EmbeddingGenerationRequired)
- {
- throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration);
- }
-
- foreach (var key in keys)
- {
- var record = await this.GetDocumentAndMapToDataModelAsync(key, includeVectors, innerOptions, cancellationToken).ConfigureAwait(false);
-
- if (record is not null)
- {
- yield return record;
- }
- }
- }
-
- ///
- public override Task DeleteAsync(TKey key, CancellationToken cancellationToken = default)
- {
- var stringKey = GetStringKey(key);
-
- // Remove record.
- return this.RunOperationAsync(
- "DeleteDocuments",
- () => this._searchClient.DeleteDocumentsAsync(this._model.KeyProperty.StorageName, [stringKey], new IndexDocumentsOptions(), cancellationToken));
- }
-
- ///
- public override Task DeleteAsync(IEnumerable keys, CancellationToken cancellationToken = default)
- {
- Verify.NotNull(keys);
- if (!keys.Any())
- {
- return Task.CompletedTask;
- }
-
- var stringKeys = keys is IEnumerable k ? k : keys.Select(GetStringKey);
-
- // Remove records.
- return this.RunOperationAsync(
- "DeleteDocuments",
- () => this._searchClient.DeleteDocumentsAsync(this._model.KeyProperty.StorageName, stringKeys, new IndexDocumentsOptions(), cancellationToken));
- }
-
- ///
- public override async Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default)
- {
- Verify.NotNull(record);
-
- // Create options.
- var innerOptions = new IndexDocumentsOptions { ThrowOnAnyError = true };
-
- // Upsert record.
- await this.MapToStorageModelAndUploadDocumentAsync([record], innerOptions, cancellationToken).ConfigureAwait(false);
- }
-
- ///
- public override async Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default)
- {
- Verify.NotNull(records);
- if (!records.Any())
- {
- return;
- }
-
- // Create Options
- var innerOptions = new IndexDocumentsOptions { ThrowOnAnyError = true };
-
- // Upsert records
- await this.MapToStorageModelAndUploadDocumentAsync(records, innerOptions, cancellationToken).ConfigureAwait(false);
- }
-
- ///
- public override IAsyncEnumerable GetAsync(Expression> filter, int top,
- FilteredRecordRetrievalOptions? options = null, CancellationToken cancellationToken = default)
- {
- Verify.NotNull(filter);
- Verify.NotLessThan(top, 1);
-
- options ??= new();
-
- var includeVectors = options.IncludeVectors;
- if (includeVectors && this._model.EmbeddingGenerationRequired)
- {
- throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration);
- }
-
- SearchOptions searchOptions = new()
- {
- VectorSearch = new(),
- Size = top,
- Skip = options.Skip,
- Filter = new AzureAISearchFilterTranslator().Translate(filter, this._model)
- };
-
- // Filter out vector fields if requested.
- if (!options.IncludeVectors)
- {
- searchOptions.Select.Add(this._model.KeyProperty.StorageName);
-
- foreach (var dataProperty in this._model.DataProperties)
- {
- searchOptions.Select.Add(dataProperty.StorageName);
- }
- }
-
- if (options.OrderBy is not null)
- {
- foreach (var pair in options.OrderBy(new()).Values)
- {
- PropertyModel property = this._model.GetDataOrKeyProperty(pair.PropertySelector);
- string name = property.StorageName;
- // From https://learn.microsoft.com/dotnet/api/azure.search.documents.searchoptions.orderby:
- // "Each expression can be followed by asc to indicate ascending, or desc to indicate descending".
- // "The default is ascending order."
- if (!pair.Ascending)
- {
- name += " desc";
- }
-
- searchOptions.OrderBy.Add(name);
- }
- }
-
- return this.SearchAndMapToDataModelAsync(null, searchOptions, options.IncludeVectors, cancellationToken)
- .Select(result => result.Record);
- }
-
- #region Search
-
- ///
- public override async IAsyncEnumerable> SearchAsync(
- TInput searchValue,
- int top,
- VectorSearchOptions? options = null,
- [EnumeratorCancellation] CancellationToken cancellationToken = default)
- {
- Verify.NotNull(searchValue);
- Verify.NotLessThan(top, 1);
-
- options ??= s_defaultVectorSearchOptions;
- var vectorProperty = this._model.GetVectorPropertyOrSingle(options);
- var floatVector = await GetSearchVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false);
-
- var searchOptions = BuildSearchOptions(
- this._model,
- options,
- top,
- floatVector is null
- ? new VectorizableTextQuery((string)(object)searchValue) { KNearestNeighborsCount = top + options.Skip, Fields = { vectorProperty.StorageName } }
- : new VectorizedQuery(floatVector.Value) { KNearestNeighborsCount = top + options.Skip, Fields = { vectorProperty.StorageName } });
-
- await foreach (var record in this.SearchAndMapToDataModelAsync(null, searchOptions, options.IncludeVectors, cancellationToken).ConfigureAwait(false))
- {
- // Azure AI Search threshold filtering is in preview:
- // https://learn.microsoft.com/azure/search/vector-search-how-to-query#set-thresholds-to-exclude-low-scoring-results-preview
- // See https://github.com/microsoft/semantic-kernel/issues/13500.
- // For now, perform post-filtering on the client-side.
- if (options.ScoreThreshold.HasValue && record.Score < options.ScoreThreshold.Value)
- {
- continue;
- }
-
- yield return record;
- }
- }
-
- ///
- public async IAsyncEnumerable> HybridSearchAsync(
- TInput searchValue,
- ICollection keywords,
- int top,
- HybridSearchOptions? options = null,
- [EnumeratorCancellation] CancellationToken cancellationToken = default)
- where TInput : notnull
- {
- Verify.NotNull(keywords);
- Verify.NotLessThan(top, 1);
-
- // Resolve options.
- options ??= s_defaultKeywordVectorizedHybridSearchOptions;
- var vectorProperty = this._model.GetVectorPropertyOrSingle(new() { VectorProperty = options.VectorProperty });
- var floatVector = await GetSearchVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false);
-
- var textDataProperty = this._model.GetFullTextDataPropertyOrSingle(options.AdditionalProperty);
-
- // Build search options.
- var searchOptions = BuildSearchOptions(
- this._model,
- new()
- {
- Filter = options.Filter,
- VectorProperty = options.VectorProperty,
- Skip = options.Skip,
- },
- top,
- floatVector is null
- ? new VectorizableTextQuery((string)(object)searchValue) { KNearestNeighborsCount = top + options.Skip, Fields = { vectorProperty.StorageName } }
- : new VectorizedQuery(floatVector.Value) { KNearestNeighborsCount = top + options.Skip, Fields = { vectorProperty.StorageName } });
-
- searchOptions.SearchFields.Add(textDataProperty.StorageName);
- var keywordsCombined = string.Join(" ", keywords);
-
- await foreach (var record in this.SearchAndMapToDataModelAsync(keywordsCombined, searchOptions, options.IncludeVectors, cancellationToken).ConfigureAwait(false))
- {
- // Azure AI Search returns scores where higher values indicate more relevant results.
- if (options.ScoreThreshold.HasValue && record.Score < options.ScoreThreshold.Value)
- {
- continue;
- }
-
- yield return record;
- }
- }
-
- private static async ValueTask?> GetSearchVectorAsync(TInput searchValue, VectorPropertyModel vectorProperty, CancellationToken cancellationToken)
- where TInput : notnull
- => searchValue switch
- {
- ReadOnlyMemory r => r,
- float[] f => new ReadOnlyMemory(f),
- Embedding e => e.Vector,
- _ when vectorProperty.EmbeddingGenerationDispatcher is not null
- => ((Embedding)await vectorProperty.GenerateEmbeddingAsync(searchValue, cancellationToken).ConfigureAwait(false)).Vector,
-
- // A string was passed without an embedding generator being configured; send the string to Azure AI Search for backend embedding generation.
- string when vectorProperty.EmbeddingGenerator is null => (ReadOnlyMemory?)null,
-
- _ => vectorProperty.EmbeddingGenerator is null
- ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), AzureAISearchModelBuilder.SupportedVectorTypes))
- : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType()))
- };
-
- #endregion Search
-
- ///
- public override object? GetService(Type serviceType, object? serviceKey = null)
- {
- Verify.NotNull(serviceType);
-
- return
- serviceKey is not null ? null :
- serviceType == typeof(VectorStoreCollectionMetadata) ? this._collectionMetadata :
- serviceType == typeof(SearchIndexClient) ? this._searchIndexClient :
- serviceType == typeof(SearchClient) ? this._searchClient :
- serviceType.IsInstanceOfType(this) ? this :
- null;
- }
-
- ///
- /// Get the document with the given key and map it to the data model using the configured mapper type.
- ///
- /// The key of the record to get.
- /// A value indicating whether to include vectors in the result or not.
- /// The Azure AI Search sdk options for getting a document.
- /// The to monitor for cancellation requests. The default is .
- /// The retrieved document, mapped to the consumer data model.
- private async Task GetDocumentAndMapToDataModelAsync(
- TKey key,
- bool includeVectors,
- GetDocumentOptions innerOptions,
- CancellationToken cancellationToken)
- {
- const string OperationName = "GetDocument";
-
- var stringKey = GetStringKey(key);
-
- var jsonObject = await this.RunOperationAsync(
- OperationName,
- () => this.GetDocumentWithNotFoundHandlingAsync(this._searchClient, stringKey, innerOptions, cancellationToken)).ConfigureAwait(false);
-
- if (jsonObject is null)
- {
- return default;
- }
-
- return (TRecord)(object)this._mappper!.MapFromStorageToDataModel(jsonObject, includeVectors);
- }
-
- ///
- /// Search for the documents matching the given options and map them to the data model using the configured mapper type.
- ///
- /// Text to use if doing a hybrid search. Null for non-hybrid search.
- /// The options controlling the behavior of the search operation.
- /// A value indicating whether to include vectors in the result or not.
- /// The to monitor for cancellation requests. The default is .
- /// The mapped search results.
- private async IAsyncEnumerable> SearchAndMapToDataModelAsync(
- string? searchText,
- SearchOptions searchOptions,
- bool includeVectors,
- [EnumeratorCancellation] CancellationToken cancellationToken)
- {
- const string OperationName = "Search";
-
- var jsonObjectResults = await this.RunOperationAsync(
- OperationName,
- () => this._searchClient.SearchAsync(searchText, searchOptions, cancellationToken)).ConfigureAwait(false);
-
- await foreach (var result in this.MapSearchResultsAsync(jsonObjectResults.Value.GetResultsAsync(), OperationName, includeVectors).ConfigureAwait(false))
- {
- yield return result;
- }
- }
-
- ///
- /// Map the data model to the storage model and upload the document.
- ///
- /// The records to upload.
- /// The Azure AI Search sdk options for uploading a document.
- /// The to monitor for cancellation requests. The default is .
- /// The document upload result.
- private async Task> MapToStorageModelAndUploadDocumentAsync(
- IEnumerable records,
- IndexDocumentsOptions innerOptions,
- CancellationToken cancellationToken)
- {
- const string OperationName = "UploadDocuments";
-
- (records, var generatedEmbeddings) = await ProcessEmbeddingsAsync(this._model, records, cancellationToken).ConfigureAwait(false);
-
- // Handle auto-generated keys (client-side for Azure AI Search, which doesn't support server-side auto-generation)
- var keyProperty = this._model.KeyProperty;
- var jsonObjects = new List();
- var recordIndex = 0;
- foreach (var record in records)
- {
- if (keyProperty.IsAutoGenerated && keyProperty.GetValue(record) == Guid.Empty)
- {
- keyProperty.SetValue(record, Guid.NewGuid());
- }
-
- jsonObjects.Add(this._mappper!.MapFromDataToStorageModel(record, recordIndex++, generatedEmbeddings));
- }
-
- return await this.RunOperationAsync(
- OperationName,
- () => this._searchClient.UploadDocumentsAsync(jsonObjects, innerOptions, cancellationToken)).ConfigureAwait(false);
- }
-
- ///
- /// Map the search results from to objects using the configured mapper type.
- ///
- /// The search results to map.
- /// The name of the current operation for telemetry purposes.
- /// A value indicating whether to include vectors in the resultset or not.
- /// The mapped results.
- private async IAsyncEnumerable> MapSearchResultsAsync(IAsyncEnumerable> results, string operationName, bool includeVectors)
- {
- await foreach (var result in results.ConfigureAwait(false))
- {
- var document = (TRecord)(object)this._mappper!.MapFromStorageToDataModel(result.Document, includeVectors);
- yield return new VectorSearchResult(document, result.Score);
- }
- }
-
- ///
- /// Map the search results from to objects.
- ///
- /// The search results to map.
- /// The mapped results.
- private async IAsyncEnumerable> MapSearchResultsAsync(IAsyncEnumerable> results)
- {
- await foreach (var result in results.ConfigureAwait(false))
- {
- yield return new VectorSearchResult(result.Document, result.Score);
- }
- }
-
- ///
- /// Convert the public options model to the Azure AI Search options model.
- ///
- /// The public options model.
- /// The Azure AI Search options model.
- private GetDocumentOptions ConvertGetDocumentOptions(RecordRetrievalOptions? options)
- {
- var innerOptions = new GetDocumentOptions();
- if (options?.IncludeVectors is not true)
- {
- innerOptions.SelectedFields.Add(this._model.KeyProperty.StorageName);
-
- foreach (var dataProperty in this._model.DataProperties)
- {
- innerOptions.SelectedFields.Add(dataProperty.StorageName);
- }
- }
-
- return innerOptions;
- }
-
- ///
- /// Build the search options for a vector search, where the type of vector search can be provided as input.
- /// E.g. VectorizedQuery or VectorizableTextQuery.
- ///
- private static SearchOptions BuildSearchOptions(CollectionModel model, VectorSearchOptions options, int top, VectorQuery? vectorQuery)
- {
- if (model.VectorProperties.Count == 0)
- {
- throw new InvalidOperationException("The collection does not have any vector fields, so vector search is not possible.");
- }
-
- if (options.IncludeVectors && model.EmbeddingGenerationRequired)
- {
- throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration);
- }
-
- // Build filter object.
- var filter = options.Filter is not null
- ? new AzureAISearchFilterTranslator().Translate(options.Filter, model)
- : null;
-
- // Build search options.
- var searchOptions = new SearchOptions
- {
- VectorSearch = new(),
- Size = top,
- Skip = options.Skip,
- };
-
- if (filter is not null)
- {
- searchOptions.Filter = filter;
- }
-
- searchOptions.VectorSearch.Queries.Add(vectorQuery);
-
- // Filter out vector fields if requested.
- if (!options.IncludeVectors)
- {
- searchOptions.Select.Add(model.KeyProperty.StorageName);
-
- foreach (var dataProperty in model.DataProperties)
- {
- searchOptions.Select.Add(dataProperty.StorageName);
- }
- }
-
- return searchOptions;
- }
-
- private static async ValueTask<(IEnumerable records, IReadOnlyList?[]?)> ProcessEmbeddingsAsync(
- CollectionModel model,
- IEnumerable records,
- CancellationToken cancellationToken)
- {
- IReadOnlyList? recordsList = null;
-
- // If an embedding generator is defined, invoke it once per property for all records.
- IReadOnlyList?[]? generatedEmbeddings = null;
-
- var vectorPropertyCount = model.VectorProperties.Count;
- for (var i = 0; i < vectorPropertyCount; i++)
- {
- var vectorProperty = model.VectorProperties[i];
-
- if (AzureAISearchModelBuilder.IsVectorPropertyTypeValidCore(vectorProperty.Type, out _))
- {
- continue;
- }
-
- // We have a vector property whose type isn't natively supported - we need to generate embeddings.
- Debug.Assert(vectorProperty.EmbeddingGenerator is not null);
-
- // We have a property with embedding generation; materialize the records' enumerable if needed, to
- // prevent multiple enumeration.
- if (recordsList is null)
- {
- recordsList = records is IReadOnlyList r ? r : records.ToList();
-
- if (recordsList.Count == 0)
- {
- return (records, null);
- }
-
- records = recordsList;
- }
-
- // TODO: Ideally we'd group together vector properties using the same generator (and with the same input and output properties),
- // and generate embeddings for them in a single batch. That's some more complexity though.
- generatedEmbeddings ??= new IReadOnlyList?[vectorPropertyCount];
- generatedEmbeddings[i] = await vectorProperty.GenerateEmbeddingsAsync(records.Select(r => vectorProperty.GetValueAsObject(r)), cancellationToken).ConfigureAwait(false);
- }
-
- return (records, generatedEmbeddings);
- }
-
- ///
- /// Get a document with the given key, and return null if it is not found.
- ///
- /// The type to deserialize the document to.
- /// The search client to use when fetching the document.
- /// The key of the record to get.
- /// The Azure AI Search sdk options for getting a document.
- /// The to monitor for cancellation requests. The default is .
- /// The retrieved document, mapped to the consumer data model, or null if not found.
- private async Task GetDocumentWithNotFoundHandlingAsync(
- SearchClient searchClient,
- string key,
- GetDocumentOptions innerOptions,
- CancellationToken cancellationToken)
- {
- const string OperationName = "GetDocument";
-
- try
- {
- return await searchClient.GetDocumentAsync(key, innerOptions, cancellationToken).ConfigureAwait(false);
- }
- catch (RequestFailedException ex) when (ex.Status == 404)
- {
- return default;
- }
- catch (AggregateException ex) when (ex.InnerException is RequestFailedException innerEx)
- {
- throw new VectorStoreException("Call to vector store failed.", ex)
- {
- VectorStoreSystemName = AzureAISearchConstants.VectorStoreSystemName,
- VectorStoreName = this._collectionMetadata.VectorStoreName,
- CollectionName = this.Name,
- OperationName = OperationName
- };
- }
- catch (RequestFailedException ex)
- {
- throw new VectorStoreException("Call to vector store failed.", ex)
- {
- VectorStoreSystemName = AzureAISearchConstants.VectorStoreSystemName,
- VectorStoreName = this._collectionMetadata.VectorStoreName,
- CollectionName = this.Name,
- OperationName = OperationName
- };
- }
- }
-
- ///
- /// Run the given operation and wrap any with ."/>
- ///
- /// The response type of the operation.
- /// The type of database operation being run.
- /// The operation to run.
- /// The result of the operation.
- private Task RunOperationAsync(string operationName, Func> operation) =>
- VectorStoreErrorHandler.RunOperationAsync(
- this._collectionMetadata,
- operationName,
- operation);
-
- private static string GetStringKey(TKey key)
- {
- Verify.NotNull(key);
-
- var stringKey = key switch
- {
- string s => s,
- Guid g => g.ToString(),
-
- _ => throw new UnreachableException("string key should have been validated during model building")
- };
-
- Verify.NotNullOrWhiteSpace(stringKey, nameof(key));
-
- return stringKey;
- }
-}
diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionCreateMapping.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionCreateMapping.cs
deleted file mode 100644
index c21053e15dda..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionCreateMapping.cs
+++ /dev/null
@@ -1,160 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Collections.Generic;
-using Azure.Search.Documents.Indexes.Models;
-using Microsoft.Extensions.VectorData;
-using Microsoft.Extensions.VectorData.ProviderServices;
-
-namespace Microsoft.SemanticKernel.Connectors.AzureAISearch;
-
-///
-/// Contains mapping helpers to use when creating a Azure AI Search vector collection.
-///
-internal static class AzureAISearchCollectionCreateMapping
-{
- ///
- /// Map from a to an Azure AI Search .
- ///
- /// The key property definition.
- /// The for the provided property definition.
- public static SearchableField MapKeyField(KeyPropertyModel keyProperty)
- {
- return new SearchableField(keyProperty.StorageName) { IsKey = true, IsFilterable = true };
- }
-
- ///
- /// Map from a to an Azure AI Search .
- ///
- /// The data property definition.
- /// The for the provided property definition.
- /// Throws when the definition is missing required information.
- public static SimpleField MapDataField(DataPropertyModel dataProperty)
- {
- if (dataProperty.IsFullTextIndexed)
- {
- if (dataProperty.Type != typeof(string))
- {
- throw new InvalidOperationException($"Property {nameof(dataProperty.IsFullTextIndexed)} on {nameof(VectorStoreDataProperty)} '{dataProperty.ModelName}' is set to true, but the property type is not a string. The Azure AI Search VectorStore supports {nameof(dataProperty.IsFullTextIndexed)} on string properties only.");
- }
-
- return new SearchableField(dataProperty.StorageName)
- {
- IsFilterable = dataProperty.IsIndexed,
- // Sometimes the users ask to also OrderBy given filterable property, so we make it sortable.
- IsSortable = dataProperty.IsIndexed
- };
- }
-
- var fieldType = AzureAISearchCollectionCreateMapping.GetSDKFieldDataType(dataProperty.Type);
- return new SimpleField(dataProperty.StorageName, fieldType)
- {
- IsFilterable = dataProperty.IsIndexed,
- // Sometimes the users ask to also OrderBy given filterable property, so we make it sortable.
- IsSortable = dataProperty.IsIndexed && !fieldType.IsCollection
- };
- }
-
- ///
- /// Map form a to an Azure AI Search and generate the required index configuration.
- ///
- /// The vector property definition.
- /// The and required index configuration.
- /// Throws when the definition is missing required information, or unsupported options are configured.
- public static (VectorSearchField vectorSearchField, VectorSearchAlgorithmConfiguration algorithmConfiguration, VectorSearchProfile vectorSearchProfile) MapVectorField(VectorPropertyModel vectorProperty)
- {
- // Build a name for the profile and algorithm configuration based on the property name
- // since we'll just create a separate one for each vector property.
- var vectorSearchProfileName = $"{vectorProperty.StorageName}Profile";
- var algorithmConfigName = $"{vectorProperty.StorageName}AlgoConfig";
-
- // Read the vector index settings from the property definition and create the right index configuration.
- var indexKind = AzureAISearchCollectionCreateMapping.GetSKIndexKind(vectorProperty);
- var algorithmMetric = AzureAISearchCollectionCreateMapping.GetSDKDistanceAlgorithm(vectorProperty);
-
- VectorSearchAlgorithmConfiguration algorithmConfiguration = indexKind switch
- {
- IndexKind.Hnsw => new HnswAlgorithmConfiguration(algorithmConfigName) { Parameters = new HnswParameters { Metric = algorithmMetric } },
- IndexKind.Flat => new ExhaustiveKnnAlgorithmConfiguration(algorithmConfigName) { Parameters = new ExhaustiveKnnParameters { Metric = algorithmMetric } },
-
- _ => throw new NotSupportedException($"Index kind '{indexKind}' on {nameof(VectorStoreVectorProperty)} '{vectorProperty.ModelName}' is not supported by the Azure AI Search VectorStore.")
- };
-
- var vectorSearchProfile = new VectorSearchProfile(vectorSearchProfileName, algorithmConfigName);
-
- return (new VectorSearchField(vectorProperty.StorageName, vectorProperty.Dimensions, vectorSearchProfileName), algorithmConfiguration, vectorSearchProfile);
- }
-
- ///
- /// Get the configured from the given .
- /// If none is configured the default is .
- ///
- /// The vector property definition.
- /// The configured or default .
- public static string GetSKIndexKind(VectorPropertyModel vectorProperty)
- => vectorProperty.IndexKind ?? IndexKind.Hnsw;
-
- ///
- /// Get the configured from the given .
- /// If none is configured, the default is .
- ///
- /// The vector property definition.
- /// The chosen .
- /// Thrown if a distance function is chosen that isn't supported by Azure AI Search.
- public static VectorSearchAlgorithmMetric GetSDKDistanceAlgorithm(VectorPropertyModel vectorProperty)
- => vectorProperty.DistanceFunction switch
- {
- DistanceFunction.CosineSimilarity or null => VectorSearchAlgorithmMetric.Cosine,
- DistanceFunction.DotProductSimilarity => VectorSearchAlgorithmMetric.DotProduct,
- DistanceFunction.EuclideanDistance => VectorSearchAlgorithmMetric.Euclidean,
-
- _ => throw new NotSupportedException($"Distance function '{vectorProperty.DistanceFunction}' for {nameof(VectorStoreVectorProperty)} '{vectorProperty.ModelName}' is not supported by the Azure AI Search VectorStore.")
- };
-
- ///
- /// Maps the given property type to the corresponding .
- ///
- /// The property type to map.
- /// The that corresponds to the given property type."
- /// Thrown if the given type is not supported.
- public static SearchFieldDataType GetSDKFieldDataType(Type propertyType)
- => (Nullable.GetUnderlyingType(propertyType) ?? propertyType) switch
- {
- Type t when t == typeof(string) => SearchFieldDataType.String,
- Type t when t == typeof(bool) => SearchFieldDataType.Boolean,
- Type t when t == typeof(int) => SearchFieldDataType.Int32,
- Type t when t == typeof(long) => SearchFieldDataType.Int64,
- // We don't map float to SearchFieldDataType.Single, because Azure AI Search doesn't support it.
- // Half is also listed by the SDK, but currently not supported.
- Type t when t == typeof(float) => SearchFieldDataType.Double,
- Type t when t == typeof(double) => SearchFieldDataType.Double,
- Type t when t == typeof(DateTime) => SearchFieldDataType.DateTimeOffset,
- Type t when t == typeof(DateTimeOffset) => SearchFieldDataType.DateTimeOffset,
-#if NET
- Type t when t == typeof(DateOnly) => SearchFieldDataType.DateTimeOffset,
-#endif
-
- Type t when t == typeof(string[]) => SearchFieldDataType.Collection(SearchFieldDataType.String),
- Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.String),
- Type t when t == typeof(bool[]) => SearchFieldDataType.Collection(SearchFieldDataType.Boolean),
- Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.Boolean),
- Type t when t == typeof(int[]) => SearchFieldDataType.Collection(SearchFieldDataType.Int32),
- Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.Int32),
- Type t when t == typeof(long[]) => SearchFieldDataType.Collection(SearchFieldDataType.Int64),
- Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.Int64),
- Type t when t == typeof(float[]) => SearchFieldDataType.Collection(SearchFieldDataType.Double),
- Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.Double),
- Type t when t == typeof(double[]) => SearchFieldDataType.Collection(SearchFieldDataType.Double),
- Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.Double),
- Type t when t == typeof(DateTime[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
- Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
- Type t when t == typeof(DateTimeOffset[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
- Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
-#if NET
- Type t when t == typeof(DateOnly[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
- Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
-#endif
-
- _ => throw new NotSupportedException($"Data type '{propertyType}' for {nameof(VectorStoreDataProperty)} is not supported by the Azure AI Search VectorStore.")
- };
-}
diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionOptions.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionOptions.cs
deleted file mode 100644
index 4197fbad7ddb..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionOptions.cs
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System.Text.Json;
-using Azure.Search.Documents.Indexes;
-using Microsoft.Extensions.VectorData;
-
-namespace Microsoft.SemanticKernel.Connectors.AzureAISearch;
-
-///
-/// Options when creating a .
-///
-public sealed class AzureAISearchCollectionOptions : VectorStoreCollectionOptions
-{
- internal static readonly AzureAISearchCollectionOptions Default = new();
-
- ///
- /// Initializes a new instance of the class.
- ///
- public AzureAISearchCollectionOptions()
- {
- }
-
- internal AzureAISearchCollectionOptions(AzureAISearchCollectionOptions? source) : base(source)
- {
- this.JsonSerializerOptions = source?.JsonSerializerOptions;
- }
-
- ///
- /// Gets or sets the JSON serializer options to use when converting between the data model and the Azure AI Search record.
- /// Note that when using the default mapper and you are constructing your own , you will need
- /// to provide the same set of both here and when constructing the .
- ///
- public JsonSerializerOptions? JsonSerializerOptions { get; set; }
-}
diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchConstants.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchConstants.cs
deleted file mode 100644
index fccfa847a0a3..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchConstants.cs
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-namespace Microsoft.SemanticKernel.Connectors.AzureAISearch;
-
-internal static class AzureAISearchConstants
-{
- internal const string VectorStoreSystemName = "azure.aisearch";
-}
diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicCollection.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicCollection.cs
deleted file mode 100644
index 934bf9bc34d0..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicCollection.cs
+++ /dev/null
@@ -1,35 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using Azure.Search.Documents.Indexes;
-
-namespace Microsoft.SemanticKernel.Connectors.AzureAISearch;
-
-///
-/// Represents a collection of vector store records in a AzureAISearch database, mapped to a dynamic Dictionary<string, object?>.
-///
-#pragma warning disable CA1711 // Identifiers should not have incorrect suffix
-public sealed class AzureAISearchDynamicCollection : AzureAISearchCollection>
-#pragma warning restore CA1711 // Identifiers should not have incorrect suffix
-{
- ///
- /// Initializes a new instance of the class.
- ///
- /// Azure AI Search client that can be used to manage the list of indices in an Azure AI Search Service.
- /// The name of the collection.
- /// Optional configuration options for this class.
- [RequiresUnreferencedCode("The Azure AI Search provider is currently incompatible with trimming.")]
- [RequiresDynamicCode("The Azure AI Search provider is currently incompatible with NativeAOT.")]
- public AzureAISearchDynamicCollection(SearchIndexClient searchIndexClient, string name, AzureAISearchCollectionOptions options)
- : base(
- searchIndexClient,
- name,
- static options => new AzureAISearchDynamicModelBuilder().BuildDynamic(
- options.Definition ?? throw new ArgumentException("Definition is required for dynamic collections"),
- options.EmbeddingGenerator),
- options)
- {
- }
-}
diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicMapper.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicMapper.cs
deleted file mode 100644
index 5b8024e5505d..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicMapper.cs
+++ /dev/null
@@ -1,283 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-using Microsoft.Extensions.AI;
-using Microsoft.Extensions.VectorData.ProviderServices;
-using MEAI = Microsoft.Extensions.AI;
-
-namespace Microsoft.SemanticKernel.Connectors.AzureAISearch;
-
-///
-/// A mapper that maps between the generic Semantic Kernel data model and the model that the data is stored under, within Azure AI Search.
-///
-internal sealed class AzureAISearchDynamicMapper(CollectionModel model, JsonSerializerOptions? jsonSerializerOptions) : IAzureAISearchMapper>
-{
- ///
- public JsonObject MapFromDataToStorageModel(Dictionary dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings)
- {
- Verify.NotNull(dataModel);
-
- var jsonObject = new JsonObject
- {
- [model.KeyProperty.StorageName] = !dataModel.TryGetValue(model.KeyProperty.ModelName, out var keyValue)
- ? throw new InvalidOperationException($"Missing value for key property '{model.KeyProperty.ModelName}")
- : keyValue switch
- {
- string s => s,
- Guid g => g.ToString(),
-
- null => throw new InvalidOperationException($"Key property '{model.KeyProperty.ModelName}' is null."),
- _ => throw new InvalidCastException($"Key property '{model.KeyProperty.ModelName}' must be a string or Guid.")
- }
- };
-
- foreach (var dataProperty in model.DataProperties)
- {
- if (dataModel.TryGetValue(dataProperty.ModelName, out var dataValue))
- {
- jsonObject[dataProperty.StorageName] = dataValue is not null ?
- JsonSerializer.SerializeToNode(ConvertToStorageValue(dataValue), GetStorageType(dataProperty.Type), jsonSerializerOptions) :
- null;
- }
- }
-
- for (var i = 0; i < model.VectorProperties.Count; i++)
- {
- var property = model.VectorProperties[i];
-
- // Don't create a property if it doesn't exist in the dictionary
- if (dataModel.TryGetValue(property.ModelName, out var vectorValue))
- {
- var vector = generatedEmbeddings?[i]?[recordIndex] is Embedding ge
- ? ge
- : vectorValue;
-
- if (vector is null)
- {
- jsonObject[property.StorageName] = null;
- continue;
- }
-
- var memory = vector switch
- {
- ReadOnlyMemory m => m,
- Embedding e => e.Vector,
- float[] a => a,
-
- _ => throw new UnreachableException()
- };
-
- var jsonArray = new JsonArray();
-
- foreach (var item in memory.Span)
- {
- jsonArray.Add(JsonValue.Create(item));
- }
-
- jsonObject.Add(property.StorageName, jsonArray);
- }
- }
-
- return jsonObject;
- }
-
- ///
- public Dictionary MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors)
- {
- Verify.NotNull(storageModel);
-
- // Create variables to store the response properties.
- var result = new Dictionary();
-
- // Loop through all known properties and map each from json to the data type.
- foreach (var property in model.Properties)
- {
- switch (property)
- {
- case KeyPropertyModel keyProperty:
- var key = (string?)storageModel[keyProperty.StorageName]
- ?? throw new InvalidOperationException($"The key property '{keyProperty.StorageName}' is missing from the record retrieved from storage.");
-
- result[keyProperty.ModelName] = keyProperty.Type switch
- {
- var t when t == typeof(string) => key,
- var t when t == typeof(Guid) => Guid.Parse(key),
- _ => throw new UnreachableException()
- };
-
- continue;
-
- case DataPropertyModel dataProperty:
- {
- if (storageModel.TryGetPropertyValue(dataProperty.StorageName, out var value))
- {
- result.Add(dataProperty.ModelName, value is null ? null : GetDataPropertyValue(property.Type, value));
- }
- continue;
- }
-
- case VectorPropertyModel vectorProperty when includeVectors:
- {
- if (storageModel.TryGetPropertyValue(vectorProperty.StorageName, out var value))
- {
- if (value is null)
- {
- result.Add(vectorProperty.ModelName, null);
- continue;
- }
-
- var vector = value.AsArray().Select(x => (float)x!).ToArray();
- result.Add(
- vectorProperty.ModelName,
- (Nullable.GetUnderlyingType(vectorProperty.Type) ?? vectorProperty.Type) switch
- {
- Type t when t == typeof(ReadOnlyMemory) => new ReadOnlyMemory(vector),
- Type t when t == typeof(Embedding) => new Embedding(vector),
- Type t when t == typeof(float[]) => vector,
-
- _ => throw new UnreachableException()
- });
- }
-
- continue;
- }
-
- case VectorPropertyModel vectorProperty when !includeVectors:
- break;
-
- default:
- throw new UnreachableException();
- }
- }
-
- return result;
- }
-
- ///
- /// Get the value of the given json node as the given property type.
- ///
- /// The type of property that is required.
- /// The json node containing the property value.
- /// The value of the json node as the required type.
- private static object? GetDataPropertyValue(Type propertyType, JsonNode value)
- {
- var result = propertyType switch
- {
- Type t when t == typeof(string) => value.GetValue(),
- Type t when t == typeof(int) => value.GetValue(),
- Type t when t == typeof(int?) => value.GetValue(),
- Type t when t == typeof(long) => value.GetValue(),
- Type t when t == typeof(long?) => value.GetValue(),
- Type t when t == typeof(float) => value.GetValue(),
- Type t when t == typeof(float?) => value.GetValue(),
- Type t when t == typeof(double) => value.GetValue(),
- Type t when t == typeof(double?) => value.GetValue(),
- Type t when t == typeof(bool) => value.GetValue(),
- Type t when t == typeof(bool?) => value.GetValue(),
- Type t when t == typeof(DateTime) => value.GetValue(),
- Type t when t == typeof(DateTime?) => value.GetValue(),
- Type t when t == typeof(DateTimeOffset) => value.GetValue(),
- Type t when t == typeof(DateTimeOffset?) => value.GetValue(),
-#if NET
- Type t when t == typeof(DateOnly) => DateOnly.FromDateTime(value.GetValue().DateTime),
- Type t when t == typeof(DateOnly?) => (DateOnly?)DateOnly.FromDateTime(value.GetValue().DateTime),
-#endif
-
- _ => (object?)null
- };
-
- if (result is not null)
- {
- return result;
- }
-
- if (propertyType.IsArray)
- {
- return propertyType switch
- {
- Type t when t == typeof(string[]) => value.AsArray().Select(x => (string?)x).ToArray(),
- Type t when t == typeof(int[]) => value.AsArray().Select(x => (int)x!).ToArray(),
- Type t when t == typeof(int?[]) => value.AsArray().Select(x => (int?)x).ToArray(),
- Type t when t == typeof(long[]) => value.AsArray().Select(x => (long)x!).ToArray(),
- Type t when t == typeof(long?[]) => value.AsArray().Select(x => (long?)x).ToArray(),
- Type t when t == typeof(float[]) => value.AsArray().Select(x => (float)x!).ToArray(),
- Type t when t == typeof(float?[]) => value.AsArray().Select(x => (float?)x).ToArray(),
- Type t when t == typeof(double[]) => value.AsArray().Select(x => (double)x!).ToArray(),
- Type t when t == typeof(double?[]) => value.AsArray().Select(x => (double?)x).ToArray(),
- Type t when t == typeof(bool[]) => value.AsArray().Select(x => (bool)x!).ToArray(),
- Type t when t == typeof(bool?[]) => value.AsArray().Select(x => (bool?)x).ToArray(),
- Type t when t == typeof(DateTime[]) => value.AsArray().Select(x => (DateTime)x!).ToArray(),
- Type t when t == typeof(DateTime?[]) => value.AsArray().Select(x => (DateTime?)x).ToArray(),
- Type t when t == typeof(DateTimeOffset[]) => value.AsArray().Select(x => (DateTimeOffset)x!).ToArray(),
- Type t when t == typeof(DateTimeOffset?[]) => value.AsArray().Select(x => (DateTimeOffset?)x).ToArray(),
-
- _ => throw new UnreachableException($"Unsupported property type '{propertyType.Name}'.")
- };
- }
-
- if (propertyType.IsGenericType && propertyType.GetGenericTypeDefinition() == typeof(List<>))
- {
- return propertyType switch
- {
- Type t when t == typeof(List) => value.AsArray().Select(x => (string?)x).ToList(),
- Type t when t == typeof(List) => value.AsArray().Select(x => (int)x!).ToList(),
- Type t when t == typeof(List) => value.AsArray().Select(x => (int?)x).ToList(),
- Type t when t == typeof(List) => value.AsArray().Select(x => (long)x!).ToList(),
- Type t when t == typeof(List) => value.AsArray().Select(x => (long?)x).ToList(),
- Type t when t == typeof(List) => value.AsArray().Select(x => (float)x!).ToList(),
- Type t when t == typeof(List) => value.AsArray().Select(x => (float?)x).ToList(),
- Type t when t == typeof(List) => value.AsArray().Select(x => (double)x!).ToList(),
- Type t when t == typeof(List) => value.AsArray().Select(x => (double?)x).ToList(),
- Type t when t == typeof(List) => value.AsArray().Select(x => (bool)x!).ToList(),
- Type t when t == typeof(List) => value.AsArray().Select(x => (bool?)x).ToList(),
- Type t when t == typeof(List) => value.AsArray().Select(x => (DateTime)x!).ToList(),
- Type t when t == typeof(List) => value.AsArray().Select(x => (DateTime?)x).ToList(),
- Type t when t == typeof(List) => value.AsArray().Select(x => (DateTimeOffset)x!).ToList(),
- Type t when t == typeof(List) => value.AsArray().Select(x => (DateTimeOffset?)x).ToList(),
-
- _ => throw new UnreachableException($"Unsupported property type '{propertyType.Name}'.")
- };
- }
-
- throw new UnreachableException($"Unsupported property type '{propertyType.Name}'.");
- }
-
- ///
- /// Converts a value to its storage representation for Azure AI Search.
- /// Azure AI Search only supports for date/time types, and always internally
- /// converts to UTC for storage.
- ///
- ///
- /// Gets the storage type for a given property type. Azure AI Search stores DateTime and DateOnly as DateTimeOffset.
- ///
- private static Type GetStorageType(Type propertyType)
- {
- var underlying = Nullable.GetUnderlyingType(propertyType) ?? propertyType;
-
- if (underlying == typeof(DateTime)
-#if NET
- || underlying == typeof(DateOnly)
-#endif
- )
- {
- return propertyType == underlying ? typeof(DateTimeOffset) : typeof(DateTimeOffset?);
- }
-
- return propertyType;
- }
-
- private static object ConvertToStorageValue(object value)
- => value switch
- {
- DateTime dateTime => new DateTimeOffset(dateTime, TimeSpan.Zero),
-#if NET
- DateOnly dateOnly => new DateTimeOffset(dateOnly.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero),
-#endif
- _ => value
- };
-}
diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicModelBuilder.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicModelBuilder.cs
deleted file mode 100644
index 2ddd797ef6a4..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicModelBuilder.cs
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Diagnostics.CodeAnalysis;
-
-using Microsoft.Extensions.VectorData.ProviderServices;
-
-namespace Microsoft.SemanticKernel.Connectors.AzureAISearch;
-
-internal class AzureAISearchDynamicModelBuilder() : CollectionModelBuilder(s_modelBuildingOptions)
-{
- internal static readonly CollectionModelBuildingOptions s_modelBuildingOptions = new()
- {
- RequiresAtLeastOneVector = false,
- SupportsMultipleVectors = true,
- UsesExternalSerializer = true
- };
-
- protected override void ValidateKeyProperty(KeyPropertyModel keyProperty)
- {
- base.ValidateKeyProperty(keyProperty);
-
- var type = keyProperty.Type;
-
- if (type != typeof(string) && type != typeof(Guid))
- {
- throw new NotSupportedException(
- $"Property '{keyProperty.ModelName}' has unsupported type '{type.Name}'. Key properties must be one of the supported types: string, Guid.");
- }
- }
-
- protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
- => AzureAISearchModelBuilder.IsDataPropertyTypeValidCore(type, out supportedTypes);
-
- protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
- => AzureAISearchModelBuilder.IsVectorPropertyTypeValidCore(type, out supportedTypes);
-}
diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchFilterTranslator.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchFilterTranslator.cs
deleted file mode 100644
index 18b58eba855c..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchFilterTranslator.cs
+++ /dev/null
@@ -1,375 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Collections;
-using System.Linq;
-using System.Linq.Expressions;
-using System.Text;
-using Microsoft.Extensions.VectorData.ProviderServices;
-using Microsoft.Extensions.VectorData.ProviderServices.Filter;
-
-namespace Microsoft.SemanticKernel.Connectors.AzureAISearch;
-
-#pragma warning disable MEVD9001 // Experimental: filter translation base types
-
-internal class AzureAISearchFilterTranslator : FilterTranslatorBase
-{
- private readonly StringBuilder _filter = new();
-
- private static readonly char[] s_searchInDefaultDelimiter = [' ', ','];
-
- internal string Translate(LambdaExpression lambdaExpression, CollectionModel model)
- {
- var preprocessedExpression = this.PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions());
-
- this.Translate(preprocessedExpression);
-
- return this._filter.ToString();
- }
-
- private void Translate(Expression? node)
- {
- switch (node)
- {
- case BinaryExpression binary:
- this.TranslateBinary(binary);
- return;
-
- case ConstantExpression constant:
- this.TranslateConstant(constant);
- return;
-
- case MemberExpression member:
- this.TranslateMember(member);
- return;
-
- case MethodCallExpression methodCall:
- this.TranslateMethodCall(methodCall);
- return;
-
- case UnaryExpression unary:
- this.TranslateUnary(unary);
- return;
-
- default:
- throw new NotSupportedException("Unsupported NodeType in filter: " + node?.NodeType);
- }
- }
-
- private void TranslateBinary(BinaryExpression binary)
- {
- this._filter.Append('(');
- this.Translate(binary.Left);
-
- this._filter.Append(binary.NodeType switch
- {
- ExpressionType.Equal => " eq ",
- ExpressionType.NotEqual => " ne ",
-
- ExpressionType.GreaterThan => " gt ",
- ExpressionType.GreaterThanOrEqual => " ge ",
- ExpressionType.LessThan => " lt ",
- ExpressionType.LessThanOrEqual => " le ",
-
- ExpressionType.AndAlso => " and ",
- ExpressionType.OrElse => " or ",
-
- _ => throw new NotSupportedException("Unsupported binary expression node type: " + binary.NodeType)
- });
-
- this.Translate(binary.Right);
- this._filter.Append(')');
- }
-
- private void TranslateConstant(ConstantExpression constant)
- => this.GenerateLiteral(constant.Value);
-
- private void GenerateLiteral(object? value)
- {
- switch (value)
- {
- case byte b:
- this._filter.Append(b);
- return;
- case short s:
- this._filter.Append(s);
- return;
- case int i:
- this._filter.Append(i);
- return;
- case long l:
- this._filter.Append(l);
- return;
-
- case float f:
- this._filter.Append(f);
- return;
- case double d:
- this._filter.Append(d);
- return;
-
- case string untrustedInput:
- // This is the only place where we allow untrusted input to be passed in, so we need to quote and escape it.
- this._filter.Append('\'').Append(untrustedInput.Replace("'", "''")).Append('\'');
- return;
- case bool b:
- this._filter.Append(b ? "true" : "false");
- return;
- case Guid g:
- this._filter.Append('\'').Append(g.ToString()).Append('\'');
- return;
-
- case DateTime d:
- this._filter.Append(new DateTimeOffset(d, TimeSpan.Zero).ToString("o"));
- return;
- case DateTimeOffset d:
- this._filter.Append(d.ToString("o"));
- return;
-#if NET
- case DateOnly d:
- this._filter.Append(new DateTimeOffset(d.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero).ToString("o"));
- return;
-#endif
-
- case Array:
- throw new NotImplementedException();
-
- case null:
- this._filter.Append("null");
- return;
-
- default:
- throw new NotSupportedException("Unsupported constant type: " + value.GetType().Name);
- }
- }
-
- private void TranslateMember(MemberExpression memberExpression)
- {
- if (this.TryBindProperty(memberExpression, out var property))
- {
- // OData identifiers cannot be escaped; storage names are validated during model building.
- this._filter.Append(property.StorageName);
- return;
- }
-
- throw new NotSupportedException($"Member access for '{memberExpression.Member.Name}' is unsupported - only member access over the filter parameter are supported");
- }
-
- private void TranslateMethodCall(MethodCallExpression methodCall)
- {
- // Dictionary access for dynamic mapping (r => r["SomeString"] == "foo")
- if (this.TryBindProperty(methodCall, out var property))
- {
- // OData identifiers cannot be escaped; storage names are validated during model building.
- this._filter.Append(property.StorageName);
- return;
- }
-
- switch (methodCall)
- {
- // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains()
- case var _ when TryMatchContains(methodCall, out var source, out var item):
- this.TranslateContains(source, item);
- return;
-
- // Enumerable.Any() with a Contains predicate (r => r.Strings.Any(s => array.Contains(s)))
- case { Method.Name: nameof(Enumerable.Any), Arguments: [var anySource, LambdaExpression lambda] } any
- when any.Method.DeclaringType == typeof(Enumerable):
- this.TranslateAny(anySource, lambda);
- return;
-
- default:
- throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}");
- }
- }
-
- private void TranslateContains(Expression source, Expression item)
- {
- switch (source)
- {
- // Contains over array field (r => r.Strings.Contains("foo"))
- case var _ when this.TryBindProperty(source, out _):
- this.Translate(source);
- this._filter.Append("/any(t: t eq ");
- this.Translate(item);
- this._filter.Append(')');
- return;
-
- // Contains over inline enumerable
- case NewArrayExpression newArray:
- var elements = ExtractArrayValues(newArray);
- this._filter.Append("search.in(");
- this.Translate(item);
- this.GenerateSearchInValues(elements);
- return;
-
- case ConstantExpression { Value: IEnumerable enumerable and not string }:
- this._filter.Append("search.in(");
- this.Translate(item);
- this.GenerateSearchInValues(enumerable);
- return;
-
- default:
- throw new NotSupportedException("Unsupported Contains expression");
- }
- }
-
- ///
- /// Translates an Any() call with a Contains predicate, e.g. r.Strings.Any(s => array.Contains(s)).
- /// This checks whether any element in the array field is contained in the given values.
- ///
- private void TranslateAny(Expression source, LambdaExpression lambda)
- {
- // We only support the pattern: r.ArrayField.Any(x => values.Contains(x))
- // Translates to: Field/any(t: search.in(t, 'value1, value2, value3'))
- if (!this.TryBindProperty(source, out var property)
- || lambda.Body is not MethodCallExpression { Method.Name: "Contains" } containsCall
- || !TryMatchContains(containsCall, out var valuesExpression, out var itemExpression))
- {
- throw new NotSupportedException("Unsupported method call: Enumerable.Any");
- }
-
- // Verify that the item is the lambda parameter
- if (itemExpression != lambda.Parameters[0])
- {
- throw new NotSupportedException("Unsupported method call: Enumerable.Any");
- }
-
- // Extract the values and generate the OData filter
- IEnumerable values = valuesExpression switch
- {
- NewArrayExpression newArray => ExtractArrayValues(newArray),
- ConstantExpression { Value: IEnumerable enumerable and not string } => enumerable,
- _ => throw new NotSupportedException("Unsupported method call: Enumerable.Any")
- };
-
- // Generate: Field/any(t: search.in(t, 'value1, value2, value3'))
- // OData identifiers cannot be escaped; storage names are validated during model building.
- this._filter.Append(property.StorageName);
- this._filter.Append("/any(t: search.in(t");
- this.GenerateSearchInValues(values);
- this._filter.Append(')');
- }
-
- ///
- /// Generates the values portion of a search.in() call, including the comma, quotes, and optional delimiter.
- /// Appends: , 'value1, value2, value3') or , 'value1|value2|value3', '|')
- ///
- private void GenerateSearchInValues(IEnumerable values)
- {
- this._filter.Append(", '");
-
- string delimiter = ", ";
- var startingPosition = this._filter.Length;
-
-RestartLoop:
- var isFirst = true;
- foreach (var element in values)
- {
- if (element is not string stringElement)
- {
- throw new NotSupportedException("search.in() over non-string arrays is not supported");
- }
-
- if (isFirst)
- {
- isFirst = false;
- }
- else
- {
- this._filter.Append(delimiter);
- }
-
- // The default delimiter for search.in() is comma or space.
- // If any element contains a comma or space, we switch to using pipe as the delimiter.
- // If any contains a pipe, we throw (for now).
- switch (delimiter)
- {
- case ", ":
- if (stringElement.IndexOfAny(s_searchInDefaultDelimiter) > -1)
- {
- delimiter = "|";
- this._filter.Length = startingPosition;
- goto RestartLoop;
- }
-
- break;
-
- case "|":
- if (stringElement.Contains('|'))
- {
- throw new NotSupportedException("Some elements contain both commas/spaces and pipes, cannot translate to search.in()");
- }
-
- break;
- }
-
- this._filter.Append(stringElement.Replace("'", "''"));
- }
-
- this._filter.Append('\'');
-
- if (delimiter != ", ")
- {
- this._filter
- .Append(", '")
- .Append(delimiter)
- .Append('\'');
- }
-
- this._filter.Append(')');
- }
-
- private static object?[] ExtractArrayValues(NewArrayExpression newArray)
- {
- var result = new object?[newArray.Expressions.Count];
- for (var i = 0; i < newArray.Expressions.Count; i++)
- {
- if (newArray.Expressions[i] is not ConstantExpression { Value: var elementValue })
- {
- throw new NotSupportedException("Invalid element in array");
- }
-
- result[i] = elementValue;
- }
-
- return result;
- }
-
- private void TranslateUnary(UnaryExpression unary)
- {
- switch (unary.NodeType)
- {
- case ExpressionType.Not:
- // Special handling for !(a == b) and !(a != b)
- if (unary.Operand is BinaryExpression { NodeType: ExpressionType.Equal or ExpressionType.NotEqual } binary)
- {
- this.TranslateBinary(
- Expression.MakeBinary(
- binary.NodeType is ExpressionType.Equal ? ExpressionType.NotEqual : ExpressionType.Equal,
- binary.Left,
- binary.Right));
- return;
- }
-
- this._filter.Append("(not ");
- this.Translate(unary.Operand);
- this._filter.Append(')');
- return;
-
- // Handle converting non-nullable to nullable; such nodes are found in e.g. r => r.Int == nullableInt
- case ExpressionType.Convert when Nullable.GetUnderlyingType(unary.Type) == unary.Operand.Type:
- this.Translate(unary.Operand);
- return;
-
- // Handle convert over member access, for dynamic dictionary access (r => (int)r["SomeInt"] == 8)
- case ExpressionType.Convert when this.TryBindProperty(unary.Operand, out var property) && unary.Type == property.Type:
- // OData identifiers cannot be escaped; storage names are validated during model building.
- this._filter.Append(property.StorageName);
- return;
-
- default:
- throw new NotSupportedException("Unsupported unary expression node type: " + unary.NodeType);
- }
- }
-}
diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchMapper.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchMapper.cs
deleted file mode 100644
index 88b554845552..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchMapper.cs
+++ /dev/null
@@ -1,134 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-using Microsoft.Extensions.AI;
-using Microsoft.Extensions.VectorData.ProviderServices;
-using MEAI = Microsoft.Extensions.AI;
-
-namespace Microsoft.SemanticKernel.Connectors.AzureAISearch;
-
-internal sealed class AzureAISearchMapper(CollectionModel model, JsonSerializerOptions? jsonSerializerOptions) : IAzureAISearchMapper
- where TRecord : class
-{
- public JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings)
- {
- Verify.NotNull(dataModel);
-
- var jsonObject = JsonSerializer.SerializeToNode(dataModel, jsonSerializerOptions)!.AsObject();
-
- // Azure AI Search only supports Edm.DateTimeOffset for date/time types.
- // DateTime properties may be serialized by STJ without timezone info (when Kind=Unspecified),
- // which Azure AI Search rejects. Convert them to DateTimeOffset (UTC).
- // DateOnly properties are serialized as "yyyy-MM-dd", which also isn't accepted.
- // Convert them to full DateTimeOffset representation (midnight UTC).
- foreach (var dataProperty in model.DataProperties)
- {
- var propertyType = Nullable.GetUnderlyingType(dataProperty.Type) ?? dataProperty.Type;
-
- if (propertyType == typeof(DateTime) && jsonObject.TryGetPropertyValue(dataProperty.StorageName, out var dtNode) && dtNode is not null)
- {
- var dateTime = dtNode.Deserialize(jsonSerializerOptions);
- jsonObject[dataProperty.StorageName] = JsonValue.Create(new DateTimeOffset(DateTime.SpecifyKind(dateTime, DateTimeKind.Utc), TimeSpan.Zero));
- }
-#if NET
- else if (propertyType == typeof(DateOnly) && jsonObject.TryGetPropertyValue(dataProperty.StorageName, out var dateNode) && dateNode is not null)
- {
- var dateOnly = dateNode.Deserialize(jsonSerializerOptions);
- jsonObject[dataProperty.StorageName] = JsonValue.Create(new DateTimeOffset(dateOnly.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero));
- }
-#endif
- }
-
- // Go over the vector properties; inject any generated embeddings to overwrite the JSON serialized above.
- // Also, for Embedding properties we also need to overwrite with a simple array (since Embedding gets serialized as a complex object).
- for (var i = 0; i < model.VectorProperties.Count; i++)
- {
- var property = model.VectorProperties[i];
-
- Embedding? embedding = generatedEmbeddings?[i]?[recordIndex] is Embedding ge ? (Embedding)ge : null;
-
- if (embedding is null)
- {
- switch (Nullable.GetUnderlyingType(property.Type) ?? property.Type)
- {
- case var t when t == typeof(ReadOnlyMemory):
- case var t2 when t2 == typeof(float[]):
- // The .NET vector property is a ReadOnlyMemory or float[] (not an Embedding), which means that JsonSerializer
- // already serialized it correctly above.
- // In addition, there's no generated embedding (which would be an Embedding which we'd need to handle manually).
- // So there's nothing for us to do.
- continue;
-
- case var t when t == typeof(Embedding):
- embedding = (Embedding)property.GetValueAsObject(dataModel)!;
- break;
-
- default:
- throw new UnreachableException();
- }
- }
-
- var jsonArray = new JsonArray();
-
- foreach (var item in embedding.Vector.Span)
- {
- jsonArray.Add(JsonValue.Create(item));
- }
-
- jsonObject[property.StorageName] = jsonArray;
- }
-
- return jsonObject;
- }
-
- public TRecord MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors)
- {
- // Azure AI Search stores DateTime properties as Edm.DateTimeOffset.
- // Convert them back to DateTime so STJ can deserialize them into the correct type.
- // DateOnly properties also need to be converted back from DateTimeOffset.
- foreach (var dataProperty in model.DataProperties)
- {
- var propertyType = Nullable.GetUnderlyingType(dataProperty.Type) ?? dataProperty.Type;
-
- if (propertyType == typeof(DateTime) && storageModel.TryGetPropertyValue(dataProperty.StorageName, out var dtNode) && dtNode is not null)
- {
- var dateTimeOffset = dtNode.Deserialize(jsonSerializerOptions);
- storageModel[dataProperty.StorageName] = JsonValue.Create(dateTimeOffset.UtcDateTime);
- }
-#if NET
- else if (propertyType == typeof(DateOnly) && storageModel.TryGetPropertyValue(dataProperty.StorageName, out var dateNode) && dateNode is not null)
- {
- var dateTimeOffset = dateNode.Deserialize(jsonSerializerOptions);
- storageModel[dataProperty.StorageName] = JsonValue.Create(DateOnly.FromDateTime(dateTimeOffset.DateTime));
- }
-#endif
- }
-
- if (includeVectors)
- {
- foreach (var vectorProperty in model.VectorProperties)
- {
- // If the vector property .NET type is Embedding, we need to create the JSON structure for it
- // (JSON array embedded inside an object representing the embedding), so that the deserialization below
- // works correctly.
- if (vectorProperty.Type == typeof(Embedding))
- {
- var arrayNode = storageModel[vectorProperty.StorageName];
- if (arrayNode is not null)
- {
- storageModel[vectorProperty.StorageName] = new JsonObject
- {
- [nameof(Embedding.Vector)] = arrayNode.DeepClone()
- };
- }
- }
- }
- }
-
- return storageModel.Deserialize(jsonSerializerOptions)!;
- }
-}
diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchModelBuilder.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchModelBuilder.cs
deleted file mode 100644
index 4b0bed52eaf6..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchModelBuilder.cs
+++ /dev/null
@@ -1,125 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using Microsoft.Extensions.AI;
-using Microsoft.Extensions.VectorData;
-using Microsoft.Extensions.VectorData.ProviderServices;
-
-namespace Microsoft.SemanticKernel.Connectors.AzureAISearch;
-
-internal class AzureAISearchModelBuilder() : CollectionJsonModelBuilder(s_modelBuildingOptions)
-{
- internal const string SupportedVectorTypes = "ReadOnlyMemory, Embedding, float[]";
-
- internal static readonly CollectionModelBuildingOptions s_modelBuildingOptions = new()
- {
- RequiresAtLeastOneVector = false,
- SupportsMultipleVectors = true,
- UsesExternalSerializer = true
- };
-
- protected override void ValidateKeyProperty(KeyPropertyModel keyProperty)
- {
- base.ValidateKeyProperty(keyProperty);
-
- var type = keyProperty.Type;
-
- if (type != typeof(string) && type != typeof(Guid))
- {
- throw new NotSupportedException(
- $"Property '{keyProperty.ModelName}' has unsupported type '{type.Name}'. Key properties must be one of the supported types: string, Guid.");
- }
- }
-
- protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
- => IsDataPropertyTypeValidCore(type, out supportedTypes);
-
- protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
- => IsVectorPropertyTypeValidCore(type, out supportedTypes);
-
- internal static bool IsDataPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes)
- {
- supportedTypes = "string, int, long, double, float, bool, DateTime, DateTimeOffset,"
-#if NET
- + " DateOnly,"
-#endif
- + " or arrays/lists of these types";
-
- if (Nullable.GetUnderlyingType(type) is Type underlyingType)
- {
- type = underlyingType;
- }
-
- return IsValid(type)
- || (type.IsArray && IsValid(type.GetElementType()!))
- || (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>) && IsValid(type.GenericTypeArguments[0]));
-
- static bool IsValid(Type type)
- => type == typeof(string) ||
- type == typeof(int) ||
- type == typeof(long) ||
- type == typeof(double) ||
- type == typeof(float) ||
- type == typeof(bool) ||
- type == typeof(DateTime) ||
-#if NET
- type == typeof(DateOnly) ||
-#endif
- type == typeof(DateTimeOffset);
- }
-
- internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes)
- {
- // Azure AI Search is adding support for more types than just float32, but these are not available for use via the
- // SDK yet. We will update this list as the SDK is updated.
- //
- supportedTypes = SupportedVectorTypes;
-
- return type == typeof(ReadOnlyMemory)
- || type == typeof(ReadOnlyMemory?)
- || type == typeof(Embedding)
- || type == typeof(float[]);
- }
-
- ///
- protected override void ValidateProperty(PropertyModel propertyModel, VectorStoreCollectionDefinition? definition)
- {
- base.ValidateProperty(propertyModel, definition);
-
- // OData identifiers cannot be escaped; storage names are validated during model building.
- // See https://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part2-url-conventions.html#sec_ODataIdentifier
- if (!IsValidIdentifier(propertyModel.StorageName))
- {
- throw new InvalidOperationException(
- $"Property '{propertyModel.ModelName}' has storage name '{propertyModel.StorageName}' which is not a valid OData identifier. " +
- "OData identifiers must start with a letter or underscore, and contain only letters, digits, and underscores.");
- }
- }
-
- private static bool IsValidIdentifier(string name)
- {
- if (string.IsNullOrEmpty(name))
- {
- return false;
- }
-
- var first = name[0];
- if (!char.IsLetter(first) && first != '_')
- {
- return false;
- }
-
- for (var i = 1; i < name.Length; i++)
- {
- var c = name[i];
- if (!char.IsLetterOrDigit(c) && c != '_')
- {
- return false;
- }
- }
-
- return true;
- }
-}
diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchServiceCollectionExtensions.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchServiceCollectionExtensions.cs
deleted file mode 100644
index 1df47640c42a..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchServiceCollectionExtensions.cs
+++ /dev/null
@@ -1,379 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Diagnostics.CodeAnalysis;
-using System.Text.Json;
-using Azure;
-using Azure.Core;
-using Azure.Core.Serialization;
-using Azure.Search.Documents;
-using Azure.Search.Documents.Indexes;
-using Microsoft.Extensions.AI;
-using Microsoft.Extensions.VectorData;
-using Microsoft.SemanticKernel;
-using Microsoft.SemanticKernel.Connectors.AzureAISearch;
-using Microsoft.SemanticKernel.Http;
-
-namespace Microsoft.Extensions.DependencyInjection;
-
-///
-/// Extension methods to register and instances on an .
-///
-public static class AzureAISearchServiceCollectionExtensions
-{
- private const string DynamicCodeMessage = "The Azure AI Search provider is currently incompatible with trimming.";
- private const string UnreferencedCodeMessage = "The Azure AI Search provider is currently incompatible with NativeAOT.";
-
- ///
- /// Registers a as
- /// with returned by
- /// or retrieved from the dependency injection container if was not provided.
- ///
- ///
- [RequiresUnreferencedCode(DynamicCodeMessage)]
- [RequiresDynamicCode(UnreferencedCodeMessage)]
- public static IServiceCollection AddAzureAISearchVectorStore(
- this IServiceCollection services,
- Func? clientProvider = default,
- Func? optionsProvider = default,
- ServiceLifetime lifetime = ServiceLifetime.Singleton)
- => AddKeyedAzureAISearchVectorStore(services, serviceKey: null, clientProvider, optionsProvider, lifetime);
-
- ///
- /// Registers a keyed as
- /// with returned by
- /// or retrieved from the dependency injection container if was not provided.
- ///
- /// The to register the on.
- /// The key with which to associate the vector store.
- /// The provider.
- /// Options provider to further configure the .
- /// The service lifetime for the store. Defaults to .
- /// Service collection.
- [RequiresUnreferencedCode(DynamicCodeMessage)]
- [RequiresDynamicCode(UnreferencedCodeMessage)]
- public static IServiceCollection AddKeyedAzureAISearchVectorStore(
- this IServiceCollection services,
- object? serviceKey,
- Func? clientProvider = default,
- Func? optionsProvider = default,
- ServiceLifetime lifetime = ServiceLifetime.Singleton)
- {
- Verify.NotNull(services);
-
- services.Add(new ServiceDescriptor(typeof(AzureAISearchVectorStore), serviceKey, (sp, _) =>
- {
- var client = clientProvider is not null ? clientProvider(sp) : sp.GetRequiredService();
- var options = GetStoreOptions(sp, optionsProvider);
-
- return new AzureAISearchVectorStore(client, options);
- }, lifetime));
-
- services.Add(new ServiceDescriptor(typeof(VectorStore), serviceKey,
- static (sp, key) => sp.GetRequiredKeyedService(key), lifetime));
-
- return services;
- }
-
- ///
- /// Registers a as
- /// using the provided and .
- ///
- ///
- [RequiresUnreferencedCode(DynamicCodeMessage)]
- [RequiresDynamicCode(UnreferencedCodeMessage)]
- public static IServiceCollection AddAzureAISearchVectorStore(
- this IServiceCollection services,
- Uri endpoint,
- TokenCredential tokenCredential,
- AzureAISearchVectorStoreOptions? options = default,
- ServiceLifetime lifetime = ServiceLifetime.Singleton)
- => AddKeyedAzureAISearchVectorStore(services, serviceKey: null, endpoint, tokenCredential, options, lifetime);
-
- ///
- /// Registers a keyed as
- /// using the provided and .
- ///
- /// The to register the on.
- /// The key with which to associate the vector store.
- /// The service endpoint for Azure AI Search.
- /// The credential to authenticate to Azure AI Search with.
- /// Optional options to further configure the .
- /// The service lifetime for the store. Defaults to .
- /// Service collection.
- [RequiresUnreferencedCode(DynamicCodeMessage)]
- [RequiresDynamicCode(UnreferencedCodeMessage)]
- public static IServiceCollection AddKeyedAzureAISearchVectorStore(
- this IServiceCollection services,
- object? serviceKey,
- Uri endpoint,
- TokenCredential tokenCredential,
- AzureAISearchVectorStoreOptions? options = default,
- ServiceLifetime lifetime = ServiceLifetime.Singleton)
- {
- Verify.NotNull(endpoint);
- Verify.NotNull(tokenCredential);
-
- return AddKeyedAzureAISearchVectorStore(services, serviceKey, sp =>
- {
- var searchClientOptions = BuildSearchClientOptions(options?.JsonSerializerOptions);
- return new SearchIndexClient(endpoint, tokenCredential, searchClientOptions);
- }, _ => options!, lifetime);
- }
-
- ///
- /// Registers a as
- /// using the provided and .
- ///
- ///
- [RequiresUnreferencedCode(DynamicCodeMessage)]
- [RequiresDynamicCode(UnreferencedCodeMessage)]
- public static IServiceCollection AddAzureAISearchVectorStore(
- this IServiceCollection services,
- Uri endpoint,
- AzureKeyCredential keyCredential,
- AzureAISearchVectorStoreOptions? options = default,
- ServiceLifetime lifetime = ServiceLifetime.Singleton)
- => AddKeyedAzureAISearchVectorStore(services, serviceKey: null, endpoint, keyCredential, options, lifetime);
-
- ///
- /// Registers a keyed as
- /// using the provided and .
- ///
- /// The to register the on.
- /// The key with which to associate the vector store.
- /// The service endpoint for Azure AI Search.
- /// The credential to authenticate to Azure AI Search with.
- /// Optional options to further configure the .
- /// The service lifetime for the store. Defaults to .
- /// Service collection.
- [RequiresUnreferencedCode(DynamicCodeMessage)]
- [RequiresDynamicCode(UnreferencedCodeMessage)]
- public static IServiceCollection AddKeyedAzureAISearchVectorStore(
- this IServiceCollection services,
- object? serviceKey,
- Uri endpoint,
- AzureKeyCredential keyCredential,
- AzureAISearchVectorStoreOptions? options = default,
- ServiceLifetime lifetime = ServiceLifetime.Singleton)
- {
- Verify.NotNull(endpoint);
- Verify.NotNull(keyCredential);
-
- return AddKeyedAzureAISearchVectorStore(services, serviceKey, sp =>
- {
- var searchClientOptions = BuildSearchClientOptions(options?.JsonSerializerOptions);
- return new SearchIndexClient(endpoint, keyCredential, searchClientOptions);
- }, _ => options!, lifetime);
- }
-
- ///
- /// Registers a as
- /// with returned by
- /// or retrieved from the dependency injection container if was not provided.
- ///
- ///
- [RequiresUnreferencedCode(DynamicCodeMessage)]
- [RequiresDynamicCode(UnreferencedCodeMessage)]
- public static IServiceCollection AddAzureAISearchCollection(
- this IServiceCollection services,
- string name,
- Func? clientProvider = default,
- Func? optionsProvider = default,
- ServiceLifetime lifetime = ServiceLifetime.Singleton)
- where TRecord : class
- => AddKeyedAzureAISearchCollection(services, serviceKey: null, name, clientProvider, optionsProvider, lifetime);
-
- ///
- /// Registers a keyed as
- /// with returned by
- /// or retrieved from the dependency injection container if was not provided.
- ///
- /// The to register the on.
- /// The key with which to associate the collection.
- /// The name of the collection.
- /// The provider.
- /// Options provider to further configure the .
- /// The service lifetime for the store. Defaults to .
- /// Service collection.
- [RequiresUnreferencedCode(DynamicCodeMessage)]
- [RequiresDynamicCode(UnreferencedCodeMessage)]
- public static IServiceCollection AddKeyedAzureAISearchCollection(
- this IServiceCollection services,
- object? serviceKey,
- string name,
- Func? clientProvider = default,
- Func? optionsProvider = default,
- ServiceLifetime lifetime = ServiceLifetime.Singleton)
- where TRecord : class
- {
- Verify.NotNull(services);
- Verify.NotNullOrWhiteSpace(name);
-
- services.Add(new ServiceDescriptor(typeof(AzureAISearchCollection), serviceKey, (sp, _) =>
- {
- var client = clientProvider is not null ? clientProvider(sp) : sp.GetRequiredService();
- var options = GetCollectionOptions(sp, optionsProvider);
-
- return new AzureAISearchCollection(client, name, options);
- }, lifetime));
-
- services.Add(new ServiceDescriptor(typeof(VectorStoreCollection), serviceKey,
- static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime));
-
- services.Add(new ServiceDescriptor(typeof(IVectorSearchable), serviceKey,
- static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime));
-
- services.Add(new ServiceDescriptor(typeof(IKeywordHybridSearchable), serviceKey,
- static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime));
-
- return services;
- }
-
- ///
- /// Registers a as
- /// using the provided and .
- ///
- ///
- [RequiresUnreferencedCode(DynamicCodeMessage)]
- [RequiresDynamicCode(UnreferencedCodeMessage)]
- public static IServiceCollection AddAzureAISearchCollection(
- this IServiceCollection services,
- string name,
- Uri endpoint,
- TokenCredential tokenCredential,
- AzureAISearchCollectionOptions? options = default,
- ServiceLifetime lifetime = ServiceLifetime.Singleton)
- where TRecord : class
- => AddKeyedAzureAISearchCollection(services, serviceKey: null, name, endpoint, tokenCredential, options, lifetime);
-
- ///
- /// Registers a keyed as
- /// using the provided and .
- ///
- /// The to register the on.
- /// The key with which to associate the collection.
- /// The name of the collection.
- /// The service endpoint for Azure AI Search.
- /// The credential to authenticate to Azure AI Search with.
- /// Optional options to further configure the .
- /// The service lifetime for the store. Defaults to .
- /// Service collection.
- [RequiresUnreferencedCode(DynamicCodeMessage)]
- [RequiresDynamicCode(UnreferencedCodeMessage)]
- public static IServiceCollection AddKeyedAzureAISearchCollection(
- this IServiceCollection services,
- object? serviceKey,
- string name,
- Uri endpoint,
- TokenCredential tokenCredential,
- AzureAISearchCollectionOptions? options = default,
- ServiceLifetime lifetime = ServiceLifetime.Singleton)
- where TRecord : class
- {
- Verify.NotNull(endpoint);
- Verify.NotNull(tokenCredential);
-
- return AddKeyedAzureAISearchCollection(services, serviceKey, name, sp =>
- {
- var searchClientOptions = BuildSearchClientOptions(options?.JsonSerializerOptions);
- return new SearchIndexClient(endpoint, tokenCredential, searchClientOptions);
- }, _ => options!, lifetime);
- }
-
- ///
- /// Registers a as
- /// using the provided and .
- ///
- ///
- [RequiresUnreferencedCode(DynamicCodeMessage)]
- [RequiresDynamicCode(UnreferencedCodeMessage)]
- public static IServiceCollection AddAzureAISearchCollection(
- this IServiceCollection services,
- string name,
- Uri endpoint,
- AzureKeyCredential keyCredential,
- AzureAISearchCollectionOptions? options = default,
- ServiceLifetime lifetime = ServiceLifetime.Singleton)
- where TRecord : class
- => AddKeyedAzureAISearchCollection(services, serviceKey: null, name, endpoint, keyCredential, options, lifetime);
-
- ///
- /// Registers a keyed as
- /// using the provided and .
- ///
- /// The to register the on.
- /// The key with which to associate the collection.
- /// The name of the collection.
- /// The service endpoint for Azure AI Search.
- /// The credential to authenticate to Azure AI Search with.
- /// Optional options to further configure the .
- /// The service lifetime for the store. Defaults to .
- /// Service collection.
- [RequiresUnreferencedCode(DynamicCodeMessage)]
- [RequiresDynamicCode(UnreferencedCodeMessage)]
- public static IServiceCollection AddKeyedAzureAISearchCollection(
- this IServiceCollection services,
- object? serviceKey,
- string name,
- Uri endpoint,
- AzureKeyCredential keyCredential,
- AzureAISearchCollectionOptions? options = default,
- ServiceLifetime lifetime = ServiceLifetime.Singleton)
- where TRecord : class
- {
- Verify.NotNull(endpoint);
- Verify.NotNull(keyCredential);
-
- return AddKeyedAzureAISearchCollection(services, serviceKey, name, sp =>
- {
- var searchClientOptions = BuildSearchClientOptions(options?.JsonSerializerOptions);
- return new SearchIndexClient(endpoint, keyCredential, searchClientOptions);
- }, _ => options!, lifetime);
- }
-
- ///
- /// Build a instance, using the provided if it's not null and add the SK user agent string.
- ///
- /// Optional to add to the options if provided.
- /// The .
- private static SearchClientOptions BuildSearchClientOptions(JsonSerializerOptions? jsonSerializerOptions)
- {
- var searchClientOptions = new SearchClientOptions();
- searchClientOptions.Diagnostics.ApplicationId = HttpHeaderConstant.Values.UserAgent;
- if (jsonSerializerOptions != null)
- {
- searchClientOptions.Serializer = new JsonObjectSerializer(jsonSerializerOptions);
- }
-
- return searchClientOptions;
- }
-
- private static AzureAISearchVectorStoreOptions? GetStoreOptions(IServiceProvider sp, Func? optionsProvider)
- {
- var options = optionsProvider?.Invoke(sp);
- if (options?.EmbeddingGenerator is not null)
- {
- return options; // The user has provided everything, there is nothing to change.
- }
-
- var embeddingGenerator = sp.GetService();
- return embeddingGenerator is null
- ? options // There is nothing to change.
- : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options.
- }
-
- private static AzureAISearchCollectionOptions? GetCollectionOptions(IServiceProvider sp, Func? optionsProvider)
- {
- var options = optionsProvider?.Invoke(sp);
- if (options?.EmbeddingGenerator is not null)
- {
- return options; // The user has provided everything, there is nothing to change.
- }
-
- var embeddingGenerator = sp.GetService();
- return embeddingGenerator is null
- ? options // There is nothing to change.
- : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options.
- }
-}
diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchVectorStore.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchVectorStore.cs
deleted file mode 100644
index 30a474d08319..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchVectorStore.cs
+++ /dev/null
@@ -1,144 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Runtime.CompilerServices;
-using System.Text.Json;
-using System.Threading;
-using System.Threading.Tasks;
-using Azure;
-using Azure.Search.Documents.Indexes;
-using Microsoft.Extensions.AI;
-using Microsoft.Extensions.VectorData;
-using Microsoft.Extensions.VectorData.ProviderServices;
-using static Microsoft.Extensions.VectorData.VectorStoreErrorHandler;
-
-namespace Microsoft.SemanticKernel.Connectors.AzureAISearch;
-
-///
-/// Class for accessing the list of collections in a Azure AI Search vector store.
-///
-///
-/// This class can be used with collections of any schema type, but requires you to provide schema information when getting a collection.
-///
-public sealed class AzureAISearchVectorStore : VectorStore
-{
- /// Metadata about vector store.
- private readonly VectorStoreMetadata _metadata;
-
- /// Azure AI Search client that can be used to manage the list of indices in an Azure AI Search Service.
- private readonly SearchIndexClient _searchIndexClient;
-
- private readonly IEmbeddingGenerator? _embeddingGenerator;
- private readonly JsonSerializerOptions? _jsonSerializerOptions;
-
- /// A general purpose definition that can be used to construct a collection when needing to proxy schema agnostic operations.
- private static readonly VectorStoreCollectionDefinition s_generalPurposeDefinition = new() { Properties = [new VectorStoreKeyProperty("Key", typeof(string))] };
-
- ///
- /// Initializes a new instance of the class.
- ///
- /// Azure AI Search client that can be used to manage the list of indices in an Azure AI Search Service.
- /// Optional configuration options for this class.
- [RequiresUnreferencedCode("The Azure AI Search provider is currently incompatible with trimming.")]
- [RequiresDynamicCode("The Azure AI Search provider is currently incompatible with NativeAOT.")]
- public AzureAISearchVectorStore(SearchIndexClient searchIndexClient, AzureAISearchVectorStoreOptions? options = default)
- {
- Verify.NotNull(searchIndexClient);
-
- this._searchIndexClient = searchIndexClient;
- this._embeddingGenerator = options?.EmbeddingGenerator;
- this._jsonSerializerOptions = options?.JsonSerializerOptions;
-
- this._metadata = new()
- {
- VectorStoreSystemName = AzureAISearchConstants.VectorStoreSystemName,
- VectorStoreName = searchIndexClient.ServiceName
- };
- }
-
-#pragma warning disable IDE0090 // Use 'new(...)'
- ///
- [RequiresDynamicCode("This overload of GetCollection() is incompatible with NativeAOT. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")]
- [RequiresUnreferencedCode("This overload of GetCollecttion() is incompatible with trimming. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")]
-#if NET
- public override AzureAISearchCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null)
-#else
- public override VectorStoreCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null)
-#endif
- => typeof(TRecord) == typeof(Dictionary)
- ? throw new ArgumentException(VectorDataStrings.GetCollectionWithDictionaryNotSupported)
- : new AzureAISearchCollection(
- this._searchIndexClient,
- name,
- new AzureAISearchCollectionOptions()
- {
- JsonSerializerOptions = this._jsonSerializerOptions,
- Definition = definition,
- EmbeddingGenerator = this._embeddingGenerator
- });
-
- ///
- [RequiresUnreferencedCode("The Azure AI Search provider is currently incompatible with trimming.")]
- [RequiresDynamicCode("The Azure AI Search provider is currently incompatible with NativeAOT.")]
-#if NET
- public override AzureAISearchDynamicCollection GetDynamicCollection(string name, VectorStoreCollectionDefinition definition)
-#else
- public override VectorStoreCollection> GetDynamicCollection(string name, VectorStoreCollectionDefinition definition)
-#endif
- => new AzureAISearchDynamicCollection(
- this._searchIndexClient,
- name,
- new()
- {
- JsonSerializerOptions = this._jsonSerializerOptions,
- Definition = definition,
- EmbeddingGenerator = this._embeddingGenerator
- }
- );
-#pragma warning restore IDE0090
-
- ///
- public override async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
- {
- const string OperationName = "GetIndexNames";
-
- var indexNamesEnumerable = this._searchIndexClient.GetIndexNamesAsync(cancellationToken).ConfigureAwait(false);
- var errorHandlingEnumerable = new ConfiguredCancelableErrorHandlingAsyncEnumerable(indexNamesEnumerable, this._metadata, OperationName);
-
-#pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task: False Positive
- await foreach (var item in errorHandlingEnumerable.ConfigureAwait(false))
-#pragma warning restore CA2007 // Consider calling ConfigureAwait on the awaited task
- {
- yield return item;
- }
- }
-
- ///
- public override Task CollectionExistsAsync(string name, CancellationToken cancellationToken = default)
- {
- var collection = this.GetDynamicCollection(name, s_generalPurposeDefinition);
- return collection.CollectionExistsAsync(cancellationToken);
- }
-
- ///
- public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default)
- {
- var collection = this.GetDynamicCollection(name, s_generalPurposeDefinition);
- return collection.EnsureCollectionDeletedAsync(cancellationToken);
- }
-
- ///
- public override object? GetService(Type serviceType, object? serviceKey = null)
- {
- Verify.NotNull(serviceType);
-
- return
- serviceKey is not null ? null :
- serviceType == typeof(VectorStoreMetadata) ? this._metadata :
- serviceType == typeof(SearchIndexClient) ? this._searchIndexClient :
- serviceType.IsInstanceOfType(this) ? this :
- null;
- }
-}
diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchVectorStoreOptions.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchVectorStoreOptions.cs
deleted file mode 100644
index fb1b2855b0e8..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchVectorStoreOptions.cs
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System.Text.Json;
-using Azure.Search.Documents.Indexes;
-using Microsoft.Extensions.AI;
-
-namespace Microsoft.SemanticKernel.Connectors.AzureAISearch;
-
-///
-/// Options when creating a .
-///
-public sealed class AzureAISearchVectorStoreOptions
-{
- ///
- /// Initializes a new instance of the class.
- ///
- public AzureAISearchVectorStoreOptions()
- {
- }
-
- internal AzureAISearchVectorStoreOptions(AzureAISearchVectorStoreOptions? source)
- {
- this.JsonSerializerOptions = source?.JsonSerializerOptions;
- this.EmbeddingGenerator = source?.EmbeddingGenerator;
- }
-
- ///
- /// Gets or sets the JSON serializer options to use when converting between the data model and the Azure AI Search record.
- /// Note that when using the default mapper and you are constructing your own , you will need
- /// to provide the same set of both here and when constructing the .
- ///
- public JsonSerializerOptions? JsonSerializerOptions { get; set; }
-
- ///
- /// Gets or sets the default embedding generator to use when generating vectors embeddings with this vector store.
- ///
- public IEmbeddingGenerator? EmbeddingGenerator { get; set; }
-}
diff --git a/dotnet/src/VectorData/AzureAISearch/IAzureAISearchMapper.cs b/dotnet/src/VectorData/AzureAISearch/IAzureAISearchMapper.cs
deleted file mode 100644
index 3166f403d856..000000000000
--- a/dotnet/src/VectorData/AzureAISearch/IAzureAISearchMapper.cs
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System.Collections.Generic;
-using System.Text.Json.Nodes;
-using MEAI = Microsoft.Extensions.AI;
-
-namespace Microsoft.SemanticKernel.Connectors.AzureAISearch;
-
-internal interface IAzureAISearchMapper
-{
- ///
- /// Maps from the consumer record data model to the storage model.
- ///
- JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings);
-
- ///
- /// Maps from the storage model to the consumer record data model.
- ///
- TRecord MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors);
-}
diff --git a/dotnet/src/VectorData/AzureAISearch/README.md b/dotnet/src/VectorData/AzureAISearch/README.md
new file mode 100644
index 000000000000..5a3ecfced340
--- /dev/null
+++ b/dotnet/src/VectorData/AzureAISearch/README.md
@@ -0,0 +1 @@
+The code for Microsoft.SemanticKernel.Connectors.AzureAISearch can now be found in the [CommunityToolkit/AI repository](https://github.com/CommunityToolkit/AI/tree/main/MEVD/src/AzureAISearch)
diff --git a/dotnet/src/VectorData/Common/SqlFilterTranslator.cs b/dotnet/src/VectorData/Common/SqlFilterTranslator.cs
deleted file mode 100644
index 0652d7675049..000000000000
--- a/dotnet/src/VectorData/Common/SqlFilterTranslator.cs
+++ /dev/null
@@ -1,372 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Diagnostics;
-using System.Linq;
-using System.Linq.Expressions;
-using System.Text;
-using Microsoft.Extensions.VectorData.ProviderServices;
-using Microsoft.Extensions.VectorData.ProviderServices.Filter;
-
-namespace Microsoft.SemanticKernel.Connectors;
-
-#pragma warning disable MEVD9001 // Microsoft.Extensions.VectorData experimental connector-facing APIs
-
-internal abstract class SqlFilterTranslator : FilterTranslatorBase
-{
- protected readonly StringBuilder _sql;
- private readonly Expression _preprocessedExpression;
-
- internal SqlFilterTranslator(
- CollectionModel model,
- LambdaExpression lambdaExpression,
- StringBuilder? sql = null)
- {
- Debug.Assert(lambdaExpression.Parameters.Count == 1);
- this._sql = sql ?? new();
-
- this._preprocessedExpression = this.PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions { SupportsParameterization = true });
- }
-
- internal StringBuilder Clause => this._sql;
-
- internal void Translate(bool appendWhere)
- {
- if (appendWhere)
- {
- this._sql.Append("WHERE ");
- }
-
- this.Translate(this._preprocessedExpression, isSearchCondition: true);
- }
-
- protected void Translate(Expression? node, bool isSearchCondition = false)
- {
- switch (node)
- {
- case BinaryExpression binary:
- this.TranslateBinary(binary);
- return;
-
- case ConstantExpression constant:
- this.TranslateConstant(constant.Value, isSearchCondition);
- return;
-
- case QueryParameterExpression { Name: var name, Value: var value }:
- this.TranslateQueryParameter(value);
- return;
-
- case MemberExpression member:
- this.TranslateMember(member, isSearchCondition);
- return;
-
- case MethodCallExpression methodCall:
- this.TranslateMethodCall(methodCall, isSearchCondition);
- return;
-
- case UnaryExpression unary:
- this.TranslateUnary(unary, isSearchCondition);
- return;
-
- default:
- throw new NotSupportedException("Unsupported NodeType in filter: " + node?.NodeType);
- }
- }
-
- protected void TranslateBinary(BinaryExpression binary)
- {
- // Special handling for null comparisons
- switch (binary.NodeType)
- {
- case ExpressionType.Equal when IsNull(binary.Right):
- this._sql.Append('(');
- this.Translate(binary.Left);
- this._sql.Append(" IS NULL)");
- return;
- case ExpressionType.NotEqual when IsNull(binary.Right):
- this._sql.Append('(');
- this.Translate(binary.Left);
- this._sql.Append(" IS NOT NULL)");
- return;
-
- case ExpressionType.Equal when IsNull(binary.Left):
- this._sql.Append('(');
- this.Translate(binary.Right);
- this._sql.Append(" IS NULL)");
- return;
- case ExpressionType.NotEqual when IsNull(binary.Left):
- this._sql.Append('(');
- this.Translate(binary.Right);
- this._sql.Append(" IS NOT NULL)");
- return;
- }
-
- this._sql.Append('(');
- this.Translate(binary.Left, isSearchCondition: binary.NodeType is ExpressionType.AndAlso or ExpressionType.OrElse);
-
- this._sql.Append(binary.NodeType switch
- {
- ExpressionType.Equal => " = ",
- ExpressionType.NotEqual => " <> ",
-
- ExpressionType.GreaterThan => " > ",
- ExpressionType.GreaterThanOrEqual => " >= ",
- ExpressionType.LessThan => " < ",
- ExpressionType.LessThanOrEqual => " <= ",
-
- ExpressionType.AndAlso => " AND ",
- ExpressionType.OrElse => " OR ",
-
- _ => throw new NotSupportedException("Unsupported binary expression node type: " + binary.NodeType)
- });
-
- this.Translate(binary.Right, isSearchCondition: binary.NodeType is ExpressionType.AndAlso or ExpressionType.OrElse);
-
- this._sql.Append(')');
-
- static bool IsNull(Expression expression)
- => expression is ConstantExpression { Value: null } or QueryParameterExpression { Value: null };
- }
-
- protected virtual void TranslateConstant(object? value, bool isSearchCondition)
- {
- switch (value)
- {
- case byte b:
- this._sql.Append(b);
- return;
- case short s:
- this._sql.Append(s);
- return;
- case int i:
- this._sql.Append(i);
- return;
- case long l:
- this._sql.Append(l);
- return;
-
- case float f:
- this._sql.Append(f);
- return;
- case double d:
- this._sql.Append(d);
- return;
- case decimal d:
- this._sql.Append(d);
- return;
-
- case string untrustedInput:
- // This is the only place where we allow untrusted input to be passed in, so we need to quote and escape it.
- // Luckily for us, values are escaped in the same way for every provider that we support so far.
- this._sql.Append('\'').Append(untrustedInput.Replace("'", "''")).Append('\'');
- return;
- case bool b:
- this._sql.Append(b ? "TRUE" : "FALSE");
- return;
- case Guid g:
- this._sql.Append('\'').Append(g.ToString()).Append('\'');
- return;
-
- case DateTime dateTime:
- case DateTimeOffset dateTimeOffset:
- case Array:
-#if NET
- case DateOnly dateOnly:
- case TimeOnly timeOnly:
-#endif
- throw new UnreachableException("Database-specific format, needs to be implemented in the provider's derived translator.");
-
- case null:
- this._sql.Append("NULL");
- return;
-
- default:
- throw new NotSupportedException("Unsupported constant type: " + value.GetType().Name);
- }
- }
-
- private void TranslateMember(MemberExpression memberExpression, bool isSearchCondition)
- {
- if (this.TryBindProperty(memberExpression, out var property))
- {
- this.GenerateColumn(property, isSearchCondition);
- return;
- }
-
- throw new NotSupportedException($"Member access for '{memberExpression.Member.Name}' is unsupported - only member access over the filter parameter are supported");
- }
-
- protected virtual void GenerateColumn(PropertyModel property, bool isSearchCondition = false)
- // StorageName is considered to be a safe input, we quote and escape it mostly to produce valid SQL.
- => this._sql.Append('"').Append(property.StorageName.Replace("\"", "\"\"")).Append('"');
-
- protected abstract void TranslateQueryParameter(object? value);
-
- private void TranslateMethodCall(MethodCallExpression methodCall, bool isSearchCondition = false)
- {
- // Dictionary access for dynamic mapping (r => r["SomeString"] == "foo")
- if (this.TryBindProperty(methodCall, out var property))
- {
- this.GenerateColumn(property, isSearchCondition);
- return;
- }
-
- switch (methodCall)
- {
- // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains()
- case var _ when TryMatchContains(methodCall, out var source, out var item):
- this.TranslateContains(source, item);
- return;
-
- // Enumerable.Any() with a Contains predicate (r => r.Strings.Any(s => array.Contains(s)))
- case { Method.Name: nameof(Enumerable.Any), Arguments: [var anySource, LambdaExpression lambda] } any
- when any.Method.DeclaringType == typeof(Enumerable):
- this.TranslateAny(anySource, lambda);
- return;
-
- default:
- throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}");
- }
- }
-
- private void TranslateContains(Expression source, Expression item)
- {
- switch (source)
- {
- // Contains over array column (r => r.Strings.Contains("foo"))
- case var _ when this.TryBindProperty(source, out _):
- this.TranslateContainsOverArrayColumn(source, item);
- return;
-
- // Contains over inline array (r => new[] { "foo", "bar" }.Contains(r.String))
- case NewArrayExpression newArray:
- this.Translate(item);
- this._sql.Append(" IN (");
-
- var isFirst = true;
- foreach (var element in newArray.Expressions)
- {
- if (isFirst)
- {
- isFirst = false;
- }
- else
- {
- this._sql.Append(", ");
- }
-
- this.Translate(element);
- }
-
- this._sql.Append(')');
- return;
-
- // Contains over captured array (r => arrayLocalVariable.Contains(r.String))
- case QueryParameterExpression { Value: var value }:
- this.TranslateContainsOverParameterizedArray(source, item, value);
- return;
-
- default:
- throw new NotSupportedException("Unsupported Contains expression");
- }
- }
-
- protected abstract void TranslateContainsOverArrayColumn(Expression source, Expression item);
-
- protected abstract void TranslateContainsOverParameterizedArray(Expression source, Expression item, object? value);
-
- ///
- /// Translates an Any() call with a Contains predicate, e.g. r.Strings.Any(s => array.Contains(s)).
- /// This checks whether any element in the array column is contained in the given values.
- ///
- private void TranslateAny(Expression source, LambdaExpression lambda)
- {
- // We only support the pattern: r.ArrayColumn.Any(x => values.Contains(x))
- // where 'values' is an inline array, captured array, or captured list.
- if (!this.TryBindProperty(source, out var property)
- || lambda.Body is not MethodCallExpression containsCall
- || !TryMatchContains(containsCall, out var valuesExpression, out var itemExpression))
- {
- throw new NotSupportedException("Unsupported method call: Enumerable.Any");
- }
-
- // Verify that the item is the lambda parameter
- if (itemExpression != lambda.Parameters[0])
- {
- throw new NotSupportedException("Unsupported method call: Enumerable.Any");
- }
-
- // Now extract the values from valuesExpression
- switch (valuesExpression)
- {
- // Inline array: r.Strings.Any(s => new[] { "a", "b" }.Contains(s))
- case NewArrayExpression newArray:
- {
- var values = new object?[newArray.Expressions.Count];
- for (var i = 0; i < newArray.Expressions.Count; i++)
- {
- values[i] = newArray.Expressions[i] switch
- {
- ConstantExpression { Value: var v } => v,
- QueryParameterExpression { Value: var v } => v,
- _ => throw new NotSupportedException("Unsupported method call: Enumerable.Any")
- };
- }
-
- this.TranslateAnyContainsOverArrayColumn(property, values);
- return;
- }
-
- // Captured/parameterized array or list: r.Strings.Any(s => capturedArray.Contains(s))
- case QueryParameterExpression { Value: var value }:
- this.TranslateAnyContainsOverArrayColumn(property, value);
- return;
-
- // Constant array: shouldn't normally happen, but handle it
- case ConstantExpression { Value: var value }:
- this.TranslateAnyContainsOverArrayColumn(property, value);
- return;
-
- default:
- throw new NotSupportedException("Unsupported method call: Enumerable.Any");
- }
- }
-
- protected abstract void TranslateAnyContainsOverArrayColumn(PropertyModel property, object? values);
-
- private void TranslateUnary(UnaryExpression unary, bool isSearchCondition)
- {
- switch (unary.NodeType)
- {
- case ExpressionType.Not:
- // Special handling for !(a == b) and !(a != b)
- if (unary.Operand is BinaryExpression { NodeType: ExpressionType.Equal or ExpressionType.NotEqual } binary)
- {
- this.TranslateBinary(
- Expression.MakeBinary(
- binary.NodeType is ExpressionType.Equal ? ExpressionType.NotEqual : ExpressionType.Equal,
- binary.Left,
- binary.Right));
- return;
- }
-
- this._sql.Append("(NOT ");
- this.Translate(unary.Operand, isSearchCondition);
- this._sql.Append(')');
- return;
-
- // Handle converting non-nullable to nullable; such nodes are found in e.g. r => r.Int == nullableInt
- case ExpressionType.Convert when Nullable.GetUnderlyingType(unary.Type) == unary.Operand.Type:
- this.Translate(unary.Operand, isSearchCondition);
- return;
-
- // Handle convert over member access, for dynamic dictionary access (r => (int)r["SomeInt"] == 8)
- case ExpressionType.Convert when this.TryBindProperty(unary.Operand, out var property) && unary.Type == property.Type:
- this.GenerateColumn(property, isSearchCondition);
- return;
-
- default:
- throw new NotSupportedException("Unsupported unary expression node type: " + unary.NodeType);
- }
- }
-}
diff --git a/dotnet/src/VectorData/CosmosMongoDB/AssemblyInfo.cs b/dotnet/src/VectorData/CosmosMongoDB/AssemblyInfo.cs
deleted file mode 100644
index cbb67c1c8afd..000000000000
--- a/dotnet/src/VectorData/CosmosMongoDB/AssemblyInfo.cs
+++ /dev/null
@@ -1 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
diff --git a/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollection.cs b/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollection.cs
deleted file mode 100644
index 52cc3ded9a2d..000000000000
--- a/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollection.cs
+++ /dev/null
@@ -1,653 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Diagnostics.CodeAnalysis;
-using System.Linq;
-using System.Linq.Expressions;
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.Extensions.AI;
-using Microsoft.Extensions.VectorData;
-using Microsoft.Extensions.VectorData.ProviderServices;
-using Microsoft.SemanticKernel.Connectors.MongoDB;
-using MongoDB.Bson;
-using MongoDB.Bson.Serialization;
-using MongoDB.Driver;
-using MEVD = Microsoft.Extensions.VectorData;
-
-namespace Microsoft.SemanticKernel.Connectors.CosmosMongoDB;
-
-///
-/// Service for storing and retrieving vector records, that uses Azure CosmosDB MongoDB as the underlying storage.
-///
-/// The data type of the record key. Must be either .
-/// The data model to use for adding, updating and retrieving data from storage.
-#pragma warning disable CA1711 // Identifiers should not have incorrect suffix
-public class CosmosMongoCollection : VectorStoreCollection
- where TKey : notnull
- where TRecord : class
-#pragma warning restore CA1711 // Identifiers should not have incorrect suffix
-{
- /// Metadata about vector store record collection.
- private readonly VectorStoreCollectionMetadata _collectionMetadata;
-
- /// Property name to be used for search similarity score value.
- private const string ScorePropertyName = "similarityScore";
-
- /// Property name to be used for search document value.
- private const string DocumentPropertyName = "document";
-
- /// The default options for vector search.
- private static readonly MEVD.VectorSearchOptions s_defaultVectorSearchOptions = new();
-
- /// that can be used to manage the collections in Azure CosmosDB MongoDB.
- private readonly IMongoDatabase _mongoDatabase;
-
- /// Azure CosmosDB MongoDB collection to perform record operations.
- private readonly IMongoCollection _mongoCollection;
-
- /// Interface for mapping between a storage model, and the consumer record data model.
- private readonly IMongoMapper _mapper;
-
- /// The model for this collection.
- private readonly CollectionModel _model;
-
- ///
- public override string Name { get; }
-
- /// This integer is the number of clusters that the inverted file (IVF) index uses to group the vector data.
- private readonly int _numLists;
-
- /// The size of the dynamic candidate list for constructing the graph.
- private readonly int _efConstruction;
-
- /// The size of the dynamic candidate list for search.
- private readonly int _efSearch;
-
- /// to use for serializing key values.
- private readonly BsonSerializationInfo? _keySerializationInfo;
-
- private static readonly Type[] s_validKeyTypes = [typeof(string), typeof(Guid), typeof(ObjectId), typeof(int), typeof(long)];
-
- ///
- /// Initializes a new instance of the class.
- ///
- /// that can be used to manage the collections in Azure CosmosDB MongoDB.
- /// The name of the collection that this will access.
- /// Optional configuration options for this class.
- [RequiresDynamicCode("This constructor is incompatible with NativeAOT. For dynamic mapping via Dictionary, instantiate CosmosMongoDynamicCollection instead.")]
- [RequiresUnreferencedCode("This constructor is incompatible with trimming. For dynamic mapping via Dictionary, instantiate CosmosMongoDynamicCollection instead.")]
- public CosmosMongoCollection(
- IMongoDatabase mongoDatabase,
- string name,
- CosmosMongoCollectionOptions? options = default)
- : this(
- mongoDatabase,
- name,
- static options => typeof(TRecord) == typeof(Dictionary)
- ? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(CosmosMongoDynamicCollection)))
- : new MongoModelBuilder().Build(typeof(TRecord), typeof(TKey), options.Definition, options.EmbeddingGenerator),
- options)
- {
- }
-
- internal CosmosMongoCollection(IMongoDatabase mongoDatabase, string name, Func modelFactory, CosmosMongoCollectionOptions? options)
- {
- // Verify.
- Verify.NotNull(mongoDatabase);
- Verify.NotNullOrWhiteSpace(name);
-
- if (!s_validKeyTypes.Contains(typeof(TKey)) && typeof(TKey) != typeof(object))
- {
- throw new NotSupportedException("Only ObjectID, string, Guid, int and long keys are supported.");
- }
-
- options ??= CosmosMongoCollectionOptions.Default;
-
- // Assign.
- this._mongoDatabase = mongoDatabase;
- this._mongoCollection = mongoDatabase.GetCollection(name);
- this.Name = name;
- this._model = modelFactory(options);
- this._numLists = options.NumLists;
- this._efConstruction = options.EfConstruction;
- this._efSearch = options.EfSearch;
-
- this._mapper = typeof(TRecord) == typeof(Dictionary)
- ? (new MongoDynamicMapper(this._model) as IMongoMapper)!
- : new MongoMapper(this._model);
-
- this._collectionMetadata = new()
- {
- VectorStoreSystemName = CosmosMongoConstants.VectorStoreSystemName,
- VectorStoreName = mongoDatabase.DatabaseNamespace?.DatabaseName,
- CollectionName = name
- };
-
- // Cache the key serialization info if possible
- this._keySerializationInfo = typeof(TKey) == typeof(object)
- ? null
- : this.GetKeySerializationInfo();
- }
-
- ///
- public override Task CollectionExistsAsync(CancellationToken cancellationToken = default)
- => this.RunOperationAsync("ListCollectionNames", () => this.InternalCollectionExistsAsync(cancellationToken));
-
- ///
- public override async Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default)
- {
- await this.RunOperationAsync("CreateCollection",
- () => this._mongoDatabase.CreateCollectionAsync(this.Name, cancellationToken: cancellationToken)).ConfigureAwait(false);
-
- await this.RunOperationAsync("CreateIndexes",
- () => this.CreateIndexesAsync(this.Name, cancellationToken: cancellationToken)).ConfigureAwait(false);
- }
-
- ///
- public override async Task DeleteAsync(TKey key, CancellationToken cancellationToken = default)
- {
- Verify.NotNull(key);
-
- await this.RunOperationAsync("DeleteOne", () => this._mongoCollection.DeleteOneAsync(this.GetFilterById(key), cancellationToken))
- .ConfigureAwait(false);
- }
-
- ///
- public override async Task DeleteAsync(IEnumerable keys, CancellationToken cancellationToken = default)
- {
- Verify.NotNull(keys);
-
- await this.RunOperationAsync("DeleteMany", () => this._mongoCollection.DeleteManyAsync(this.GetFilterByIds(keys), cancellationToken))
- .ConfigureAwait(false);
- }
-
- ///
- public override Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default)
- => this.RunOperationAsync("DropCollection", () => this._mongoDatabase.DropCollectionAsync(this.Name, cancellationToken));
-
- ///
- public override async Task GetAsync(TKey key, RecordRetrievalOptions? options = null, CancellationToken cancellationToken = default)
- {
- Verify.NotNull(key);
-
- var includeVectors = options?.IncludeVectors ?? false;
- if (includeVectors && this._model.EmbeddingGenerationRequired)
- {
- throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration);
- }
-
- using var cursor = await this
- .FindAsync(this.GetFilterById(key), top: 1, skip: null, includeVectors, sortDefinition: null, cancellationToken)
- .ConfigureAwait(false);
-
- var record = await cursor.SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false);
-
- if (record is null)
- {
- return default;
- }
-
- return this._mapper.MapFromStorageToDataModel(record, includeVectors);
- }
-
- ///
- public override async IAsyncEnumerable GetAsync(
- IEnumerable keys,
- RecordRetrievalOptions? options = null,
- [EnumeratorCancellation] CancellationToken cancellationToken = default)
- {
- Verify.NotNull(keys);
-
- var includeVectors = options?.IncludeVectors ?? false;
- if (includeVectors && this._model.EmbeddingGenerationRequired)
- {
- throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration);
- }
-
- using var cursor = await this
- .FindAsync(this.GetFilterByIds(keys), top: null, skip: null, includeVectors, sortDefinition: null, cancellationToken)
- .ConfigureAwait(false);
-
- while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
- {
- foreach (var record in cursor.Current)
- {
- if (record is not null)
- {
- yield return this._mapper.MapFromStorageToDataModel(record, includeVectors);
- }
- }
- }
- }
-
- ///
- public override async Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default)
- {
- Verify.NotNull(record);
-
- (_, var generatedEmbeddings) = await ProcessEmbeddingsAsync(this._model, [record], cancellationToken).ConfigureAwait(false);
-
- await this.UpsertCoreAsync(record, recordIndex: 0, generatedEmbeddings, cancellationToken).ConfigureAwait(false);
- }
-
- ///
- public override async Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default)
- {
- Verify.NotNull(records);
-
- (records, var generatedEmbeddings) = await ProcessEmbeddingsAsync(this._model, records, cancellationToken).ConfigureAwait(false);
-
- var i = 0;
-
- foreach (var record in records)
- {
- await this.UpsertCoreAsync(record, i++, generatedEmbeddings, cancellationToken).ConfigureAwait(false);
- }
- }
-
- private async Task UpsertCoreAsync(TRecord record, int recordIndex, IReadOnlyList?[]? generatedEmbeddings, CancellationToken cancellationToken = default)
- {
- const string OperationName = "ReplaceOne";
-
- // Handle auto-generated keys
- var keyProperty = this._model.KeyProperty;
- if (keyProperty.IsAutoGenerated)
- {
- switch (keyProperty.Type)
- {
- case var t when t == typeof(Guid):
- if (keyProperty.GetValue(record) == Guid.Empty)
- {
- keyProperty.SetValue(record, Guid.NewGuid());
- }
- break;
-
- case var t when t == typeof(ObjectId):
- if (keyProperty.GetValue(record) == ObjectId.Empty)
- {
- keyProperty.SetValue(record, ObjectId.GenerateNewId());
- }
- break;
-
- default:
- throw new UnreachableException();
- }
- }
-
- var replaceOptions = new ReplaceOptions { IsUpsert = true };
- var storageModel = this._mapper.MapFromDataToStorageModel(record, recordIndex, generatedEmbeddings);
-
- var key = GetStorageKey(storageModel);
-
- await this.RunOperationAsync(OperationName, async () =>
- await this._mongoCollection
- .ReplaceOneAsync(this.GetFilterById(key), storageModel, replaceOptions, cancellationToken)
- .ConfigureAwait(false)).ConfigureAwait(false);
- }
-
- private static TKey GetStorageKey(BsonDocument document)
- => (TKey)BsonTypeMapper.MapToDotNetValue(document[MongoConstants.MongoReservedKeyPropertyName]);
-
- private static async ValueTask<(IEnumerable records, IReadOnlyList?[]?)> ProcessEmbeddingsAsync(
- CollectionModel model,
- IEnumerable records,
- CancellationToken cancellationToken)
- {
- IReadOnlyList? recordsList = null;
-
- // If an embedding generator is defined, invoke it once per property for all records.
- IReadOnlyList?[]? generatedEmbeddings = null;
-
- var vectorPropertyCount = model.VectorProperties.Count;
- for (var i = 0; i < vectorPropertyCount; i++)
- {
- var vectorProperty = model.VectorProperties[i];
-
- if (MongoModelBuilder.IsVectorPropertyTypeValidCore(vectorProperty.Type, out _))
- {
- continue;
- }
-
- // We have a vector property whose type isn't natively supported - we need to generate embeddings.
- Debug.Assert(vectorProperty.EmbeddingGenerator is not null);
-
- // We have a property with embedding generation; materialize the records' enumerable if needed, to
- // prevent multiple enumeration.
- if (recordsList is null)
- {
- recordsList = records is IReadOnlyList r ? r : records.ToList();
-
- if (recordsList.Count == 0)
- {
- return (records, null);
- }
-
- records = recordsList;
- }
-
- // TODO: Ideally we'd group together vector properties using the same generator (and with the same input and output properties),
- // and generate embeddings for them in a single batch. That's some more complexity though.
- generatedEmbeddings ??= new IReadOnlyList?[vectorPropertyCount];
- generatedEmbeddings[i] = await vectorProperty.GenerateEmbeddingsAsync(records.Select(r => vectorProperty.GetValueAsObject(r)), cancellationToken).ConfigureAwait(false);
- }
-
- return (records, generatedEmbeddings);
- }
-
- #region Search
-
- ///
- public override async IAsyncEnumerable> SearchAsync(
- TInput searchValue,
- int top,
- MEVD.VectorSearchOptions? options = null,
- [EnumeratorCancellation] CancellationToken cancellationToken = default)
- {
- Verify.NotNull(searchValue);
- Verify.NotLessThan(top, 1);
-
- options ??= s_defaultVectorSearchOptions;
- if (options.IncludeVectors && this._model.EmbeddingGenerationRequired)
- {
- throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration);
- }
-
- var vectorProperty = this._model.GetVectorPropertyOrSingle(options);
-
- float[] vector = searchValue switch
- {
- ReadOnlyMemory r => Unwrap(r),
- float[] f => f,
- Embedding e => Unwrap(e.Vector),
-
- _ when vectorProperty.EmbeddingGenerationDispatcher is not null
- => Unwrap(((Embedding)await vectorProperty.GenerateEmbeddingAsync(searchValue, cancellationToken).ConfigureAwait(false)).Vector),
-
- _ => vectorProperty.EmbeddingGenerator is null
- ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), MongoModelBuilder.SupportedVectorTypes))
- : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType()))
- };
-
- var filter = options.Filter is not null
- ? new CosmosMongoFilterTranslator().Translate(options.Filter, this._model)
- : null;
-
- // Constructing a query to fetch "skip + top" total items
- // to perform skip logic locally, since skip option is not part of API.
- var itemsAmount = options.Skip + top;
-
- var vectorPropertyIndexKind = CosmosMongoCollectionSearchMapping.GetVectorPropertyIndexKind(vectorProperty.IndexKind);
-
- var searchQuery = vectorPropertyIndexKind switch
- {
- IndexKind.Hnsw => CosmosMongoCollectionSearchMapping.GetSearchQueryForHnswIndex(
- vector,
- vectorProperty.StorageName,
- itemsAmount,
- this._efSearch,
- filter),
- IndexKind.IvfFlat => CosmosMongoCollectionSearchMapping.GetSearchQueryForIvfIndex(
- vector,
- vectorProperty.StorageName,
- itemsAmount,
- filter),
- _ => throw new InvalidOperationException(
- $"Index kind '{vectorProperty.IndexKind}' on {nameof(VectorStoreVectorProperty)} '{vectorProperty.StorageName}' is not supported by the Azure CosmosDB for MongoDB VectorStore. " +
- $"Supported index kinds are: {string.Join(", ", [IndexKind.Hnsw, IndexKind.IvfFlat])}")
- };
-
- var projectionQuery = CosmosMongoCollectionSearchMapping.GetProjectionQuery(
- ScorePropertyName,
- DocumentPropertyName);
-
- List pipeline = [searchQuery, projectionQuery];
-
- // Add score threshold filter as a $match stage if specified
- if (options.ScoreThreshold.HasValue)
- {
- pipeline.Add(CosmosMongoCollectionSearchMapping.GetScoreThresholdMatchQuery(ScorePropertyName, options.ScoreThreshold.Value));
- }
-
- const string OperationName = "Aggregate";
- var cursor = await this.RunOperationAsync(
- OperationName,
- () => this._mongoCollection.AggregateAsync(pipeline, cancellationToken: cancellationToken)).ConfigureAwait(false);
- using var errorHandlingAsyncCursor = new ErrorHandlingAsyncCursor(cursor, this._collectionMetadata, OperationName);
-
- await foreach (var result in this.EnumerateAndMapSearchResultsAsync(errorHandlingAsyncCursor, options, cancellationToken).ConfigureAwait(false))
- {
- yield return result;
- }
-
- static float[] Unwrap(ReadOnlyMemory memory)
- => MemoryMarshal.TryGetArray(memory, out ArraySegment segment) && segment.Count == segment.Array!.Length
- ? segment.Array
- : memory.ToArray();
- }
-
- #endregion Search
-
- ///
- public override object? GetService(Type serviceType, object? serviceKey = null)
- {
- Verify.NotNull(serviceType);
-
- return
- serviceKey is not null ? null :
- serviceType == typeof(VectorStoreCollectionMetadata) ? this._collectionMetadata :
- serviceType == typeof(IMongoDatabase) ? this._mongoDatabase :
- serviceType == typeof(IMongoCollection) ? this._mongoCollection :
- serviceType.IsInstanceOfType(this) ? this :
- null;
- }
-
- ///
- public override async IAsyncEnumerable GetAsync(
- Expression> filter,
- int top,
- FilteredRecordRetrievalOptions? options = null,
- [EnumeratorCancellation] CancellationToken cancellationToken = default)
- {
- Verify.NotNull(filter);
- Verify.NotLessThan(top, 1);
-
- options ??= new();
-
- // Translate the filter now, so if it fails, we throw immediately.
- var translatedFilter = new CosmosMongoFilterTranslator().Translate(filter, this._model);
-
- SortDefinition? sortDefinition = null;
- var orderBy = options.OrderBy?.Invoke(new()).Values;
- if (orderBy is { Count: > 0 })
- {
- sortDefinition = Builders.Sort.Combine(
- orderBy.Select(pair =>
- {
- var storageName = this._model.GetDataOrKeyProperty(pair.PropertySelector).StorageName;
-
- return pair.Ascending
- ? Builders.Sort.Ascending(storageName)
- : Builders.Sort.Descending(storageName);
- }));
- }
-
- using IAsyncCursor cursor = await this.FindAsync(
- translatedFilter,
- top,
- options.Skip,
- options.IncludeVectors,
- sortDefinition,
- cancellationToken).ConfigureAwait(false);
-
- while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
- {
- foreach (var response in cursor.Current)
- {
- var record = this._mapper.MapFromStorageToDataModel(response, options.IncludeVectors);
-
- yield return record;
- }
- }
- }
-
- #region private
-
- private async Task CreateIndexesAsync(string collectionName, CancellationToken cancellationToken)
- {
- const string OperationName = "CreateIndexes";
-
- var indexCursor = await this._mongoCollection.Indexes.ListAsync(cancellationToken).ConfigureAwait(false);
- var indexes = indexCursor.ToList(cancellationToken).Select(index => index["name"].ToString()) ?? [];
- var uniqueIndexes = new HashSet(indexes);
-
- var indexArray = new BsonArray();
-
- indexArray.AddRange(CosmosMongoCollectionCreateMapping.GetVectorIndexes(
- this._model.VectorProperties,
- uniqueIndexes,
- this._numLists,
- this._efConstruction));
-
- indexArray.AddRange(CosmosMongoCollectionCreateMapping.GetFilterableDataIndexes(
- this._model.DataProperties,
- uniqueIndexes));
-
- if (indexArray.Count > 0)
- {
- var createIndexCommand = new BsonDocument
- {
- { "createIndexes", collectionName },
- { "indexes", indexArray }
- };
-
- var cursor = await this.RunOperationAsync(OperationName, () =>
- this._mongoDatabase.RunCommandAsync(createIndexCommand, cancellationToken: cancellationToken)).ConfigureAwait(false);
- }
- }
-
- private async Task> FindAsync(
- FilterDefinition filter,
- int? top,
- int? skip,
- bool includeVectors,
- SortDefinition? sortDefinition,
- CancellationToken cancellationToken)
- {
- const string OperationName = "Find";
-
- ProjectionDefinitionBuilder projectionBuilder = Builders.Projection;
- ProjectionDefinition? projectionDefinition = null;
-
- if (!includeVectors && this._model.VectorProperties.Count > 0)
- {
- foreach (var vectorProperty in this._model.VectorProperties)
- {
- projectionDefinition = projectionDefinition is not null ?
- projectionDefinition.Exclude(vectorProperty.StorageName) :
- projectionBuilder.Exclude(vectorProperty.StorageName);
- }
- }
-
- var findOptions = projectionDefinition is not null ?
- new FindOptions { Projection = projectionDefinition, Limit = top, Skip = skip, Sort = sortDefinition } :
- new FindOptions { Limit = top, Skip = skip, Sort = sortDefinition };
-
- var cursor = await this.RunOperationAsync(OperationName, () =>
- this._mongoCollection.FindAsync(filter, findOptions, cancellationToken)).ConfigureAwait(false);
-
- return new ErrorHandlingAsyncCursor(cursor, this._collectionMetadata, OperationName);
- }
-
- private async IAsyncEnumerable