diff --git a/dotnet/samples/Concepts/AudioToText/OpenAI_AudioToText.cs b/dotnet/samples/Concepts/AudioToText/OpenAI_AudioToText.cs index c20ceb662428..ff53e6a66502 100644 --- a/dotnet/samples/Concepts/AudioToText/OpenAI_AudioToText.cs +++ b/dotnet/samples/Concepts/AudioToText/OpenAI_AudioToText.cs @@ -50,4 +50,26 @@ public async Task AudioToTextAsync() // Output the transcribed text Console.WriteLine(textContent.Text); } + + [Fact(Skip = "Requires a local OpenAI-compatible audio-to-text server, such as FunASR.")] + public async Task OpenAICompatibleAudioToTextAsync() + { + // FunASR exposes the OpenAI-compatible endpoint at /v1/audio/transcriptions. + var kernel = Kernel.CreateBuilder() + .AddOpenAIAudioToText( + modelId: "sensevoice", + endpoint: new Uri("http://localhost:8000/v1")) + .Build(); + + var audioToTextService = kernel.GetRequiredService(); + await using var audioFileStream = EmbeddedResource.ReadStream(AudioFilename); + var audioFileBinaryData = await BinaryData.FromStreamAsync(audioFileStream!); + AudioContent audioContent = new(audioFileBinaryData, mimeType: null); + + var textContent = await audioToTextService.GetTextContentAsync( + audioContent, + new OpenAIAudioToTextExecutionSettings(AudioFilename)); + + Console.WriteLine(textContent.Text); + } } diff --git a/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/KernelBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/KernelBuilderExtensionsTests.cs index 972cdb62b2ac..524fc01dd630 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/KernelBuilderExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/KernelBuilderExtensionsTests.cs @@ -129,6 +129,32 @@ public void ItCanAddAudioToTextService() Assert.Equal("model", service.Attributes[AIServiceExtensions.ModelIdKey]); } + [Fact] + public void ItCanAddAudioToTextServiceWithCustomEndpoint() + { + // Arrange + var endpoint = new Uri("http://localhost:10095/v1"); + var sut = Kernel.CreateBuilder(); + + // Act + var service = sut.AddOpenAIAudioToText("model", endpoint) + .Build() + .GetRequiredService(); + + // Assert + Assert.Equal(endpoint.ToString(), service.Attributes[AIServiceExtensions.EndpointKey]); + } + + [Fact] + public void ItThrowsWhenAddingAudioToTextServiceWithoutCustomEndpoint() + { + // Arrange + var sut = Kernel.CreateBuilder(); + + // Act & Assert + Assert.Throws(() => sut.AddOpenAIAudioToText("model", endpoint: null!)); + } + [Fact] public void ItCanAddAudioToTextServiceWithOpenAIClient() { diff --git a/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/ServiceCollectionExtensionsTests.cs b/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/ServiceCollectionExtensionsTests.cs index cdedc54a90d5..d248757f6823 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/ServiceCollectionExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/ServiceCollectionExtensionsTests.cs @@ -157,6 +157,32 @@ public void ItCanAddAudioToTextService() Assert.Equal("model", service.Attributes[AIServiceExtensions.ModelIdKey]); } + [Fact] + public void ItCanAddAudioToTextServiceWithCustomEndpoint() + { + // Arrange + var endpoint = new Uri("http://localhost:10095/v1"); + var sut = new ServiceCollection(); + + // Act + var service = sut.AddOpenAIAudioToText("model", endpoint) + .BuildServiceProvider() + .GetRequiredService(); + + // Assert + Assert.Equal(endpoint.ToString(), service.Attributes[AIServiceExtensions.EndpointKey]); + } + + [Fact] + public void ItThrowsWhenAddingAudioToTextServiceWithoutCustomEndpoint() + { + // Arrange + var sut = new ServiceCollection(); + + // Act & Assert + Assert.Throws(() => sut.AddOpenAIAudioToText("model", endpoint: null!)); + } + [Fact] public void ItCanAddAudioToTextServiceWithOpenAIClient() { diff --git a/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Services/OpenAIAudioToTextServiceTests.cs b/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Services/OpenAIAudioToTextServiceTests.cs index ec415ac8e9f7..fed7ea7cfc73 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Services/OpenAIAudioToTextServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Services/OpenAIAudioToTextServiceTests.cs @@ -45,6 +45,35 @@ public void ConstructorWithApiKeyWorksCorrectly(bool includeLoggerFactory) Assert.Equal("model-id", service.Attributes["ModelId"]); } + [Fact] + public async Task ItUsesCustomEndpointWithoutApiKeyAsync() + { + // Arrange + var endpoint = new Uri("http://localhost:10095/v1"); + var service = new OpenAIAudioToTextService("model-id", endpoint, httpClient: this._httpClient); + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent("Test audio-to-text response") + }; + + // Act + await service.GetTextContentsAsync( + new AudioContent(new BinaryData("data"), mimeType: null), + new OpenAIAudioToTextExecutionSettings("file.mp3")); + + // Assert + Assert.Equal("http://localhost:10095/v1/audio/transcriptions", this._messageHandlerStub.RequestUri!.ToString()); + Assert.Equal(endpoint.ToString(), service.Attributes["Endpoint"]); + } + + [Fact] + public void ItThrowsIfCustomEndpointIsNotProvided() + { + // Act & Assert + var exception = Assert.Throws(() => new OpenAIAudioToTextService("model-id", endpoint: null!)); + Assert.Equal("endpoint", exception.ParamName); + } + [Fact] public void ItThrowsIfModelIdIsNotProvided() { diff --git a/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIKernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIKernelBuilderExtensions.cs index 1d6f79766974..7e76dde07943 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIKernelBuilderExtensions.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIKernelBuilderExtensions.cs @@ -270,6 +270,44 @@ public static IKernelBuilder AddOpenAIAudioToText( return builder; } + /// + /// Adds an OpenAI-compatible audio-to-text service with a custom endpoint to the list. + /// + /// The instance to augment. + /// Model name. + /// OpenAI-compatible API endpoint. + /// Optional API key. + /// OpenAI organization id. + /// A local identifier for the given AI service. + /// The HttpClient to use with this service. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IKernelBuilder AddOpenAIAudioToText( + this IKernelBuilder builder, + string modelId, + Uri endpoint, + string? apiKey = null, + string? orgId = null, + string? serviceId = null, + HttpClient? httpClient = null) + { + Verify.NotNull(builder); + Verify.NotNullOrWhiteSpace(modelId); + Verify.NotNull(endpoint); + + Func factory = (serviceProvider, _) => + new(modelId, + endpoint, + apiKey, + orgId, + HttpClientProvider.GetHttpClient(httpClient, serviceProvider), + serviceProvider.GetService()); + + builder.Services.AddKeyedSingleton(serviceId, factory); + + return builder; + } + /// /// Adds the OpenAI audio-to-text service to the list. /// diff --git a/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIServiceCollectionExtensions.cs b/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIServiceCollectionExtensions.cs index df36afb11ca2..20f86092779c 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIServiceCollectionExtensions.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIServiceCollectionExtensions.cs @@ -186,6 +186,42 @@ public static IServiceCollection AddOpenAIAudioToText( return services; } + /// + /// Adds an OpenAI-compatible audio-to-text service with a custom endpoint to the list. + /// + /// The instance to augment. + /// Model name. + /// OpenAI-compatible API endpoint. + /// Optional API key. + /// OpenAI organization id. + /// A local identifier for the given AI service. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IServiceCollection AddOpenAIAudioToText( + this IServiceCollection services, + string modelId, + Uri endpoint, + string? apiKey = null, + string? orgId = null, + string? serviceId = null) + { + Verify.NotNull(services); + Verify.NotNullOrWhiteSpace(modelId); + Verify.NotNull(endpoint); + + Func factory = (serviceProvider, _) => + new(modelId, + endpoint, + apiKey, + orgId, + HttpClientProvider.GetHttpClient(serviceProvider), + serviceProvider.GetService()); + + services.AddKeyedSingleton(serviceId, factory); + + return services; + } + /// /// Adds the OpenAI audio-to-text service to the list. /// diff --git a/dotnet/src/Connectors/Connectors.OpenAI/Services/OpenAIAudioToTextService.cs b/dotnet/src/Connectors/Connectors.OpenAI/Services/OpenAIAudioToTextService.cs index 331da48cc08c..300f1197282a 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/Services/OpenAIAudioToTextService.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/Services/OpenAIAudioToTextService.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Net.Http; @@ -12,13 +13,13 @@ namespace Microsoft.SemanticKernel.Connectors.OpenAI; /// -/// OpenAI text-to-audio service. +/// OpenAI audio-to-text service. /// [Experimental("SKEXP0010")] public sealed class OpenAIAudioToTextService : IAudioToTextService { /// - /// OpenAI text-to-audio client for HTTP operations. + /// OpenAI audio-to-text client for HTTP operations. /// private readonly ClientCore _client; @@ -44,6 +45,28 @@ public OpenAIAudioToTextService( this._client = new(modelId, apiKey, organization, null, httpClient, loggerFactory?.CreateLogger(typeof(OpenAIAudioToTextService))); } + /// + /// Initializes a new instance of the class for a custom OpenAI-compatible endpoint. + /// + /// Model name. + /// OpenAI-compatible API endpoint. + /// Optional API key. + /// OpenAI Organization Id (usually optional). + /// Custom for HTTP requests. + /// The to use for logging. If null, no logging will be performed. + public OpenAIAudioToTextService( + string modelId, + Uri endpoint, + string? apiKey = null, + string? organization = null, + HttpClient? httpClient = null, + ILoggerFactory? loggerFactory = null) + { + Verify.NotNullOrWhiteSpace(modelId, nameof(modelId)); + Verify.NotNull(endpoint); + this._client = new(modelId, apiKey, organization, endpoint, httpClient, loggerFactory?.CreateLogger(typeof(OpenAIAudioToTextService))); + } + /// /// Initializes a new instance of the class. ///