Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,75 @@ public async Task AsChatClientConvertsServiceToIChatClientAsync()
Assert.IsAssignableFrom<IChatClient>(chatClient);
}

[Fact]
public async Task AutoInvokeDoesNotInvokeFunctionThatWasNotAdvertisedByFunctionChoiceBehaviorAsync()
{
// Arrange - register two functions in the same kernel but only advertise one of them via the
// FunctionChoiceBehavior. The model is then made to request the function that was not advertised.
bool unadvertisedFunctionInvoked = false;
var plugin = KernelPluginFactory.CreateFromFunctions("TestPlugin", new[]
{
KernelFunctionFactory.CreateFromMethod(() => "allowed-result", "AllowedFunction"),
KernelFunctionFactory.CreateFromMethod(() => { unadvertisedFunctionInvoked = true; return "secret"; }, "UnadvertisedFunction"),
});
var kernel = new Kernel();
kernel.Plugins.Add(plugin);

var allowedFunction = kernel.Plugins.GetFunction("TestPlugin", "AllowedFunction");
var settings = new GeminiPromptExecutionSettings
{
FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(functions: [allowedFunction], autoInvoke: true)
};

// The model requests the function that was not advertised, then produces a final text answer.
string functionCallResponse = $$"""
{
"candidates": [
{
"content": {
"parts": [ { "functionCall": { "name": "TestPlugin{{GeminiFunction.NameSeparator}}UnadvertisedFunction", "args": {} } } ],
"role": "model"
},
"finishReason": "STOP",
"index": 0
}
],
"usageMetadata": { "promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2 }
}
""";
string finalTextResponse = """
{
"candidates": [
{
"content": { "parts": [ { "text": "done" } ], "role": "model" },
"finishReason": "STOP",
"index": 0
}
],
"usageMetadata": { "promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2 }
}
""";
using var functionCallHttpResponse = new HttpResponseMessage(System.Net.HttpStatusCode.OK) { Content = new StringContent(functionCallResponse) };
this._messageHandlerStub.ResponseQueue.Enqueue(functionCallHttpResponse);

using var finalTextHttpResponse = new HttpResponseMessage(System.Net.HttpStatusCode.OK) { Content = new StringContent(finalTextResponse) };
this._messageHandlerStub.ResponseQueue.Enqueue(finalTextHttpResponse);

var service = this.CreateChatCompletionService();
var chatHistory = new ChatHistory();
chatHistory.AddUserMessage("Do something");

// Act
var result = await service.GetChatMessageContentsAsync(chatHistory, settings, kernel);

// Assert - the unadvertised function must never be invoked, even though it exists in the kernel.
Assert.False(unadvertisedFunctionInvoked);
// The requested tool call was processed and rejected (not silently ignored): both responses were
// consumed and the model's final text answer was returned.
Assert.Empty(this._messageHandlerStub.ResponseQueue);
Assert.Contains(result, m => m.Content is not null && m.Content.Contains("done", StringComparison.Ordinal));
}

private GoogleAIGeminiChatCompletionService CreateChatCompletionService(HttpClient? httpClient = null)
{
return new GoogleAIGeminiChatCompletionService(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -195,78 +195,210 @@ public void KernelFunctionsCloneReturnsCorrectClone()
}

[Fact]
public void FunctionChoiceBehaviorAutoConvertsToAutoInvokeKernelFunctions()
public void FunctionChoiceBehaviorAutoConvertsToEnabledFunctionsWithAllKernelFunctions()
{
// Arrange
var kernel = CreateKernelWithFunctions("FunctionA", "FunctionB");
var settings = new GeminiPromptExecutionSettings
{
FunctionChoiceBehavior = FunctionChoiceBehavior.Auto()
};

// Act
var converted = GeminiPromptExecutionSettings.FromExecutionSettings(settings);
var converted = GeminiPromptExecutionSettings.FromExecutionSettings(settings, kernel);

// Assert
Assert.NotNull(converted.ToolCallBehavior);
Assert.IsType<GeminiToolCallBehavior.KernelFunctions>(converted.ToolCallBehavior);
var enabledFunctions = Assert.IsType<GeminiToolCallBehavior.EnabledFunctions>(converted.ToolCallBehavior);
Assert.True(converted.ToolCallBehavior.MaximumAutoInvokeAttempts > 0);
// The provided set is validated against the requested functions.
Assert.False(GetAllowAnyRequestedKernelFunction(converted.ToolCallBehavior));
// Auto continues to provide the functions across follow-up requests.
Assert.Equal(int.MaxValue, converted.ToolCallBehavior.MaximumUseAttempts);
Assert.Equal(new[] { $"TestPlugin{GeminiFunction.NameSeparator}FunctionA", $"TestPlugin{GeminiFunction.NameSeparator}FunctionB" }, GetAdvertisedFunctionNames(enabledFunctions, kernel));
}

[Fact]
public void FunctionChoiceBehaviorAutoWithNoAutoInvokeConvertsToEnableKernelFunctions()
public void FunctionChoiceBehaviorAutoWithNoAutoInvokeConvertsToEnabledFunctionsWithoutAutoInvoke()
{
// Arrange
var kernel = CreateKernelWithFunctions("FunctionA");
var settings = new GeminiPromptExecutionSettings
{
FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(autoInvoke: false)
};

// Act
var converted = GeminiPromptExecutionSettings.FromExecutionSettings(settings);
var converted = GeminiPromptExecutionSettings.FromExecutionSettings(settings, kernel);

// Assert
Assert.NotNull(converted.ToolCallBehavior);
Assert.IsType<GeminiToolCallBehavior.KernelFunctions>(converted.ToolCallBehavior);
Assert.IsType<GeminiToolCallBehavior.EnabledFunctions>(converted.ToolCallBehavior);
Assert.Equal(0, converted.ToolCallBehavior.MaximumAutoInvokeAttempts);
}

[Fact]
public void FunctionChoiceBehaviorRequiredConvertsToAutoInvokeKernelFunctions()
public void FunctionChoiceBehaviorRequiredConvertsToEnabledFunctions()
{
// Arrange
var kernel = CreateKernelWithFunctions("FunctionA");
var settings = new GeminiPromptExecutionSettings
{
FunctionChoiceBehavior = FunctionChoiceBehavior.Required()
};

// Act
var converted = GeminiPromptExecutionSettings.FromExecutionSettings(settings);
var converted = GeminiPromptExecutionSettings.FromExecutionSettings(settings, kernel);

// Assert
Assert.NotNull(converted.ToolCallBehavior);
Assert.IsType<GeminiToolCallBehavior.KernelFunctions>(converted.ToolCallBehavior);
Assert.True(converted.ToolCallBehavior.MaximumAutoInvokeAttempts > 0);
Assert.IsType<GeminiToolCallBehavior.EnabledFunctions>(converted.ToolCallBehavior);
Assert.False(GetAllowAnyRequestedKernelFunction(converted.ToolCallBehavior));
// Required stops providing the functions after the first request so the model is not
// repeatedly forced to call them on follow-up iterations. Auto-invocation is capped to the same
// value so it does not outlast the advertised tools (MaximumUseAttempts >= MaximumAutoInvokeAttempts).
Assert.Equal(1, converted.ToolCallBehavior.MaximumUseAttempts);
Assert.Equal(1, converted.ToolCallBehavior.MaximumAutoInvokeAttempts);
Assert.True(converted.ToolCallBehavior.MaximumUseAttempts >= converted.ToolCallBehavior.MaximumAutoInvokeAttempts);
}

[Fact]
public void FunctionChoiceBehaviorNoneConvertsToEnableKernelFunctions()
public void FunctionChoiceBehaviorNoneConvertsToEnabledFunctionsWithoutAutoInvoke()
{
// Arrange
var kernel = CreateKernelWithFunctions("FunctionA");
var settings = new GeminiPromptExecutionSettings
{
FunctionChoiceBehavior = FunctionChoiceBehavior.None()
};

// Act
var converted = GeminiPromptExecutionSettings.FromExecutionSettings(settings);
var converted = GeminiPromptExecutionSettings.FromExecutionSettings(settings, kernel);

// Assert
Assert.NotNull(converted.ToolCallBehavior);
Assert.IsType<GeminiToolCallBehavior.KernelFunctions>(converted.ToolCallBehavior);
Assert.IsType<GeminiToolCallBehavior.EnabledFunctions>(converted.ToolCallBehavior);
// None behavior doesn't auto-invoke
Assert.Equal(0, converted.ToolCallBehavior.MaximumAutoInvokeAttempts);
}

[Fact]
public void FunctionChoiceBehaviorAutoWithEmptyFunctionListDisablesFunctionCalling()
{
// Arrange
// An empty function list is documented as being equivalent to disabling function calling.
var kernel = CreateKernelWithFunctions("FunctionA", "FunctionB");
var settings = new GeminiPromptExecutionSettings
{
FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(functions: [], autoInvoke: true)
};

// Act
var converted = GeminiPromptExecutionSettings.FromExecutionSettings(settings, kernel);

// Assert
Assert.Null(converted.ToolCallBehavior);
}

[Fact]
public void FunctionChoiceBehaviorAutoWithSubsetProvidesOnlyThatSubset()
{
// Arrange
var kernel = CreateKernelWithFunctions("First", "Second");
var first = kernel.Plugins.GetFunction("TestPlugin", "First");
var settings = new GeminiPromptExecutionSettings
{
FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(functions: [first], autoInvoke: true)
};

// Act
var converted = GeminiPromptExecutionSettings.FromExecutionSettings(settings, kernel);

// Assert
Assert.NotNull(converted.ToolCallBehavior);
var enabledFunctions = Assert.IsType<GeminiToolCallBehavior.EnabledFunctions>(converted.ToolCallBehavior);
Assert.False(GetAllowAnyRequestedKernelFunction(converted.ToolCallBehavior));
// Only the function specified in the behavior is provided to the model.
Assert.Equal(new[] { $"TestPlugin{GeminiFunction.NameSeparator}First" }, GetAdvertisedFunctionNames(enabledFunctions, kernel));
}

[Fact]
public void FunctionChoiceBehaviorConfigurationErrorsArePropagated()
{
// Arrange - auto-invocation requires the declared function to exist in the kernel; when it does
// not, the behavior resolution throws and the error must surface rather than be swallowed.
var kernelWithFunction = CreateKernelWithFunctions("First");
var first = kernelWithFunction.Plugins.GetFunction("TestPlugin", "First");
var emptyKernel = Kernel.CreateBuilder().Build();
var settings = new GeminiPromptExecutionSettings
{
FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(functions: [first], autoInvoke: true)
};

// Act & Assert
Assert.Throws<KernelException>(() => GeminiPromptExecutionSettings.FromExecutionSettings(settings, emptyKernel));
}

[Fact]
public void FromExecutionSettingsDoesNotMutateCallerSettingsWhenConvertingFunctionChoiceBehavior()
{
// Arrange
var kernel = CreateKernelWithFunctions("FunctionA");
var settings = new GeminiPromptExecutionSettings
{
FunctionChoiceBehavior = FunctionChoiceBehavior.Auto()
};

// Act
var converted = GeminiPromptExecutionSettings.FromExecutionSettings(settings, kernel);

// Assert - the caller's original settings object must remain untouched.
Assert.NotSame(settings, converted);
Assert.Null(settings.ToolCallBehavior);
Assert.NotNull(converted.ToolCallBehavior);
}

[Fact]
public void FromExecutionSettingsWithFrozenSettingsDoesNotThrowWhenConvertingFunctionChoiceBehavior()
{
// Arrange
var kernel = CreateKernelWithFunctions("FunctionA");
var settings = new GeminiPromptExecutionSettings
{
FunctionChoiceBehavior = FunctionChoiceBehavior.Auto()
};
settings.Freeze();

// Act - converting a frozen settings object must not attempt to mutate it.
var converted = GeminiPromptExecutionSettings.FromExecutionSettings(settings, kernel);

// Assert
Assert.NotNull(converted.ToolCallBehavior);
Assert.IsType<GeminiToolCallBehavior.EnabledFunctions>(converted.ToolCallBehavior);
}

[Fact]
public void FromExecutionSettingsReResolvesFunctionChoiceBehaviorPerKernel()
{
// Arrange - a single reused settings object must reflect each request's own kernel.
var settings = new GeminiPromptExecutionSettings
{
FunctionChoiceBehavior = FunctionChoiceBehavior.Auto()
};
var kernelA = CreateKernelWithFunctions("FunctionA");
var kernelB = CreateKernelWithFunctions("FunctionB");

// Act
var convertedA = GeminiPromptExecutionSettings.FromExecutionSettings(settings, kernelA);
var convertedB = GeminiPromptExecutionSettings.FromExecutionSettings(settings, kernelB);

// Assert - each request reflects its own kernel.
var enabledA = Assert.IsType<GeminiToolCallBehavior.EnabledFunctions>(convertedA.ToolCallBehavior);
var enabledB = Assert.IsType<GeminiToolCallBehavior.EnabledFunctions>(convertedB.ToolCallBehavior);
Assert.Equal(new[] { $"TestPlugin{GeminiFunction.NameSeparator}FunctionA" }, GetAdvertisedFunctionNames(enabledA, kernelA));
Assert.Equal(new[] { $"TestPlugin{GeminiFunction.NameSeparator}FunctionB" }, GetAdvertisedFunctionNames(enabledB, kernelB));
}

[Fact]
public void GeminiPromptExecutionSettingsWithNoFunctionChoiceBehaviorDoesNotSetToolCallBehavior()
{
Expand Down Expand Up @@ -311,6 +443,32 @@ private static KernelPlugin GetTestPlugin()
return KernelPluginFactory.CreateFromFunctions("MyPlugin", [function]);
}

private static Kernel CreateKernelWithFunctions(params string[] functionNames)
{
var functions = functionNames
.Select(name => KernelFunctionFactory.CreateFromMethod(() => "result", name))
.ToArray();

var kernel = Kernel.CreateBuilder().Build();
kernel.Plugins.Add(KernelPluginFactory.CreateFromFunctions("TestPlugin", functions));
return kernel;
}

private static bool GetAllowAnyRequestedKernelFunction(GeminiToolCallBehavior behavior)
=> behavior.AllowAnyRequestedKernelFunction;

private static string[] GetAdvertisedFunctionNames(GeminiToolCallBehavior behavior, Kernel kernel)
{
var request = new GeminiRequest();
behavior.ConfigureGeminiRequest(kernel, request);
if (request.Tools is null)
{
return [];
}

return request.Tools[0].Functions.Select(f => f.Name).ToArray();
}

private static void AssertFunctions(GeminiRequest request)
{
Assert.NotNull(request.Tools);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ private ChatCompletionState ValidateInputAndCreateChatCompletionState(
{
ValidateChatHistory(chatHistory);

var geminiExecutionSettings = GeminiPromptExecutionSettings.FromExecutionSettings(executionSettings);
var geminiExecutionSettings = GeminiPromptExecutionSettings.FromExecutionSettings(executionSettings, kernel);
ValidateMaxTokens(geminiExecutionSettings.MaxTokens);

if (this.Logger.IsEnabled(LogLevel.Trace))
Expand Down
Loading
Loading