Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
RubricScore,
)
from agent_framework._feature_stage import ExperimentalFeature, experimental
from agent_framework._settings import load_settings
from agent_framework._telemetry import mark_feature_used
from agent_framework._types import Message

Expand Down Expand Up @@ -859,7 +860,9 @@ class FoundryEvals:
project_client: An async ``AIProjectClient`` instance
(from ``azure.ai.projects.aio``). Provide this or *client*.
model: Model deployment name for the evaluator LLM judge.
Resolved from ``client.model`` when omitted.
Resolved from ``client.model`` when omitted. When a client is
auto-created, uses ``FOUNDRY_MODEL`` before falling back to
``"gpt-4o"``.
evaluators: Evaluator specifications. Entries may be built-in
short names (e.g. ``"relevance"``), fully-qualified
``"builtin.*"`` names, or :class:`GeneratedEvaluatorRef`
Expand Down Expand Up @@ -922,9 +925,10 @@ def __init__(

# Auto-create a FoundryChatClient from env vars when no client is provided
if client is None and project_client is None:
from ._chat_client import FoundryChatClient
from ._chat_client import FoundryChatClient, FoundrySettings

client = FoundryChatClient(model=model or "gpt-4o")
foundry_settings = load_settings(FoundrySettings, env_prefix="FOUNDRY_", model=model)
client = FoundryChatClient(model=foundry_settings.get("model") or "gpt-4o")

self._client = _resolve_openai_client(client, project_client)
# Resolve model: explicit param > client.model > error
Expand Down
60 changes: 60 additions & 0 deletions python/packages/foundry/tests/test_foundry_evals.py
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,66 @@ def test_with_context_and_tools(self) -> None:


class TestFoundryEvals:
@pytest.mark.parametrize(
("model", "env_model", "expected_model"),
[
("explicit-model", "env-model", "explicit-model"),
("explicit-model", None, "explicit-model"),
(None, "env-model", "env-model"),
(None, None, "gpt-4o"),
(None, "", "gpt-4o"),
],
)
def test_auto_created_client_model_precedence(
self,
monkeypatch: pytest.MonkeyPatch,
model: str | None,
env_model: str | None,
expected_model: str,
) -> None:
if env_model is None:
monkeypatch.delenv("FOUNDRY_MODEL", raising=False)
else:
monkeypatch.setenv("FOUNDRY_MODEL", env_model)

mock_openai_client = MagicMock()

@dataclass(kw_only=True)
class StubFoundryChatClient:
model: str
client: Any = mock_openai_client

monkeypatch.setattr("agent_framework_foundry._chat_client.FoundryChatClient", StubFoundryChatClient)

fe = FoundryEvals(model=model)

assert fe._model == expected_model
assert fe._client is mock_openai_client

@pytest.mark.parametrize("injected_client", ["client", "project_client"])
def test_injected_client_does_not_auto_create_chat_client(
self, monkeypatch: pytest.MonkeyPatch, injected_client: str
) -> None:
from agent_framework_foundry import FoundryChatClient

monkeypatch.setenv("FOUNDRY_MODEL", "env-model")
constructor = MagicMock(side_effect=AssertionError("Must use the injected client"))
monkeypatch.setattr(FoundryChatClient, "__init__", constructor)
mock_openai_client = MagicMock(spec=AsyncOpenAI)
kwargs: dict[str, Any] = {"model": "explicit-model"}
if injected_client == "client":
kwargs["client"] = mock_openai_client
else:
mock_project = MagicMock()
mock_project.get_openai_client.return_value = mock_openai_client
kwargs["project_client"] = mock_project

fe = FoundryEvals(**kwargs)

assert fe._model == "explicit-model"
assert fe._client is mock_openai_client
constructor.assert_not_called()

def test_constructor_with_openai_client(self) -> None:
mock_client = MagicMock()
fe = FoundryEvals(client=mock_client, model="gpt-4o")
Expand Down
Loading