diff --git a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py index db7bd8dcc08..d604a780285 100644 --- a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py +++ b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py @@ -43,6 +43,7 @@ RubricScore, ) from agent_framework._feature_stage import ExperimentalFeature, experimental +from agent_framework._settings import load_settings from agent_framework._telemetry import mark_feature_used from agent_framework._types import Message @@ -859,7 +860,9 @@ class FoundryEvals: project_client: An async ``AIProjectClient`` instance (from ``azure.ai.projects.aio``). Provide this or *client*. model: Model deployment name for the evaluator LLM judge. - Resolved from ``client.model`` when omitted. + Resolved from ``client.model`` when omitted. When a client is + auto-created, uses ``FOUNDRY_MODEL`` before falling back to + ``"gpt-4o"``. evaluators: Evaluator specifications. Entries may be built-in short names (e.g. ``"relevance"``), fully-qualified ``"builtin.*"`` names, or :class:`GeneratedEvaluatorRef` @@ -922,9 +925,10 @@ def __init__( # Auto-create a FoundryChatClient from env vars when no client is provided if client is None and project_client is None: - from ._chat_client import FoundryChatClient + from ._chat_client import FoundryChatClient, FoundrySettings - client = FoundryChatClient(model=model or "gpt-4o") + foundry_settings = load_settings(FoundrySettings, env_prefix="FOUNDRY_", model=model) + client = FoundryChatClient(model=foundry_settings.get("model") or "gpt-4o") self._client = _resolve_openai_client(client, project_client) # Resolve model: explicit param > client.model > error diff --git a/python/packages/foundry/tests/test_foundry_evals.py b/python/packages/foundry/tests/test_foundry_evals.py index 88e7db6f0f5..98c1554664e 100644 --- a/python/packages/foundry/tests/test_foundry_evals.py +++ b/python/packages/foundry/tests/test_foundry_evals.py @@ -523,6 +523,66 @@ def test_with_context_and_tools(self) -> None: class TestFoundryEvals: + @pytest.mark.parametrize( + ("model", "env_model", "expected_model"), + [ + ("explicit-model", "env-model", "explicit-model"), + ("explicit-model", None, "explicit-model"), + (None, "env-model", "env-model"), + (None, None, "gpt-4o"), + (None, "", "gpt-4o"), + ], + ) + def test_auto_created_client_model_precedence( + self, + monkeypatch: pytest.MonkeyPatch, + model: str | None, + env_model: str | None, + expected_model: str, + ) -> None: + if env_model is None: + monkeypatch.delenv("FOUNDRY_MODEL", raising=False) + else: + monkeypatch.setenv("FOUNDRY_MODEL", env_model) + + mock_openai_client = MagicMock() + + @dataclass(kw_only=True) + class StubFoundryChatClient: + model: str + client: Any = mock_openai_client + + monkeypatch.setattr("agent_framework_foundry._chat_client.FoundryChatClient", StubFoundryChatClient) + + fe = FoundryEvals(model=model) + + assert fe._model == expected_model + assert fe._client is mock_openai_client + + @pytest.mark.parametrize("injected_client", ["client", "project_client"]) + def test_injected_client_does_not_auto_create_chat_client( + self, monkeypatch: pytest.MonkeyPatch, injected_client: str + ) -> None: + from agent_framework_foundry import FoundryChatClient + + monkeypatch.setenv("FOUNDRY_MODEL", "env-model") + constructor = MagicMock(side_effect=AssertionError("Must use the injected client")) + monkeypatch.setattr(FoundryChatClient, "__init__", constructor) + mock_openai_client = MagicMock(spec=AsyncOpenAI) + kwargs: dict[str, Any] = {"model": "explicit-model"} + if injected_client == "client": + kwargs["client"] = mock_openai_client + else: + mock_project = MagicMock() + mock_project.get_openai_client.return_value = mock_openai_client + kwargs["project_client"] = mock_project + + fe = FoundryEvals(**kwargs) + + assert fe._model == "explicit-model" + assert fe._client is mock_openai_client + constructor.assert_not_called() + def test_constructor_with_openai_client(self) -> None: mock_client = MagicMock() fe = FoundryEvals(client=mock_client, model="gpt-4o")