diff --git a/src/openai/types/evals/__init__.py b/src/openai/types/evals/__init__.py index 1c58a78067..b55d75d146 100644 --- a/src/openai/types/evals/__init__.py +++ b/src/openai/types/evals/__init__.py @@ -2,6 +2,12 @@ from __future__ import annotations +from .openeval import ( + OpenEvalItem as OpenEvalItem, + OpenEvalGrader as OpenEvalGrader, + to_openeval as to_openeval, + from_openeval as from_openeval, +) from .eval_api_error import EvalAPIError as EvalAPIError from .run_list_params import RunListParams as RunListParams from .run_create_params import RunCreateParams as RunCreateParams diff --git a/src/openai/types/evals/openeval.py b/src/openai/types/evals/openeval.py new file mode 100644 index 0000000000..580c8b6948 --- /dev/null +++ b/src/openai/types/evals/openeval.py @@ -0,0 +1,147 @@ +# File generated for OpenEval dataset import/export support. + +from __future__ import annotations + +import uuid +from typing import Any, Dict, List, Union, Optional, cast +from typing_extensions import Required, TypedDict + +__all__ = ["OpenEvalItem", "OpenEvalGrader", "from_openeval", "to_openeval"] + + +class OpenEvalMessage(TypedDict, total=False): + role: str + content: str + + +class OpenEvalGrader(TypedDict, total=False): + """Inline grader object -- see spec/schemas/grader.json. Most callers + will instead pass a bare grader-id string in OpenEvalItem['graders'].""" + + id: Required[str] + type: Required[str] + params: Dict[str, Any] + + +class OpenEvalItem(TypedDict, total=False): + # id and graders are REQUIRED by spec/schemas/testcase.json + # ("required": ["id", "input", "graders"]) -- not optional. + id: Required[str] + input: Required[Union[str, List[str]]] + graders: Required[List[Union[str, OpenEvalGrader]]] + expected_output: Optional[str] + metadata: Optional[Dict[str, Any]] + + +def from_openeval(item: OpenEvalItem) -> Dict[str, Any]: + """ + Convert a spec-valid OpenEval TestCase into OpenAI Chat Completion + messages format. + + ``item["input"]`` is a string or an array of strings per the EvalPort + TestCase schema, not an array of {role, content} objects -- that chat + shape is OpenAI's own native format, produced by to_openeval() below, + not something to_openeval()'s *caller* is expected to hand you. + + If this item carries the original chat messages this adapter itself + exported (see to_openeval()), they're restored losslessly from + ``metadata["openai"]["messages"]``. Otherwise -- e.g. a hand-authored + TestCase, or one from a different tool -- every input string is + reconstructed as a single "user" message, the same fallback every other + adapter in the EvalPort ecosystem uses for a grader/tool type it doesn't + recognize. + """ + metadata: Dict[str, Any] = {} + raw_metadata = item.get("metadata") + if isinstance(raw_metadata, dict): + metadata = {str(k): v for k, v in cast(Dict[Any, Any], raw_metadata).items()} + + openai_meta: Dict[str, Any] = {} + raw_openai_meta = metadata.get("openai") + if isinstance(raw_openai_meta, dict): + openai_meta = {str(k): v for k, v in cast(Dict[Any, Any], raw_openai_meta).items()} + + raw_saved_messages = openai_meta.get("messages") + if isinstance(raw_saved_messages, list): + messages: List[Dict[str, Any]] = [] + for m in cast(List[Any], raw_saved_messages): + if isinstance(m, dict): + messages.append({str(k): v for k, v in cast(Dict[Any, Any], m).items()}) + else: + messages.append({"role": "user", "content": str(m)}) + else: + raw_input = item.get("input", "") + input_strings = [raw_input] if isinstance(raw_input, str) else list(raw_input) + messages = [{"role": "user", "content": s} for s in input_strings] + + result: Dict[str, Any] = {"messages": messages} + if "id" in item and item["id"]: + result["id"] = item["id"] + expected_output = item.get("expected_output") + if expected_output is not None: + result["expected_output"] = expected_output + if metadata: + result["metadata"] = metadata + return result + + +def to_openeval( + messages: List[Dict[str, Any]], + graders: List[Union[str, OpenEvalGrader]], + id: Optional[str] = None, + expected_output: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, +) -> OpenEvalItem: + """ + Export OpenAI Chat Completion messages into a spec-valid OpenEval + TestCase (spec/schemas/testcase.json). + + Args: + messages: OpenAI chat messages ({role, content} dicts). + graders: REQUIRED -- TestCase.graders must have >=1 entry per spec. + Pass grader-id strings (referencing graders already defined on + the enclosing Suite) or inline grader objects. + id: TestCase.id is required by spec; if omitted, a uuid4 is + generated so the output always validates. + expected_output: Target reference output for the test case (if any). + metadata: Optional metadata dictionary. + + ``input`` is built as one string per message ("{role}: {content}"), + satisfying the schema's string-or-array-of-strings requirement. The + *original* messages are additionally preserved verbatim under + metadata["openai"]["messages"] so from_openeval() can reconstruct the + exact role/content structure on import instead of collapsing everything + to "user" turns -- "openai.*" following the same reserved-namespace + convention every adapter in the EvalPort ecosystem uses ("openeval.*" + is the only prefix EvalPort itself reserves). + """ + input_strings: List[str] = [] + for msg in messages: + role = msg.get("role", "user") + content = msg.get("content", "") + if isinstance(content, str): + input_strings.append(f"{role}: {content}") + else: + input_strings.append(f"{role}: {content!s}") + + merged_metadata: Dict[str, Any] = {} + if metadata is not None: + merged_metadata = {str(k): v for k, v in metadata.items()} + + openai_meta: Dict[str, Any] = {} + raw_openai_meta = merged_metadata.get("openai") + if isinstance(raw_openai_meta, dict): + openai_meta = {str(k): v for k, v in cast(Dict[Any, Any], raw_openai_meta).items()} + + openai_meta["messages"] = [{str(k): v for k, v in m.items()} for m in messages] + merged_metadata["openai"] = openai_meta + + item: OpenEvalItem = { + "id": id or str(uuid.uuid4()), + "input": input_strings if len(input_strings) != 1 else input_strings[0], + "graders": graders, + "metadata": merged_metadata, + } + if expected_output is not None: + item["expected_output"] = expected_output + return item diff --git a/tests/test_openeval.py b/tests/test_openeval.py new file mode 100644 index 0000000000..01911887b5 --- /dev/null +++ b/tests/test_openeval.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +from openai.types.evals import OpenEvalItem, OpenEvalGrader, to_openeval, from_openeval + + +def test_to_openeval_single_message() -> None: + messages = [{"role": "user", "content": "Hello"}] + exported = to_openeval(messages, graders=["grader-1"], id="eval-1", expected_output="World") + assert exported["id"] == "eval-1" + assert exported["input"] == "user: Hello" + assert exported["graders"] == ["grader-1"] + assert exported.get("expected_output") == "World" + metadata = exported.get("metadata") + assert isinstance(metadata, dict) + assert metadata.get("openai", {}).get("messages") == [{"role": "user", "content": "Hello"}] + + +def test_to_openeval_multiple_messages() -> None: + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is 2+2?"}, + ] + exported = to_openeval(messages, graders=["grader-1", "grader-2"], id="eval-2") + assert exported["id"] == "eval-2" + assert exported["input"] == ["system: You are a helpful assistant.", "user: What is 2+2?"] + assert exported["graders"] == ["grader-1", "grader-2"] + metadata = exported.get("metadata") + assert isinstance(metadata, dict) + assert metadata.get("openai", {}).get("messages") == messages + + +def test_to_openeval_auto_generates_id() -> None: + messages = [{"role": "user", "content": "Hi"}] + exported = to_openeval(messages, graders=["grader-1"]) + assert isinstance(exported["id"], str) + assert len(exported["id"]) > 0 + + +def test_to_openeval_with_inline_graders() -> None: + inline_grader: OpenEvalGrader = { + "id": "g-inline", + "type": "exact_match", + "params": {"case_sensitive": True}, + } + messages = [{"role": "user", "content": "Test"}] + exported = to_openeval(messages, graders=[inline_grader], id="eval-inline") + assert exported["graders"] == [inline_grader] + + +def test_to_openeval_preserves_empty_expected_output() -> None: + messages = [{"role": "user", "content": "Silence"}] + exported = to_openeval(messages, graders=["grader-1"], expected_output="") + assert exported.get("expected_output") == "" + + +def test_from_openeval_with_openai_metadata_lossless() -> None: + messages = [ + {"role": "system", "content": "Act as a calculator."}, + {"role": "user", "content": [{"type": "text", "text": "Calculate this"}]}, + {"role": "assistant", "content": None, "tool_calls": [{"id": "call_1", "type": "function"}]}, + {"role": "tool", "tool_call_id": "call_1", "content": "42"}, + ] + item: OpenEvalItem = { + "id": "eval-lossless", + "input": "system: Act as a calculator.", + "graders": ["grader-1"], + "expected_output": "42", + "metadata": {"custom": "meta", "openai": {"messages": messages}}, + } + converted = from_openeval(item) + assert converted["id"] == "eval-lossless" + assert converted["messages"] == messages + assert converted["expected_output"] == "42" + assert converted["metadata"] == {"custom": "meta", "openai": {"messages": messages}} + + +def test_from_openeval_scalar_input_fallback() -> None: + item: OpenEvalItem = { + "id": "eval-scalar", + "input": "What is 2+2?", + "graders": ["grader-1"], + } + converted = from_openeval(item) + assert converted["id"] == "eval-scalar" + assert converted["messages"] == [{"role": "user", "content": "What is 2+2?"}] + assert "expected_output" not in converted + + +def test_from_openeval_list_input_fallback() -> None: + item: OpenEvalItem = { + "id": "eval-list", + "input": ["Turn 1", "Turn 2"], + "graders": ["grader-1"], + "expected_output": "", + } + converted = from_openeval(item) + assert converted["id"] == "eval-list" + assert converted["messages"] == [ + {"role": "user", "content": "Turn 1"}, + {"role": "user", "content": "Turn 2"}, + ] + assert converted["expected_output"] == "" + + +def test_round_trip_conversion() -> None: + original_messages = [ + {"role": "user", "content": "Translate 'hello' to French."}, + ] + exported = to_openeval( + original_messages, + graders=["exact-match-grader"], + id="round-trip-1", + expected_output="bonjour", + metadata={"source": "unit-test"}, + ) + imported = from_openeval(exported) + assert imported["id"] == "round-trip-1" + assert imported["messages"] == original_messages + assert imported["expected_output"] == "bonjour" + assert imported["metadata"]["source"] == "unit-test"