feat(evals): add OpenEval dataset import and export helpers (fix #3549)

src/openai/types/evals/__init__.py

@@ -2,6 +2,12 @@
 
 from __future__ import annotations
 
+from .openeval import (
+    OpenEvalItem as OpenEvalItem,
+    OpenEvalGrader as OpenEvalGrader,
+    to_openeval as to_openeval,
+    from_openeval as from_openeval,
+)
 from .eval_api_error import EvalAPIError as EvalAPIError
 from .run_list_params import RunListParams as RunListParams
 from .run_create_params import RunCreateParams as RunCreateParams

src/openai/types/evals/openeval.py

@@ -0,0 +1,147 @@
+# File generated for OpenEval dataset import/export support.
+
+from __future__ import annotations
+
+import uuid
+from typing import Any, Dict, List, Union, Optional, cast
+from typing_extensions import Required, TypedDict
+
+__all__ = ["OpenEvalItem", "OpenEvalGrader", "from_openeval", "to_openeval"]
+
+
+class OpenEvalMessage(TypedDict, total=False):
+    role: str
+    content: str
+
+
+class OpenEvalGrader(TypedDict, total=False):
+    """Inline grader object -- see spec/schemas/grader.json. Most callers
+    will instead pass a bare grader-id string in OpenEvalItem['graders']."""
+
+    id: Required[str]
+    type: Required[str]
+    params: Dict[str, Any]
+
+
+class OpenEvalItem(TypedDict, total=False):
+    # id and graders are REQUIRED by spec/schemas/testcase.json
+    # ("required": ["id", "input", "graders"]) -- not optional.
+    id: Required[str]
+    input: Required[Union[str, List[str]]]
+    graders: Required[List[Union[str, OpenEvalGrader]]]
+    expected_output: Optional[str]
+    metadata: Optional[Dict[str, Any]]
+
+
+def from_openeval(item: OpenEvalItem) -> Dict[str, Any]:
+    """
+    Convert a spec-valid OpenEval TestCase into OpenAI Chat Completion
+    messages format.
+
+    ``item["input"]`` is a string or an array of strings per the EvalPort
+    TestCase schema, not an array of {role, content} objects -- that chat
+    shape is OpenAI's own native format, produced by to_openeval() below,
+    not something to_openeval()'s *caller* is expected to hand you.
+
+    If this item carries the original chat messages this adapter itself
+    exported (see to_openeval()), they're restored losslessly from
+    ``metadata["openai"]["messages"]``. Otherwise -- e.g. a hand-authored
+    TestCase, or one from a different tool -- every input string is
+    reconstructed as a single "user" message, the same fallback every other
+    adapter in the EvalPort ecosystem uses for a grader/tool type it doesn't
+    recognize.
+    """
+    metadata: Dict[str, Any] = {}
+    raw_metadata = item.get("metadata")
+    if isinstance(raw_metadata, dict):
+        metadata = {str(k): v for k, v in cast(Dict[Any, Any], raw_metadata).items()}
+
+    openai_meta: Dict[str, Any] = {}
+    raw_openai_meta = metadata.get("openai")
+    if isinstance(raw_openai_meta, dict):
+        openai_meta = {str(k): v for k, v in cast(Dict[Any, Any], raw_openai_meta).items()}
+
+    raw_saved_messages = openai_meta.get("messages")
+    if isinstance(raw_saved_messages, list):
+        messages: List[Dict[str, Any]] = []
+        for m in cast(List[Any], raw_saved_messages):
+            if isinstance(m, dict):
+                messages.append({str(k): v for k, v in cast(Dict[Any, Any], m).items()})
+            else:
+                messages.append({"role": "user", "content": str(m)})
+    else:
+        raw_input = item.get("input", "")
+        input_strings = [raw_input] if isinstance(raw_input, str) else list(raw_input)
+        messages = [{"role": "user", "content": s} for s in input_strings]
+
+    result: Dict[str, Any] = {"messages": messages}
+    if "id" in item and item["id"]:
+        result["id"] = item["id"]
+    expected_output = item.get("expected_output")
+    if expected_output is not None:
+        result["expected_output"] = expected_output
+    if metadata:
+        result["metadata"] = metadata
+    return result
+
+
+def to_openeval(
+    messages: List[Dict[str, Any]],
+    graders: List[Union[str, OpenEvalGrader]],
+    id: Optional[str] = None,
+    expected_output: Optional[str] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+) -> OpenEvalItem:
+    """
+    Export OpenAI Chat Completion messages into a spec-valid OpenEval
+    TestCase (spec/schemas/testcase.json).
+
+    Args:
+        messages: OpenAI chat messages ({role, content} dicts).
+        graders: REQUIRED -- TestCase.graders must have >=1 entry per spec.
+            Pass grader-id strings (referencing graders already defined on
+            the enclosing Suite) or inline grader objects.
+        id: TestCase.id is required by spec; if omitted, a uuid4 is
+            generated so the output always validates.
+        expected_output: Target reference output for the test case (if any).
+        metadata: Optional metadata dictionary.
+
+    ``input`` is built as one string per message ("{role}: {content}"),
+    satisfying the schema's string-or-array-of-strings requirement. The
+    *original* messages are additionally preserved verbatim under
+    metadata["openai"]["messages"] so from_openeval() can reconstruct the
+    exact role/content structure on import instead of collapsing everything
+    to "user" turns -- "openai.*" following the same reserved-namespace
+    convention every adapter in the EvalPort ecosystem uses ("openeval.*"
+    is the only prefix EvalPort itself reserves).
+    """
+    input_strings: List[str] = []
+    for msg in messages:
+        role = msg.get("role", "user")
+        content = msg.get("content", "")
+        if isinstance(content, str):
+            input_strings.append(f"{role}: {content}")
+        else:
+            input_strings.append(f"{role}: {content!s}")
+
+    merged_metadata: Dict[str, Any] = {}
+    if metadata is not None:
+        merged_metadata = {str(k): v for k, v in metadata.items()}
+
+    openai_meta: Dict[str, Any] = {}
+    raw_openai_meta = merged_metadata.get("openai")
+    if isinstance(raw_openai_meta, dict):
+        openai_meta = {str(k): v for k, v in cast(Dict[Any, Any], raw_openai_meta).items()}
+
+    openai_meta["messages"] = [{str(k): v for k, v in m.items()} for m in messages]
+    merged_metadata["openai"] = openai_meta
+
+    item: OpenEvalItem = {
+        "id": id or str(uuid.uuid4()),
+        "input": input_strings if len(input_strings) != 1 else input_strings[0],
+        "graders": graders,
+        "metadata": merged_metadata,
+    }
+    if expected_output is not None:
+        item["expected_output"] = expected_output
+    return item

tests/test_openeval.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+from openai.types.evals import OpenEvalItem, OpenEvalGrader, to_openeval, from_openeval
+
+
+def test_to_openeval_single_message() -> None:
+    messages = [{"role": "user", "content": "Hello"}]
+    exported = to_openeval(messages, graders=["grader-1"], id="eval-1", expected_output="World")
+    assert exported["id"] == "eval-1"
+    assert exported["input"] == "user: Hello"
+    assert exported["graders"] == ["grader-1"]
+    assert exported.get("expected_output") == "World"
+    metadata = exported.get("metadata")
+    assert isinstance(metadata, dict)
+    assert metadata.get("openai", {}).get("messages") == [{"role": "user", "content": "Hello"}]
+
+
+def test_to_openeval_multiple_messages() -> None:
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "What is 2+2?"},
+    ]
+    exported = to_openeval(messages, graders=["grader-1", "grader-2"], id="eval-2")
+    assert exported["id"] == "eval-2"
+    assert exported["input"] == ["system: You are a helpful assistant.", "user: What is 2+2?"]
+    assert exported["graders"] == ["grader-1", "grader-2"]
+    metadata = exported.get("metadata")
+    assert isinstance(metadata, dict)
+    assert metadata.get("openai", {}).get("messages") == messages
+
+
+def test_to_openeval_auto_generates_id() -> None:
+    messages = [{"role": "user", "content": "Hi"}]
+    exported = to_openeval(messages, graders=["grader-1"])
+    assert isinstance(exported["id"], str)
+    assert len(exported["id"]) > 0
+
+
+def test_to_openeval_with_inline_graders() -> None:
+    inline_grader: OpenEvalGrader = {
+        "id": "g-inline",
+        "type": "exact_match",
+        "params": {"case_sensitive": True},
+    }
+    messages = [{"role": "user", "content": "Test"}]
+    exported = to_openeval(messages, graders=[inline_grader], id="eval-inline")
+    assert exported["graders"] == [inline_grader]
+
+
+def test_to_openeval_preserves_empty_expected_output() -> None:
+    messages = [{"role": "user", "content": "Silence"}]
+    exported = to_openeval(messages, graders=["grader-1"], expected_output="")
+    assert exported.get("expected_output") == ""
+
+
+def test_from_openeval_with_openai_metadata_lossless() -> None:
+    messages = [
+        {"role": "system", "content": "Act as a calculator."},
+        {"role": "user", "content": [{"type": "text", "text": "Calculate this"}]},
+        {"role": "assistant", "content": None, "tool_calls": [{"id": "call_1", "type": "function"}]},
+        {"role": "tool", "tool_call_id": "call_1", "content": "42"},
+    ]
+    item: OpenEvalItem = {
+        "id": "eval-lossless",
+        "input": "system: Act as a calculator.",
+        "graders": ["grader-1"],
+        "expected_output": "42",
+        "metadata": {"custom": "meta", "openai": {"messages": messages}},
+    }
+    converted = from_openeval(item)
+    assert converted["id"] == "eval-lossless"
+    assert converted["messages"] == messages
+    assert converted["expected_output"] == "42"
+    assert converted["metadata"] == {"custom": "meta", "openai": {"messages": messages}}
+
+
+def test_from_openeval_scalar_input_fallback() -> None:
+    item: OpenEvalItem = {
+        "id": "eval-scalar",
+        "input": "What is 2+2?",
+        "graders": ["grader-1"],
+    }
+    converted = from_openeval(item)
+    assert converted["id"] == "eval-scalar"
+    assert converted["messages"] == [{"role": "user", "content": "What is 2+2?"}]
+    assert "expected_output" not in converted
+
+
+def test_from_openeval_list_input_fallback() -> None:
+    item: OpenEvalItem = {
+        "id": "eval-list",
+        "input": ["Turn 1", "Turn 2"],
+        "graders": ["grader-1"],
+        "expected_output": "",
+    }
+    converted = from_openeval(item)
+    assert converted["id"] == "eval-list"
+    assert converted["messages"] == [
+        {"role": "user", "content": "Turn 1"},
+        {"role": "user", "content": "Turn 2"},
+    ]
+    assert converted["expected_output"] == ""
+
+
+def test_round_trip_conversion() -> None:
+    original_messages = [
+        {"role": "user", "content": "Translate 'hello' to French."},
+    ]
+    exported = to_openeval(
+        original_messages,
+        graders=["exact-match-grader"],
+        id="round-trip-1",
+        expected_output="bonjour",
+        metadata={"source": "unit-test"},
+    )
+    imported = from_openeval(exported)
+    assert imported["id"] == "round-trip-1"
+    assert imported["messages"] == original_messages
+    assert imported["expected_output"] == "bonjour"
+    assert imported["metadata"]["source"] == "unit-test"