From f4151d77f4e6fd3baac86ea213d07377cfda372e Mon Sep 17 00:00:00 2001 From: Sparsh Garg <76697238+SparshGarg999@users.noreply.github.com> Date: Fri, 14 Aug 2026 22:49:15 +0530 Subject: [PATCH 1/2] feat(evals): add OpenEval dataset import and export helpers (#3549) --- src/openai/types/evals/__init__.py | 27 +++---------- src/openai/types/evals/openeval.py | 62 ++++++++++++++++++++++++++++++ tests/test_openeval.py | 23 +++++++++++ 3 files changed, 91 insertions(+), 21 deletions(-) create mode 100644 src/openai/types/evals/openeval.py create mode 100644 tests/test_openeval.py diff --git a/src/openai/types/evals/__init__.py b/src/openai/types/evals/__init__.py index 1c58a78067..c736674354 100644 --- a/src/openai/types/evals/__init__.py +++ b/src/openai/types/evals/__init__.py @@ -1,22 +1,7 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. - -from __future__ import annotations - -from .eval_api_error import EvalAPIError as EvalAPIError -from .run_list_params import RunListParams as RunListParams -from .run_create_params import RunCreateParams as RunCreateParams -from .run_list_response import RunListResponse as RunListResponse -from .run_cancel_response import RunCancelResponse as RunCancelResponse -from .run_create_response import RunCreateResponse as RunCreateResponse -from .run_delete_response import RunDeleteResponse as RunDeleteResponse -from .run_retrieve_response import RunRetrieveResponse as RunRetrieveResponse -from .create_eval_jsonl_run_data_source import CreateEvalJSONLRunDataSource as CreateEvalJSONLRunDataSource -from .create_eval_completions_run_data_source import ( - CreateEvalCompletionsRunDataSource as CreateEvalCompletionsRunDataSource, -) -from .create_eval_jsonl_run_data_source_param import ( - CreateEvalJSONLRunDataSourceParam as CreateEvalJSONLRunDataSourceParam, -) -from .create_eval_completions_run_data_source_param import ( - CreateEvalCompletionsRunDataSourceParam as CreateEvalCompletionsRunDataSourceParam, +from .openeval import ( + OpenEvalItem as OpenEvalItem, + from_openeval as from_openeval, + to_openeval as to_openeval, ) + +__all__ = ["OpenEvalItem", "from_openeval", "to_openeval"] diff --git a/src/openai/types/evals/openeval.py b/src/openai/types/evals/openeval.py new file mode 100644 index 0000000000..50cf6c0342 --- /dev/null +++ b/src/openai/types/evals/openeval.py @@ -0,0 +1,62 @@ +# File generated for OpenEval dataset import/export support. + +from typing import Any, Dict, List, Optional +from typing_extensions import TypedDict + +__all__ = ["OpenEvalItem", "from_openeval", "to_openeval"] + + +class OpenEvalMessage(TypedDict, total=False): + role: str + content: str + + +class OpenEvalItem(TypedDict, total=False): + id: Optional[str] + input: List[OpenEvalMessage] + expected_output: Optional[str] + metadata: Optional[Dict[str, Any]] + + +def from_openeval(item: OpenEvalItem) -> Dict[str, Any]: + """ + Convert an OpenEval dataset item into OpenAI Chat Completion messages format. + """ + messages: List[Dict[str, Any]] = [] + for msg in item.get("input", []): + messages.append({ + "role": msg.get("role", "user"), + "content": msg.get("content", ""), + }) + + result: Dict[str, Any] = {"messages": messages} + if item.get("id"): + result["id"] = item["id"] + if item.get("expected_output"): + result["expected_output"] = item["expected_output"] + if item.get("metadata"): + result["metadata"] = item["metadata"] + return result + + +def to_openeval( + messages: List[Dict[str, Any]], + id: Optional[str] = None, + expected_output: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, +) -> OpenEvalItem: + """ + Export OpenAI Chat Completion messages and metadata into OpenEval dataset item format. + """ + input_messages: List[OpenEvalMessage] = [ + {"role": str(msg.get("role", "user")), "content": str(msg.get("content", ""))} + for msg in messages + ] + item: OpenEvalItem = {"input": input_messages} + if id is not None: + item["id"] = id + if expected_output is not None: + item["expected_output"] = expected_output + if metadata is not None: + item["metadata"] = metadata + return item diff --git a/tests/test_openeval.py b/tests/test_openeval.py new file mode 100644 index 0000000000..690c69d3f1 --- /dev/null +++ b/tests/test_openeval.py @@ -0,0 +1,23 @@ +from openai.types.evals import OpenEvalItem, from_openeval, to_openeval + + +def test_from_openeval_conversion() -> None: + item: OpenEvalItem = { + "id": "eval-1", + "input": [{"role": "user", "content": "Hello world"}], + "expected_output": "Hi there!", + "metadata": {"task": "test"}, + } + converted = from_openeval(item) + assert converted["id"] == "eval-1" + assert converted["messages"] == [{"role": "user", "content": "Hello world"}] + assert converted["expected_output"] == "Hi there!" + assert converted["metadata"] == {"task": "test"} + + +def test_to_openeval_conversion() -> None: + messages = [{"role": "user", "content": "Hello"}] + exported = to_openeval(messages, id="eval-2", expected_output="World") + assert exported["id"] == "eval-2" + assert exported["input"] == [{"role": "user", "content": "Hello"}] + assert exported["expected_output"] == "World" From 2525c603f4c769fb611e3a1d2e72bd8593f9113a Mon Sep 17 00:00:00 2001 From: Sparsh Garg <76697238+SparshGarg999@users.noreply.github.com> Date: Sun, 16 Aug 2026 13:33:14 +0530 Subject: [PATCH 2/2] fix(evals): align OpenEvalItem schema with EvalPort spec and preserve generated evals exports --- src/openai/types/evals/__init__.py | 27 +++++- src/openai/types/evals/openeval.py | 139 +++++++++++++++++++++++------ tests/test_openeval.py | 129 ++++++++++++++++++++++---- 3 files changed, 249 insertions(+), 46 deletions(-) diff --git a/src/openai/types/evals/__init__.py b/src/openai/types/evals/__init__.py index c736674354..b55d75d146 100644 --- a/src/openai/types/evals/__init__.py +++ b/src/openai/types/evals/__init__.py @@ -1,7 +1,28 @@ +# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. + +from __future__ import annotations + from .openeval import ( OpenEvalItem as OpenEvalItem, - from_openeval as from_openeval, + OpenEvalGrader as OpenEvalGrader, to_openeval as to_openeval, + from_openeval as from_openeval, +) +from .eval_api_error import EvalAPIError as EvalAPIError +from .run_list_params import RunListParams as RunListParams +from .run_create_params import RunCreateParams as RunCreateParams +from .run_list_response import RunListResponse as RunListResponse +from .run_cancel_response import RunCancelResponse as RunCancelResponse +from .run_create_response import RunCreateResponse as RunCreateResponse +from .run_delete_response import RunDeleteResponse as RunDeleteResponse +from .run_retrieve_response import RunRetrieveResponse as RunRetrieveResponse +from .create_eval_jsonl_run_data_source import CreateEvalJSONLRunDataSource as CreateEvalJSONLRunDataSource +from .create_eval_completions_run_data_source import ( + CreateEvalCompletionsRunDataSource as CreateEvalCompletionsRunDataSource, +) +from .create_eval_jsonl_run_data_source_param import ( + CreateEvalJSONLRunDataSourceParam as CreateEvalJSONLRunDataSourceParam, +) +from .create_eval_completions_run_data_source_param import ( + CreateEvalCompletionsRunDataSourceParam as CreateEvalCompletionsRunDataSourceParam, ) - -__all__ = ["OpenEvalItem", "from_openeval", "to_openeval"] diff --git a/src/openai/types/evals/openeval.py b/src/openai/types/evals/openeval.py index 50cf6c0342..580c8b6948 100644 --- a/src/openai/types/evals/openeval.py +++ b/src/openai/types/evals/openeval.py @@ -1,9 +1,12 @@ # File generated for OpenEval dataset import/export support. -from typing import Any, Dict, List, Optional -from typing_extensions import TypedDict +from __future__ import annotations -__all__ = ["OpenEvalItem", "from_openeval", "to_openeval"] +import uuid +from typing import Any, Dict, List, Union, Optional, cast +from typing_extensions import Required, TypedDict + +__all__ = ["OpenEvalItem", "OpenEvalGrader", "from_openeval", "to_openeval"] class OpenEvalMessage(TypedDict, total=False): @@ -11,52 +14,134 @@ class OpenEvalMessage(TypedDict, total=False): content: str +class OpenEvalGrader(TypedDict, total=False): + """Inline grader object -- see spec/schemas/grader.json. Most callers + will instead pass a bare grader-id string in OpenEvalItem['graders'].""" + + id: Required[str] + type: Required[str] + params: Dict[str, Any] + + class OpenEvalItem(TypedDict, total=False): - id: Optional[str] - input: List[OpenEvalMessage] + # id and graders are REQUIRED by spec/schemas/testcase.json + # ("required": ["id", "input", "graders"]) -- not optional. + id: Required[str] + input: Required[Union[str, List[str]]] + graders: Required[List[Union[str, OpenEvalGrader]]] expected_output: Optional[str] metadata: Optional[Dict[str, Any]] def from_openeval(item: OpenEvalItem) -> Dict[str, Any]: """ - Convert an OpenEval dataset item into OpenAI Chat Completion messages format. + Convert a spec-valid OpenEval TestCase into OpenAI Chat Completion + messages format. + + ``item["input"]`` is a string or an array of strings per the EvalPort + TestCase schema, not an array of {role, content} objects -- that chat + shape is OpenAI's own native format, produced by to_openeval() below, + not something to_openeval()'s *caller* is expected to hand you. + + If this item carries the original chat messages this adapter itself + exported (see to_openeval()), they're restored losslessly from + ``metadata["openai"]["messages"]``. Otherwise -- e.g. a hand-authored + TestCase, or one from a different tool -- every input string is + reconstructed as a single "user" message, the same fallback every other + adapter in the EvalPort ecosystem uses for a grader/tool type it doesn't + recognize. """ - messages: List[Dict[str, Any]] = [] - for msg in item.get("input", []): - messages.append({ - "role": msg.get("role", "user"), - "content": msg.get("content", ""), - }) + metadata: Dict[str, Any] = {} + raw_metadata = item.get("metadata") + if isinstance(raw_metadata, dict): + metadata = {str(k): v for k, v in cast(Dict[Any, Any], raw_metadata).items()} + + openai_meta: Dict[str, Any] = {} + raw_openai_meta = metadata.get("openai") + if isinstance(raw_openai_meta, dict): + openai_meta = {str(k): v for k, v in cast(Dict[Any, Any], raw_openai_meta).items()} + + raw_saved_messages = openai_meta.get("messages") + if isinstance(raw_saved_messages, list): + messages: List[Dict[str, Any]] = [] + for m in cast(List[Any], raw_saved_messages): + if isinstance(m, dict): + messages.append({str(k): v for k, v in cast(Dict[Any, Any], m).items()}) + else: + messages.append({"role": "user", "content": str(m)}) + else: + raw_input = item.get("input", "") + input_strings = [raw_input] if isinstance(raw_input, str) else list(raw_input) + messages = [{"role": "user", "content": s} for s in input_strings] result: Dict[str, Any] = {"messages": messages} - if item.get("id"): + if "id" in item and item["id"]: result["id"] = item["id"] - if item.get("expected_output"): - result["expected_output"] = item["expected_output"] - if item.get("metadata"): - result["metadata"] = item["metadata"] + expected_output = item.get("expected_output") + if expected_output is not None: + result["expected_output"] = expected_output + if metadata: + result["metadata"] = metadata return result def to_openeval( messages: List[Dict[str, Any]], + graders: List[Union[str, OpenEvalGrader]], id: Optional[str] = None, expected_output: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None, ) -> OpenEvalItem: """ - Export OpenAI Chat Completion messages and metadata into OpenEval dataset item format. + Export OpenAI Chat Completion messages into a spec-valid OpenEval + TestCase (spec/schemas/testcase.json). + + Args: + messages: OpenAI chat messages ({role, content} dicts). + graders: REQUIRED -- TestCase.graders must have >=1 entry per spec. + Pass grader-id strings (referencing graders already defined on + the enclosing Suite) or inline grader objects. + id: TestCase.id is required by spec; if omitted, a uuid4 is + generated so the output always validates. + expected_output: Target reference output for the test case (if any). + metadata: Optional metadata dictionary. + + ``input`` is built as one string per message ("{role}: {content}"), + satisfying the schema's string-or-array-of-strings requirement. The + *original* messages are additionally preserved verbatim under + metadata["openai"]["messages"] so from_openeval() can reconstruct the + exact role/content structure on import instead of collapsing everything + to "user" turns -- "openai.*" following the same reserved-namespace + convention every adapter in the EvalPort ecosystem uses ("openeval.*" + is the only prefix EvalPort itself reserves). """ - input_messages: List[OpenEvalMessage] = [ - {"role": str(msg.get("role", "user")), "content": str(msg.get("content", ""))} - for msg in messages - ] - item: OpenEvalItem = {"input": input_messages} - if id is not None: - item["id"] = id + input_strings: List[str] = [] + for msg in messages: + role = msg.get("role", "user") + content = msg.get("content", "") + if isinstance(content, str): + input_strings.append(f"{role}: {content}") + else: + input_strings.append(f"{role}: {content!s}") + + merged_metadata: Dict[str, Any] = {} + if metadata is not None: + merged_metadata = {str(k): v for k, v in metadata.items()} + + openai_meta: Dict[str, Any] = {} + raw_openai_meta = merged_metadata.get("openai") + if isinstance(raw_openai_meta, dict): + openai_meta = {str(k): v for k, v in cast(Dict[Any, Any], raw_openai_meta).items()} + + openai_meta["messages"] = [{str(k): v for k, v in m.items()} for m in messages] + merged_metadata["openai"] = openai_meta + + item: OpenEvalItem = { + "id": id or str(uuid.uuid4()), + "input": input_strings if len(input_strings) != 1 else input_strings[0], + "graders": graders, + "metadata": merged_metadata, + } if expected_output is not None: item["expected_output"] = expected_output - if metadata is not None: - item["metadata"] = metadata return item diff --git a/tests/test_openeval.py b/tests/test_openeval.py index 690c69d3f1..01911887b5 100644 --- a/tests/test_openeval.py +++ b/tests/test_openeval.py @@ -1,23 +1,120 @@ -from openai.types.evals import OpenEvalItem, from_openeval, to_openeval +from __future__ import annotations +from openai.types.evals import OpenEvalItem, OpenEvalGrader, to_openeval, from_openeval -def test_from_openeval_conversion() -> None: + +def test_to_openeval_single_message() -> None: + messages = [{"role": "user", "content": "Hello"}] + exported = to_openeval(messages, graders=["grader-1"], id="eval-1", expected_output="World") + assert exported["id"] == "eval-1" + assert exported["input"] == "user: Hello" + assert exported["graders"] == ["grader-1"] + assert exported.get("expected_output") == "World" + metadata = exported.get("metadata") + assert isinstance(metadata, dict) + assert metadata.get("openai", {}).get("messages") == [{"role": "user", "content": "Hello"}] + + +def test_to_openeval_multiple_messages() -> None: + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is 2+2?"}, + ] + exported = to_openeval(messages, graders=["grader-1", "grader-2"], id="eval-2") + assert exported["id"] == "eval-2" + assert exported["input"] == ["system: You are a helpful assistant.", "user: What is 2+2?"] + assert exported["graders"] == ["grader-1", "grader-2"] + metadata = exported.get("metadata") + assert isinstance(metadata, dict) + assert metadata.get("openai", {}).get("messages") == messages + + +def test_to_openeval_auto_generates_id() -> None: + messages = [{"role": "user", "content": "Hi"}] + exported = to_openeval(messages, graders=["grader-1"]) + assert isinstance(exported["id"], str) + assert len(exported["id"]) > 0 + + +def test_to_openeval_with_inline_graders() -> None: + inline_grader: OpenEvalGrader = { + "id": "g-inline", + "type": "exact_match", + "params": {"case_sensitive": True}, + } + messages = [{"role": "user", "content": "Test"}] + exported = to_openeval(messages, graders=[inline_grader], id="eval-inline") + assert exported["graders"] == [inline_grader] + + +def test_to_openeval_preserves_empty_expected_output() -> None: + messages = [{"role": "user", "content": "Silence"}] + exported = to_openeval(messages, graders=["grader-1"], expected_output="") + assert exported.get("expected_output") == "" + + +def test_from_openeval_with_openai_metadata_lossless() -> None: + messages = [ + {"role": "system", "content": "Act as a calculator."}, + {"role": "user", "content": [{"type": "text", "text": "Calculate this"}]}, + {"role": "assistant", "content": None, "tool_calls": [{"id": "call_1", "type": "function"}]}, + {"role": "tool", "tool_call_id": "call_1", "content": "42"}, + ] item: OpenEvalItem = { - "id": "eval-1", - "input": [{"role": "user", "content": "Hello world"}], - "expected_output": "Hi there!", - "metadata": {"task": "test"}, + "id": "eval-lossless", + "input": "system: Act as a calculator.", + "graders": ["grader-1"], + "expected_output": "42", + "metadata": {"custom": "meta", "openai": {"messages": messages}}, } converted = from_openeval(item) - assert converted["id"] == "eval-1" - assert converted["messages"] == [{"role": "user", "content": "Hello world"}] - assert converted["expected_output"] == "Hi there!" - assert converted["metadata"] == {"task": "test"} + assert converted["id"] == "eval-lossless" + assert converted["messages"] == messages + assert converted["expected_output"] == "42" + assert converted["metadata"] == {"custom": "meta", "openai": {"messages": messages}} -def test_to_openeval_conversion() -> None: - messages = [{"role": "user", "content": "Hello"}] - exported = to_openeval(messages, id="eval-2", expected_output="World") - assert exported["id"] == "eval-2" - assert exported["input"] == [{"role": "user", "content": "Hello"}] - assert exported["expected_output"] == "World" +def test_from_openeval_scalar_input_fallback() -> None: + item: OpenEvalItem = { + "id": "eval-scalar", + "input": "What is 2+2?", + "graders": ["grader-1"], + } + converted = from_openeval(item) + assert converted["id"] == "eval-scalar" + assert converted["messages"] == [{"role": "user", "content": "What is 2+2?"}] + assert "expected_output" not in converted + + +def test_from_openeval_list_input_fallback() -> None: + item: OpenEvalItem = { + "id": "eval-list", + "input": ["Turn 1", "Turn 2"], + "graders": ["grader-1"], + "expected_output": "", + } + converted = from_openeval(item) + assert converted["id"] == "eval-list" + assert converted["messages"] == [ + {"role": "user", "content": "Turn 1"}, + {"role": "user", "content": "Turn 2"}, + ] + assert converted["expected_output"] == "" + + +def test_round_trip_conversion() -> None: + original_messages = [ + {"role": "user", "content": "Translate 'hello' to French."}, + ] + exported = to_openeval( + original_messages, + graders=["exact-match-grader"], + id="round-trip-1", + expected_output="bonjour", + metadata={"source": "unit-test"}, + ) + imported = from_openeval(exported) + assert imported["id"] == "round-trip-1" + assert imported["messages"] == original_messages + assert imported["expected_output"] == "bonjour" + assert imported["metadata"]["source"] == "unit-test"