From 39fe79b630a3eee42b4b3f07f036b5c02281887c Mon Sep 17 00:00:00 2001 From: Robert Gruen Date: Mon, 12 Oct 2026 18:33:57 -0700 Subject: [PATCH] Retain rejected output in TypeScript and Python failures Preserve unvalidated candidates, raw response text, custom failure metadata, and repair context without changing success results or repair limits. Add regression coverage and document the additive failure payload APIs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- python/README.md | 31 ++ python/src/typechat/_internal/result.py | 21 +- python/src/typechat/_internal/translator.py | 38 ++- python/src/typechat/_internal/validator.py | 8 +- python/tests/test_failure.py | 243 +++++++++++++++ site/src/docs/python/basic-usage.md | 28 +- site/src/docs/typescript/basic-usage.md | 29 +- typescript/README.md | 28 ++ typescript/package.json | 2 +- typescript/src/result.ts | 30 +- typescript/src/ts/program.ts | 4 +- typescript/src/ts/validate.ts | 13 +- typescript/src/typechat.ts | 35 ++- typescript/src/zod/validate.ts | 2 +- typescript/test/failure.test.ts | 323 ++++++++++++++++++++ 15 files changed, 801 insertions(+), 34 deletions(-) create mode 100644 python/tests/test_failure.py create mode 100644 typescript/test/failure.test.ts diff --git a/python/README.md b/python/README.md index 11571366..7ee7bca5 100644 --- a/python/README.md +++ b/python/README.md @@ -50,6 +50,37 @@ To learn more about TypeChat, visit the [documentation](https://microsoft.github.io/TypeChat/docs/python/basic-usage/) which includes more information on TypeChat and how to get started. +## Accessing failed output + +`Failure` retains its existing `message` and can also carry an unvalidated `value: object` and +`response_text: str | None`. Validators return the rejected input in `value`; translators also retain +the original model text, including when it is not valid JSON: + +```python +from typechat import Failure + +result = await translator.translate(request) +if isinstance(result, Failure): + if result.has_value: + rejected = result.value + # Validate or repair this value before treating it as the target type. + original_response = result.response_text +``` + +Check `has_value` before using `value`. It distinguishes unavailable data from a rejected `None`, +`False`, `0`, or empty string. An unavailable value is represented by an internal missing-value marker, +not `None`. Custom validators can supply their own payloads using +`Failure(message, value=rejected, response_text=text)`, and the translator preserves those payloads. +`Failure(message)` remains valid, and payload fields are omitted from its representation. +Use `isinstance` with `Success` or `Failure` to distinguish outcomes, not the presence of `value`. + +On final failure, automatic repair retains the latest rejected candidate and available response text, +not a full attempt history. If a repair response cannot be parsed, `value` can come from an earlier +attempt than `response_text`. An initial network failure has no candidate value. Successful results, +error messages, and repair limits are unchanged, including Python's existing repair of malformed JSON. + +Rejected values and response text may contain sensitive output. Do not log or serialize them blindly. + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/python/src/typechat/_internal/result.py b/python/src/typechat/_internal/result.py index a9578cec..92029967 100644 --- a/python/src/typechat/_internal/result.py +++ b/python/src/typechat/_internal/result.py @@ -1,8 +1,12 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field +from enum import Enum from typing_extensions import Generic, TypeAlias, TypeVar T = TypeVar("T", covariant=True) +class _MissingValue(Enum): + MISSING = 0 + @dataclass class Success(Generic[T]): "An object representing a successful operation with a result of type `T`." @@ -11,8 +15,21 @@ class Success(Generic[T]): @dataclass class Failure: - "An object representing an operation that failed for the reason given in `message`." + """ + An operation that failed for the reason given in `message`. + + `value` is unvalidated and is available only when `has_value` is true. `response_text` contains + the latest model response, when available. If a repair response couldn't be parsed, `value` + may refer to a rejected candidate from an earlier attempt. + """ message: str + value: object = field(default=_MissingValue.MISSING, kw_only=True, repr=False) + response_text: str | None = field(default=None, kw_only=True, repr=False) + + @property + def has_value(self) -> bool: + """Distinguishes missing data from an explicitly supplied value, including `None`.""" + return self.value is not _MissingValue.MISSING """ diff --git a/python/src/typechat/_internal/translator.py b/python/src/typechat/_internal/translator.py index ee077195..db44b4b2 100644 --- a/python/src/typechat/_internal/translator.py +++ b/python/src/typechat/_internal/translator.py @@ -1,3 +1,4 @@ +from copy import copy from typing_extensions import Generic, TypeVar import pydantic_core @@ -55,6 +56,8 @@ async def translate(self, input: str, *, prompt_preamble: str | list[PromptSecti the language model fails to validate, repair attempts will be made up until `_max_repair_attempts`. The prompt for the subsequent attempts will include the diagnostics produced for the prior attempt. This often helps produce a valid instance. + A failure retains the latest rejected value and available model response text. These may come + from different attempts if a repair response couldn't be parsed. Args: input: A natural language request. @@ -72,33 +75,54 @@ async def translate(self, input: str, *, prompt_preamble: str | list[PromptSecti messages.append({"role": "user", "content": self._create_request_prompt(input)}) num_repairs_attempted = 0 + last_failure: Failure | None = None while True: completion_response = await self.model.complete(messages) if isinstance(completion_response, Failure): - return completion_response + if last_failure is None: + return completion_response + failure = copy(completion_response) + if not failure.has_value: + failure.value = last_failure.value + if failure.response_text is None: + failure.response_text = last_failure.response_text + return failure text_response = completion_response.value first_curly = text_response.find("{") last_curly = text_response.rfind("}") + 1 - error_message: str + failure: Failure if 0 <= first_curly < last_curly: trimmed_response = text_response[first_curly:last_curly] try: parsed_response = pydantic_core.from_json(trimmed_response, allow_inf_nan=False, cache_strings=False) except ValueError as e: - error_message = f"Error: {e}\n\nAttempted to parse:\n\n{trimmed_response}" + failure = Failure( + f"Error: {e}\n\nAttempted to parse:\n\n{trimmed_response}", + response_text=text_response, + ) else: result = self.validator.validate_object(parsed_response) if isinstance(result, Success): return result - error_message = result.message + failure = copy(result) + if failure.response_text is None: + failure.response_text = text_response + if not failure.has_value: + failure.value = parsed_response else: - error_message = f"Response did not contain any text resembling JSON.\nResponse was\n\n{text_response}" + failure = Failure( + f"Response did not contain any text resembling JSON.\nResponse was\n\n{text_response}", + response_text=text_response, + ) + if not failure.has_value and last_failure is not None: + failure.value = last_failure.value + last_failure = failure if num_repairs_attempted >= self._max_repair_attempts: - return Failure(error_message) + return failure num_repairs_attempted += 1 messages.append({"role": "assistant", "content": text_response}) - messages.append({"role": "user", "content": self._create_repair_prompt(error_message)}) + messages.append({"role": "user", "content": self._create_repair_prompt(failure.message)}) def _create_request_prompt(self, intent: str) -> str: prompt = f""" diff --git a/python/src/typechat/_internal/validator.py b/python/src/typechat/_internal/validator.py index a1d17f1d..5ed87512 100644 --- a/python/src/typechat/_internal/validator.py +++ b/python/src/typechat/_internal/validator.py @@ -29,7 +29,7 @@ def validate_object(self, obj: object) -> Result[T]: Validates the given Python object according to the associated schema type. Returns a `Success[T]` object containing the object if validation was successful. - Otherwise, returns a `Failure` object with a `message` property describing the error. + Otherwise, returns a `Failure` with an error message and the rejected object in `value`. """ try: # TODO: Switch to `validate_python` when validation modes are exposed. @@ -42,10 +42,10 @@ def validate_object(self, obj: object) -> Result[T]: typed_dict = self._adapted_type.validate_json(json_str, strict=True) return Success(typed_dict) except pydantic.ValidationError as validation_error: - return _handle_error(validation_error) + return _handle_error(validation_error, obj) -def _handle_error(validation_error: pydantic.ValidationError) -> Failure: +def _handle_error(validation_error: pydantic.ValidationError, value: object) -> Failure: error_strings: list[str] = [] for error in validation_error.errors(include_url=False): error_string = "" @@ -64,4 +64,4 @@ def _handle_error(validation_error: pydantic.ValidationError) -> Failure: failure_message = "" failure_message += "\n".join(error_strings) - return Failure(failure_message) + return Failure(failure_message, value=value) diff --git a/python/tests/test_failure.py b/python/tests/test_failure.py new file mode 100644 index 00000000..86c283c8 --- /dev/null +++ b/python/tests/test_failure.py @@ -0,0 +1,243 @@ +import asyncio +from copy import deepcopy +from dataclasses import dataclass + +import pytest +from typing_extensions import assert_type, override + +from typechat import ( + Failure, + PromptSection, + Result, + Success, + TypeChatJsonTranslator, + TypeChatLanguageModel, + TypeChatValidator, +) + + +@dataclass +class Example: + value: int + + +class FixedResultModel(TypeChatLanguageModel): + def __init__(self, responses: list[Result[str]]) -> None: + super().__init__() + self._responses = iter(responses) + self.prompts: list[str | list[PromptSection]] = [] + + @override + async def complete(self, prompt: str | list[PromptSection]) -> Result[str]: + self.prompts.append(prompt.copy() if isinstance(prompt, list) else prompt) + return next(self._responses) + + +class NoRepairTranslator(TypeChatJsonTranslator[Example]): + _max_repair_attempts = 0 + + +class RejectingValidator(TypeChatValidator[Example]): + def __init__(self, failure: Failure) -> None: + super().__init__(Example) + self.failure = failure + + @override + def validate_object(self, obj: object) -> Result[Example]: + return self.failure + + +class CustomFailure(Failure): + def __init__(self, message: str, code: str) -> None: + super().__init__(message) + self.code = code + + +def translator_for( + responses: list[Result[str]], + validator: TypeChatValidator[Example] | None = None, + *, + repair: bool = True, +) -> tuple[TypeChatJsonTranslator[Example], FixedResultModel]: + model = FixedResultModel(responses) + validator = validator if validator is not None else TypeChatValidator(Example) + if repair: + translator = TypeChatJsonTranslator(model, validator, Example) + else: + translator = NoRepairTranslator(model, validator, Example) + return translator, model + + +def test_message_only_failure_has_no_value(): + result = Failure("failed") + assert not result.has_value + assert result.response_text is None + assert repr(result) == "Failure(message='failed')" + assert not deepcopy(result).has_value + + +@pytest.mark.parametrize("value", [None, False, 0, "", {}, []]) +def test_failure_retains_explicit_values(value: object): + result = Failure("failed", value=value, response_text="") + assert_type(result.value, object) + assert result.has_value + assert result.value is value + assert result.response_text == "" + assert "response_text" not in repr(result) + + +@pytest.mark.parametrize("value", [{"value": "invalid"}, None, False, 0, ""]) +def test_validator_retains_the_entire_rejected_value(value: object): + result = TypeChatValidator(Example).validate_object(value) + assert isinstance(result, Failure) + assert result.has_value + assert result.value is value + assert result.response_text is None + + +def test_translator_preserves_success(): + translator, model = translator_for([Success('{"value":42}')]) + result = asyncio.run(translator.translate("request")) + assert result == Success(Example(42)) + assert len(model.prompts) == 1 + + +def test_translator_retains_rejected_data_without_repair(): + response_text = 'before {"value":"invalid"} after' + translator, model = translator_for([Success(response_text)], repair=False) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, Failure) + assert result.has_value + assert result.value == {"value": "invalid"} + assert result.response_text == response_text + assert "failed for value" in result.message + assert len(model.prompts) == 1 + + +@pytest.mark.parametrize("value", [None, False, 0, ""]) +def test_translator_preserves_custom_failure_payloads(value: object): + original = Failure("custom", value=value, response_text="") + translator, _ = translator_for([Success('{"value":1}')], RejectingValidator(original), repair=False) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, Failure) + assert result.has_value + assert result.value is value + assert result.response_text == "" + assert result.message == "custom" + assert original == Failure("custom", value=value, response_text="") + + +def test_translator_does_not_mutate_custom_message_only_failures(): + original = Failure("custom") + translator, _ = translator_for([Success('{"value":1}')], RejectingValidator(original), repair=False) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, Failure) + assert result.has_value + assert result.value == {"value": 1} + assert result.response_text == '{"value":1}' + assert not original.has_value + assert original.response_text is None + + +def test_translator_preserves_failure_subclasses_and_metadata(): + original = CustomFailure("custom", "CUSTOM") + translator, _ = translator_for([Success('{"value":1}')], RejectingValidator(original), repair=False) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, CustomFailure) + assert result.code == "CUSTOM" + assert result.has_value + assert result.value == {"value": 1} + assert not original.has_value + + +@pytest.mark.parametrize("response_text", ["not JSON", 'before {"value":} after', ""]) +def test_translator_retains_unparseable_text_without_inventing_data(response_text: str): + translator, model = translator_for([Success(response_text)], repair=False) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, Failure) + assert not result.has_value + assert result.response_text == response_text + assert len(model.prompts) == 1 + + +def test_translator_propagates_initial_model_failures_unchanged(): + original = Failure("network failure") + translator, _ = translator_for([original]) + assert asyncio.run(translator.translate("request")) is original + + +def test_translator_preserves_successful_repair(): + first = '{"value":"invalid"}' + translator, model = translator_for([Success(first), Success('{"value":42}')]) + assert asyncio.run(translator.translate("request")) == Success(Example(42)) + assert len(model.prompts) == 2 + repair_prompt = model.prompts[1] + assert isinstance(repair_prompt, list) + assert repair_prompt[1]["content"] == first + assert "failed for value" in repair_prompt[2]["content"] + + +def test_translator_returns_latest_rejected_candidate_after_one_repair(): + response_text = '{"value":"second"}' + translator, model = translator_for([Success('{"value":"first"}'), Success(response_text)]) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, Failure) + assert result.has_value + assert result.value == {"value": "second"} + assert result.response_text == response_text + assert len(model.prompts) == 2 + + +def test_translator_retains_previous_candidate_when_repair_request_fails(): + response_text = '{"value":"invalid"}' + original = Failure("network failure") + translator, _ = translator_for([Success(response_text), original]) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, Failure) + assert result.message == "network failure" + assert result.has_value + assert result.value == {"value": "invalid"} + assert result.response_text == response_text + assert not original.has_value + assert original.response_text is None + + +def test_translator_preserves_context_supplied_by_the_model(): + original = Failure("model failure", value=None, response_text="") + translator, _ = translator_for([Success('{"value":"invalid"}'), original]) + assert asyncio.run(translator.translate("request")) == original + + +def test_translator_preserves_model_failure_subclasses(): + original = CustomFailure("network failure", "NETWORK") + translator, _ = translator_for([Success('{"value":"invalid"}'), original]) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, CustomFailure) + assert result.code == "NETWORK" + assert result.value == {"value": "invalid"} + assert not original.has_value + + +def test_translator_does_not_leak_failure_context_into_another_translation(): + original = Failure("new request failed") + translator, _ = translator_for([Success('{"value":"invalid"}'), Failure("repair failed"), original]) + asyncio.run(translator.translate("first")) + assert asyncio.run(translator.translate("second")) is original + + +@pytest.mark.parametrize("response_text", ["not JSON", '{"value":}']) +def test_translator_retains_previous_candidate_when_repair_cannot_be_parsed(response_text: str): + translator, model = translator_for([Success('{"value":"invalid"}'), Success(response_text)]) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, Failure) + assert result.has_value + assert result.value == {"value": "invalid"} + assert result.response_text == response_text + assert len(model.prompts) == 2 + + +@pytest.mark.parametrize("response_text", ["not JSON", '{"value":}']) +def test_python_still_repairs_unparseable_output(response_text: str): + translator, model = translator_for([Success(response_text), Success('{"value":42}')]) + assert asyncio.run(translator.translate("request")) == Success(Example(42)) + assert len(model.prompts) == 2 diff --git a/site/src/docs/python/basic-usage.md b/site/src/docs/python/basic-usage.md index de556fdf..d4b0fa29 100644 --- a/site/src/docs/python/basic-usage.md +++ b/site/src/docs/python/basic-usage.md @@ -170,7 +170,7 @@ class TypeChatValidator(Generic[T]): Validates the given Python object according to the associated schema type. Returns a `Success[T]` object containing the object if validation was successful. - Otherwise, returns a `Failure` object with a `message` property describing the error. + Otherwise, returns a `Failure` with an error message and the rejected object in `value`. """ ... ``` @@ -263,6 +263,32 @@ access our well-typed `value` property! This will correspond to the type that we passed in when we created our translator object (i.e. `Sentiment`). +## Accessing Failed Output + +A `Failure` can also provide an unvalidated `value: object` and `response_text: str | None`. +Check `has_value` before using `value`; it distinguishes missing data from a rejected `None`, +`False`, `0`, or empty string. + +```py +result = await translator.translate(message) +if isinstance(result, Failure): + original_response = result.response_text + if result.has_value: + rejected = result.value + # Validate or repair this value before using it as a Sentiment. +``` + +Use `isinstance` with `Success` or `Failure` to distinguish outcomes, not the presence of `value`. +Validators also retain rejected inputs when used directly. +Custom validators can return `Failure(message, value=rejected, response_text=text)`, and the +translator preserves their payloads. + +If automatic repair fails, TypeChat retains the latest rejected candidate and available model text, +not a full attempt history. These can refer to different attempts if the repair response could not +be parsed. A network failure before any output has no candidate value. Repair limits and successful +results are unchanged, including Python's existing repair of malformed JSON. +Rejected values and response text may contain sensitive output; do not log or serialize them blindly. + That's it! You should now have a basic idea of TypeChat's APIs and how to get started with a new project. 🎉 diff --git a/site/src/docs/typescript/basic-usage.md b/site/src/docs/typescript/basic-usage.md index 9bf343b2..085a5c9c 100644 --- a/site/src/docs/typescript/basic-usage.md +++ b/site/src/docs/typescript/basic-usage.md @@ -166,7 +166,7 @@ export interface TypeChatJsonValidator { /** * Validates the given JSON object according to the associated TypeScript schema. Returns a * `Success` object containing the JSON object if validation was successful. Otherwise, returns - * an `Error` object with a `message` property describing the error. + * an `Error` object with a `message` describing the error and, when available, rejected `data`. * @param jsonText The JSON object to validate. * @returns The JSON object or an error message. */ @@ -318,6 +318,33 @@ If something goes wrong, TypeChat will retry requests up to a maximum specified However, if the initial request as well as all retries fail, `response.success` will be `false` and we'll be able to grab a `message` explaining what went wrong. In the ideal case, `response.success` will be `true` and we'll be able to access our well-typed `data` property! + +## Accessing Failed Output + +A failed response may also provide `data: unknown` and `responseText: string`. +The data is the rejected value, not a validated `SentimentResponse`. +The response text is the original model output, including when it cannot be parsed as JSON. + +```ts +const response = await translator.translate(request); +if (!response.success) { + const originalResponse = response.responseText; + if ("data" in response) { + const rejected: unknown = response.data; + // Validate or repair this value before using it as a SentimentResponse. + } +} +``` + +Use `success` to distinguish outcomes, not the presence of `data`. +TypeScript and Zod validators also retain rejected inputs when used directly. +Custom validators can return `error(message, { data, responseText })`, and the translator preserves +their payloads. To manage validation repairs yourself, set `translator.attemptRepair = false`. + +If automatic repair fails, TypeChat retains the latest rejected candidate and available model text, +not a full attempt history. These can refer to different attempts if the repair response could not +be parsed. A network failure before any output has no candidate data. +Rejected values and response text may contain sensitive output; do not log them automatically. This will correspond to the type that we passed in when we created our translator object (i.e. `SentimentResponse`). That's it! diff --git a/typescript/README.md b/typescript/README.md index 4ed66add..cc4df6b9 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -40,6 +40,34 @@ To see TypeChat in action, we recommend exploring the [TypeChat example projects To learn more about TypeChat, visit the [documentation](https://microsoft.github.io/TypeChat) which includes more information on TypeChat and how to get started. +## Accessing failed output + +Failed results include a `message` and may also include `data: unknown` and `responseText: string`. +Validators return the rejected input in `data`. Translators also retain the original model text, +including when that text is not valid JSON: + +```ts +const result = await translator.translate(request); +if (!result.success) { + const rejected: unknown = result.data; + const originalResponse = result.responseText; + // Validate or repair the rejected value before treating it as the target type. +} +``` + +On final failure, automatic repair retains the latest rejected candidate and available response text, +not a full attempt history. If a repair response cannot be parsed, `data` can come from an earlier +attempt than `responseText`. An initial network failure has no candidate data. Successful results, +error messages, and repair limits are unchanged; `getData` still throws on failure. + +Use `success` to distinguish successful and failed results, not the presence of `data`. +Custom validators can supply their own failure payloads, which the translator preserves. +To create such a failure, use `error(message, { data, responseText })`; `error(message)` still returns +the original message-only shape. A present `data` property can contain any value, including `null` +or `undefined`, so check property presence when that distinction matters. + +Rejected values and response text may contain sensitive output. Do not log them automatically. + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/typescript/package.json b/typescript/package.json index 1654f47e..0c58996a 100644 --- a/typescript/package.json +++ b/typescript/package.json @@ -21,7 +21,7 @@ }, "scripts": { "build": "tsc -p src", - "test": "npm run build && tsc -p test && node --test out/validate.test.js out/zod.test.js out/program.test.js tests/model.test.mjs tests/examples.test.mjs", + "test": "npm run build && tsc -p test && node --test out/validate.test.js out/zod.test.js out/program.test.js out/failure.test.js tests/model.test.mjs tests/examples.test.mjs", "build-all": "npm run build --workspaces", "prepare": "npm run build-all", "prepublishOnly": "node -e \"require('fs').copyFileSync('../SECURITY.md','SECURITY.md')\"", diff --git a/typescript/src/result.ts b/typescript/src/result.ts index 7ec8c078..d2b8f4c0 100644 --- a/typescript/src/result.ts +++ b/typescript/src/result.ts @@ -6,7 +6,20 @@ export type Success = { success: true, data: T }; /** * An object representing an operation that failed for the reason given in `message`. */ -export type Error = { success: false, message: string }; +export type Error = { + success: false; + message: string; + /** + * An unvalidated value associated with the failure, when available. A translation retains the + * latest rejected candidate, which may precede a repair response that couldn't be parsed. + */ + data?: unknown; + /** + * The latest model response text, when available. This may come from a different attempt than + * `data` if a repair response couldn't be parsed. + */ + responseText?: string; +}; /** * An object representing a successful or failed operation of type `T`. @@ -25,10 +38,21 @@ export function success(data: T): Success { /** * Returns an `Error` object. * @param message The value for the `message` property of the result. + * @param details Optional unvalidated data and original model response text. * @returns An `Error` object. */ -export function error(message: string): Error { - return { success: false, message }; +export function error(message: string, details?: Pick): Error { + const result: Error = { success: false, message }; + if (details) { + if ("data" in details) { + result.data = details.data; + } + const responseText = details.responseText; + if (responseText !== undefined) { + result.responseText = responseText; + } + } + return result; } /** diff --git a/typescript/src/ts/program.ts b/typescript/src/ts/program.ts index 5fc21b5d..c19d01f8 100644 --- a/typescript/src/ts/program.ts +++ b/typescript/src/ts/program.ts @@ -99,7 +99,7 @@ function isValidFunctionName(name: unknown): name is string { export function createModuleTextFromProgram(jsonObject: object): Result { const steps = (jsonObject as Program)["@steps"]; if (!(Array.isArray(steps) && steps.every(step => typeof step === "object" && step !== null && step.hasOwnProperty("@func")))) { - return error("JSON object is not a valid program"); + return error("JSON object is not a valid program", { data: jsonObject }); } let hasError = false; let functionBody = ""; @@ -109,7 +109,7 @@ export function createModuleTextFromProgram(jsonObject: object): Result currentStep++; } return hasError ? - error("JSON program contains an invalid expression") : + error("JSON program contains an invalid expression", { data: jsonObject }) : success(`import { API } from "./schema";\nfunction program(api: API) {\n${functionBody}}`); function exprToString(expr: unknown): string { diff --git a/typescript/src/ts/validate.ts b/typescript/src/ts/validate.ts index b2b0b9c5..be202b74 100644 --- a/typescript/src/ts/validate.ts +++ b/typescript/src/ts/validate.ts @@ -89,16 +89,21 @@ export function createTypeScriptJsonValidator(schema: }; return validator; - function validate(jsonObject: object) { + function validate(jsonObject: object): Result { const moduleResult = validator.createModuleTextFromJson(jsonObject); if (!moduleResult.success) { - return moduleResult; + return "data" in moduleResult ? moduleResult : { + ...moduleResult, + success: false, + message: moduleResult.message, + data: jsonObject + }; } writeJsonFile(jsonFileName, moduleResult.data); const snapshot = api.updateSnapshot({ fileChanges: { changed: [jsonFileName] } }); const project = snapshot.getProject(configFileName); if (!project) { - return error(`The TypeScript schema project '${configFileName}' couldn't be loaded.`); + return error(`The TypeScript schema project '${configFileName}' couldn't be loaded.`, { data: jsonObject }); } const program = project.program; const syntacticDiagnostics = program.getSyntacticDiagnostics(); @@ -115,7 +120,7 @@ export function createTypeScriptJsonValidator(schema: } return message; }).join("\n"); - return error(diagnostics); + return error(diagnostics, { data: jsonObject }); } return success(jsonObject as T); } diff --git a/typescript/src/typechat.ts b/typescript/src/typechat.ts index 74c67331..af7a6a8b 100644 --- a/typescript/src/typechat.ts +++ b/typescript/src/typechat.ts @@ -1,4 +1,4 @@ -import { Result, success, error } from "./result"; +import { Result, success, error, Error as TypeChatError } from "./result"; import { TypeChatLanguageModel, PromptSection, PromptContent } from "./model"; /** @@ -55,6 +55,8 @@ export interface TypeChatJsonTranslator { * the language model fails to validate and the `attemptRepair` property is `true`, a second * attempt to translate the request will be made. The prompt for the second attempt will include the * diagnostics produced for the first attempt. This often helps produce a valid instance. + * A failure retains the latest rejected candidate and available model response text. These may + * come from different attempts if a repair response couldn't be parsed. * @param request The natural language request. * @param promptPreamble An optional string or array of prompt sections to prepend to the generated * prompt. If a string is specified, it is converted into a single "user" role prompt section. @@ -78,7 +80,7 @@ export interface TypeChatJsonValidator { /** * Validates the given JSON object according to the associated TypeScript schema. Returns a * `Success` object containing the JSON object if validation was successful. Otherwise, returns - * an `Error` object with a `message` property describing the error. + * an `Error` object with a `message` describing the error and, when available, rejected `data`. * @param jsonText The JSON object to validate. * @returns The JSON object or an error message. */ @@ -121,20 +123,29 @@ export function createJsonTranslator(model: TypeChatLanguageMo `The following is a revised JSON object:\n`; } - async function translate(request: string, promptPreamble?: string | PromptSection[]) { + async function translate(request: string, promptPreamble?: string | PromptSection[]): Promise> { const preamble: PromptSection[] = typeof promptPreamble === "string" ? [{ role: "user", content: promptPreamble }] : promptPreamble ?? []; let prompt: PromptSection[] = [...preamble, { role: "user", content: typeChat.createRequestPrompt(request) }]; let attemptRepair = typeChat.attemptRepair; + let failureContext: (Pick & { responseText: string }) | undefined; while (true) { const response = await model.complete(prompt); if (!response.success) { - return response; + return failureContext ? { + ...failureContext, + ...response, + success: false, + message: response.message, + ...("data" in response ? { data: response.data } : {}), + responseText: response.responseText ?? failureContext.responseText + } : response; } const responseText = response.data; + failureContext = { ...failureContext, responseText }; const startIndex = responseText.indexOf("{"); const endIndex = responseText.lastIndexOf("}"); if (!(startIndex >= 0 && endIndex > startIndex)) { - return error(`Response is not JSON:\n${responseText}`); + return error(`Response is not JSON:\n${responseText}`, failureContext); } const jsonText = responseText.slice(startIndex, endIndex + 1); let jsonObject; @@ -142,7 +153,7 @@ export function createJsonTranslator(model: TypeChatLanguageMo jsonObject = JSON.parse(jsonText) as object; } catch (e) { - return error(e instanceof SyntaxError ? e.message : "JSON parse error"); + return error(e instanceof SyntaxError ? e.message : "JSON parse error", failureContext); } if (typeChat.stripNulls) { stripNulls(jsonObject); @@ -152,11 +163,19 @@ export function createJsonTranslator(model: TypeChatLanguageMo if (validation.success) { return validation; } + const failure: TypeChatError & { responseText: string } = { + ...validation, + success: false, + message: validation.message, + data: "data" in validation ? validation.data : schemaValidation.success ? schemaValidation.data : jsonObject, + responseText: validation.responseText ?? responseText + }; + failureContext = { data: failure.data, responseText: failure.responseText }; if (!attemptRepair) { - return error(`JSON validation failed: ${validation.message}\n${jsonText}`); + return { ...failure, message: `JSON validation failed: ${failure.message}\n${jsonText}` }; } prompt.push({ role: "assistant", content: responseText }); - prompt.push({ role: "user", content: typeChat.createRepairPrompt(validation.message) }); + prompt.push({ role: "user", content: typeChat.createRepairPrompt(failure.message) }); attemptRepair = false; } } diff --git a/typescript/src/zod/validate.ts b/typescript/src/zod/validate.ts index 02e5bb98..4babb5cc 100644 --- a/typescript/src/zod/validate.ts +++ b/typescript/src/zod/validate.ts @@ -23,7 +23,7 @@ export function createZodJsonValidator, K ex function validate(jsonObject: object) { const result = schema[typeName].safeParse(jsonObject); if (!result.success) { - return error(result.error.issues.map(({ path, message }) => `${path.map(key => `[${JSON.stringify(key)}]`).join("")}: ${message}`).join("\n")); + return error(result.error.issues.map(({ path, message }) => `${path.map(key => `[${JSON.stringify(key)}]`).join("")}: ${message}`).join("\n"), { data: jsonObject }); } return success(result.data as z.infer & object); } diff --git a/typescript/test/failure.test.ts b/typescript/test/failure.test.ts new file mode 100644 index 00000000..5a5666d1 --- /dev/null +++ b/typescript/test/failure.test.ts @@ -0,0 +1,323 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { z } from "zod"; +import { + createJsonTranslator, error, getData, success, + Result, TypeChatJsonValidator, TypeChatLanguageModel, PromptSection +} from "../dist/index.js"; +import { createModuleTextFromProgram, createTypeScriptJsonValidator } from "../dist/ts/index.js"; +import { createZodJsonValidator } from "../dist/zod/index.js"; + +type Example = { value: number }; +type IsUnknown = unknown extends T ? keyof T extends never ? true : false : false; + +class GetterFailure { + get success(): false { return false; } + get message() { return "custom"; } + get data(): unknown { return null; } + get responseText() { return ""; } +} + +function exampleValidator(): TypeChatJsonValidator { + return { + getSchemaText: () => "export interface Example { value: number; }", + getTypeName: () => "Example", + validate: value => "value" in value && typeof value.value === "number" ? + success({ value: value.value }) : error("value must be a number") + }; +} + +function translatorFor(responses: Result[], validator = exampleValidator()) { + const prompts: (string | PromptSection[])[] = []; + let index = 0; + const model: TypeChatLanguageModel = { + async complete(prompt) { + prompts.push(typeof prompt === "string" ? prompt : [...prompt]); + const response = responses[index++]; + if (!response) { + throw new Error("Unexpected model call"); + } + return response; + } + }; + return { translator: createJsonTranslator(model, validator), prompts }; +} + +describe("failure result contract", () => { + it("preserves the shape of message-only failures", () => { + assert.deepEqual(error("failed"), { success: false, message: "failed" }); + }); + + for (const data of [null, false, 0, "", undefined]) { + it(`retains an explicitly supplied ${String(data)} payload`, () => { + const result = error("failed", { data }); + assert.ok("data" in result); + assert.equal(result.data, data); + }); + } + + it("keeps failure data unknown and getData throwing", () => { + const result = error("invalid", { data: { value: "invalid" } }); + const isUnknown: IsUnknown = true; + assert.equal(isUnknown, true); + assert.throws(() => getData(result), /invalid/); + }); + + it("preserves typed successful results", () => { + const result = success({ value: 42 }); + const value: number = result.data.value; + assert.equal(value, 42); + assert.deepEqual(getData(result), { value: 42 }); + }); + + it("retains an empty response text", () => { + assert.equal(error("failed", { responseText: "" }).responseText, ""); + }); + + it("accepts failure details supplied through getters", () => { + assert.deepEqual(error("failed", new GetterFailure()), { + success: false, message: "failed", data: null, responseText: "" + }); + }); +}); + +describe("direct validator failure payloads", () => { + it("retains the TypeScript validator's rejected object", () => { + const validator = createTypeScriptJsonValidator( + "export interface Example { value: number; }", "Example" + ); + try { + const input = { value: "invalid" }; + const result = validator.validate(input); + assert.ok(!result.success); + assert.equal(result.data, input); + assert.ok(!("responseText" in result)); + assert.match(result.message, /not assignable/); + } + finally { + validator.close(); + } + }); + + it("preserves module-conversion failure metadata", () => { + const validator = createTypeScriptJsonValidator( + "export interface Example { value: number; }", "Example" + ); + const original = error("conversion failed", { data: null, responseText: "custom" }); + validator.createModuleTextFromJson = () => original; + try { + assert.deepEqual(validator.validate({ value: 1 }), original); + } + finally { + validator.close(); + } + }); + + it("adds the input to message-only module-conversion failures", () => { + const validator = createTypeScriptJsonValidator( + "export interface Example { value: number; }", "Example" + ); + validator.createModuleTextFromJson = () => error("conversion failed"); + try { + const input = { value: 1 }; + const result = validator.validate(input); + assert.ok(!result.success); + assert.equal(result.data, input); + } + finally { + validator.close(); + } + }); + + it("retains the Zod validator's rejected object", () => { + const validator = createZodJsonValidator({ Example: z.object({ value: z.number() }) }, "Example"); + const input = { value: "invalid" }; + const result = validator.validate(input); + assert.ok(!result.success); + assert.equal(result.data, input); + }); + + for (const input of [ + { "@steps": "invalid" }, + { "@steps": [{ "@func": "valid", "@args": [{ "@ref": -1 }] }] } + ]) { + it("retains a rejected program during module conversion", () => { + const result = createModuleTextFromProgram(input); + assert.ok(!result.success); + assert.equal(result.data, input); + }); + } +}); + +describe("translation failure context", () => { + it("preserves immediate successful results", async () => { + const { translator, prompts } = translatorFor([success('{"value":42}')]); + assert.deepEqual(await translator.translate("request"), success({ value: 42 })); + assert.equal(prompts.length, 1); + }); + + it("returns rejected data and the original response without repair", async () => { + const responseText = 'before {"value":"invalid"} after'; + const { translator, prompts } = translatorFor([success(responseText)]); + translator.attemptRepair = false; + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.deepEqual(result.data, { value: "invalid" }); + assert.equal(result.responseText, responseText); + assert.equal(result.message, 'JSON validation failed: value must be a number\n{"value":"invalid"}'); + assert.equal(prompts.length, 1); + }); + + for (const data of [null, false, 0, "", undefined]) { + it(`preserves a custom validator's ${String(data)} payload and metadata`, async () => { + const original = { ...error("custom", { data, responseText: "" }), code: "CUSTOM" }; + const validator = { ...exampleValidator(), validate: () => original }; + const { translator } = translatorFor([success('{"value":1}')], validator); + translator.attemptRepair = false; + const result = await translator.translate("request"); + assert.deepEqual(result, { ...original, message: 'JSON validation failed: custom\n{"value":1}' }); + assert.equal(original.message, "custom"); + }); + } + + it("does not mutate message-only custom failures", async () => { + const original = error("custom"); + const validator = { ...exampleValidator(), validate: () => original }; + const { translator } = translatorFor([success('{"value":1}')], validator); + translator.attemptRepair = false; + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.deepEqual(result.data, { value: 1 }); + assert.deepEqual(original, error("custom")); + }); + + it("preserves a custom validator's inherited failure fields", async () => { + const validator = { ...exampleValidator(), validate: () => new GetterFailure() }; + const { translator } = translatorFor([success('{"value":1}')], validator); + translator.attemptRepair = false; + assert.deepEqual(await translator.translate("request"), { + success: false, message: 'JSON validation failed: custom\n{"value":1}', data: null, responseText: "" + }); + }); + + it("retains the transformed instance rejected by additional validation", async () => { + const validator = createZodJsonValidator({ Example: z.object({ value: z.coerce.number() }) }, "Example"); + const responseText = '{"value":"42"}'; + const { translator } = translatorFor([success(responseText)], validator); + translator.attemptRepair = false; + translator.validateInstance = () => error("instance rejected"); + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.deepEqual(result.data, { value: 42 }); + assert.equal(result.responseText, responseText); + }); + + it("distinguishes preprocessed data from the original response", async () => { + const responseText = '{"value":"invalid","optional":null}'; + const { translator } = translatorFor([success(responseText)]); + translator.attemptRepair = false; + translator.stripNulls = true; + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.deepEqual(result.data, { value: "invalid" }); + assert.equal(result.responseText, responseText); + }); + + for (const responseText of ["not JSON", 'before {"value":} after', ""]) { + it("retains unparseable output without inventing data or changing repair behavior", async () => { + const { translator, prompts } = translatorFor([success(responseText)]); + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.ok(!("data" in result)); + assert.equal(result.responseText, responseText); + assert.equal(prompts.length, 1); + }); + } + + it("propagates an initial model failure unchanged", async () => { + const original = error("network failure"); + const { translator } = translatorFor([original]); + assert.equal(await translator.translate("request"), original); + }); + + it("does not convert validator exceptions to failures", async () => { + const validator = { + ...exampleValidator(), + validate: (): Result => { throw new Error("validator bug"); } + }; + const { translator } = translatorFor([success('{"value":1}')], validator); + await assert.rejects(translator.translate("request"), /validator bug/); + }); +}); + +describe("repair failure context", () => { + it("preserves a successful repair and the existing repair prompt", async () => { + const first = '{"value":"invalid"}'; + const { translator, prompts } = translatorFor([success(first), success('{"value":42}')]); + assert.deepEqual(await translator.translate("request"), success({ value: 42 })); + assert.equal(prompts.length, 2); + const repairPrompt = prompts[1]; + assert.ok(Array.isArray(repairPrompt)); + assert.equal(repairPrompt[1].content, first); + assert.match(String(repairPrompt[2].content), /value must be a number/); + }); + + it("returns the latest rejected candidate after exactly one repair", async () => { + const responseText = '{"value":"second"}'; + const { translator, prompts } = translatorFor([ + success('{"value":"first"}'), success(responseText) + ]); + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.deepEqual(result.data, { value: "second" }); + assert.equal(result.responseText, responseText); + assert.equal(prompts.length, 2); + }); + + it("retains the previous candidate when the repair request fails", async () => { + const responseText = '{"value":"invalid"}'; + const original = error("network failure"); + const { translator } = translatorFor([success(responseText), original]); + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.equal(result.message, "network failure"); + assert.deepEqual(result.data, { value: "invalid" }); + assert.equal(result.responseText, responseText); + assert.deepEqual(original, error("network failure")); + }); + + it("preserves context explicitly supplied by the model", async () => { + const original = error("model failure", { data: null, responseText: "" }); + const { translator } = translatorFor([success('{"value":"invalid"}'), original]); + assert.deepEqual(await translator.translate("request"), original); + }); + + it("preserves a model's inherited failure fields", async () => { + const { translator } = translatorFor([success('{"value":"invalid"}'), new GetterFailure()]); + assert.deepEqual(await translator.translate("request"), { + success: false, message: "custom", data: null, responseText: "" + }); + }); + + it("does not leak failure context into another translation", async () => { + const original = error("new request failed"); + const { translator } = translatorFor([ + success('{"value":"invalid"}'), error("repair failed"), original + ]); + await translator.translate("first"); + assert.equal(await translator.translate("second"), original); + }); + + for (const responseText of ["not JSON", '{"value":}']) { + it("retains the previous candidate when the repair response cannot be parsed", async () => { + const { translator, prompts } = translatorFor([ + success('{"value":"invalid"}'), success(responseText) + ]); + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.deepEqual(result.data, { value: "invalid" }); + assert.equal(result.responseText, responseText); + assert.equal(prompts.length, 2); + }); + } +});