diff --git a/python/README.md b/python/README.md index 11571366..7ee7bca5 100644 --- a/python/README.md +++ b/python/README.md @@ -50,6 +50,37 @@ To learn more about TypeChat, visit the [documentation](https://microsoft.github.io/TypeChat/docs/python/basic-usage/) which includes more information on TypeChat and how to get started. +## Accessing failed output + +`Failure` retains its existing `message` and can also carry an unvalidated `value: object` and +`response_text: str | None`. Validators return the rejected input in `value`; translators also retain +the original model text, including when it is not valid JSON: + +```python +from typechat import Failure + +result = await translator.translate(request) +if isinstance(result, Failure): + if result.has_value: + rejected = result.value + # Validate or repair this value before treating it as the target type. + original_response = result.response_text +``` + +Check `has_value` before using `value`. It distinguishes unavailable data from a rejected `None`, +`False`, `0`, or empty string. An unavailable value is represented by an internal missing-value marker, +not `None`. Custom validators can supply their own payloads using +`Failure(message, value=rejected, response_text=text)`, and the translator preserves those payloads. +`Failure(message)` remains valid, and payload fields are omitted from its representation. +Use `isinstance` with `Success` or `Failure` to distinguish outcomes, not the presence of `value`. + +On final failure, automatic repair retains the latest rejected candidate and available response text, +not a full attempt history. If a repair response cannot be parsed, `value` can come from an earlier +attempt than `response_text`. An initial network failure has no candidate value. Successful results, +error messages, and repair limits are unchanged, including Python's existing repair of malformed JSON. + +Rejected values and response text may contain sensitive output. Do not log or serialize them blindly. + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/python/src/typechat/_internal/result.py b/python/src/typechat/_internal/result.py index a9578cec..92029967 100644 --- a/python/src/typechat/_internal/result.py +++ b/python/src/typechat/_internal/result.py @@ -1,8 +1,12 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field +from enum import Enum from typing_extensions import Generic, TypeAlias, TypeVar T = TypeVar("T", covariant=True) +class _MissingValue(Enum): + MISSING = 0 + @dataclass class Success(Generic[T]): "An object representing a successful operation with a result of type `T`." @@ -11,8 +15,21 @@ class Success(Generic[T]): @dataclass class Failure: - "An object representing an operation that failed for the reason given in `message`." + """ + An operation that failed for the reason given in `message`. + + `value` is unvalidated and is available only when `has_value` is true. `response_text` contains + the latest model response, when available. If a repair response couldn't be parsed, `value` + may refer to a rejected candidate from an earlier attempt. + """ message: str + value: object = field(default=_MissingValue.MISSING, kw_only=True, repr=False) + response_text: str | None = field(default=None, kw_only=True, repr=False) + + @property + def has_value(self) -> bool: + """Distinguishes missing data from an explicitly supplied value, including `None`.""" + return self.value is not _MissingValue.MISSING """ diff --git a/python/src/typechat/_internal/translator.py b/python/src/typechat/_internal/translator.py index ee077195..db44b4b2 100644 --- a/python/src/typechat/_internal/translator.py +++ b/python/src/typechat/_internal/translator.py @@ -1,3 +1,4 @@ +from copy import copy from typing_extensions import Generic, TypeVar import pydantic_core @@ -55,6 +56,8 @@ async def translate(self, input: str, *, prompt_preamble: str | list[PromptSecti the language model fails to validate, repair attempts will be made up until `_max_repair_attempts`. The prompt for the subsequent attempts will include the diagnostics produced for the prior attempt. This often helps produce a valid instance. + A failure retains the latest rejected value and available model response text. These may come + from different attempts if a repair response couldn't be parsed. Args: input: A natural language request. @@ -72,33 +75,54 @@ async def translate(self, input: str, *, prompt_preamble: str | list[PromptSecti messages.append({"role": "user", "content": self._create_request_prompt(input)}) num_repairs_attempted = 0 + last_failure: Failure | None = None while True: completion_response = await self.model.complete(messages) if isinstance(completion_response, Failure): - return completion_response + if last_failure is None: + return completion_response + failure = copy(completion_response) + if not failure.has_value: + failure.value = last_failure.value + if failure.response_text is None: + failure.response_text = last_failure.response_text + return failure text_response = completion_response.value first_curly = text_response.find("{") last_curly = text_response.rfind("}") + 1 - error_message: str + failure: Failure if 0 <= first_curly < last_curly: trimmed_response = text_response[first_curly:last_curly] try: parsed_response = pydantic_core.from_json(trimmed_response, allow_inf_nan=False, cache_strings=False) except ValueError as e: - error_message = f"Error: {e}\n\nAttempted to parse:\n\n{trimmed_response}" + failure = Failure( + f"Error: {e}\n\nAttempted to parse:\n\n{trimmed_response}", + response_text=text_response, + ) else: result = self.validator.validate_object(parsed_response) if isinstance(result, Success): return result - error_message = result.message + failure = copy(result) + if failure.response_text is None: + failure.response_text = text_response + if not failure.has_value: + failure.value = parsed_response else: - error_message = f"Response did not contain any text resembling JSON.\nResponse was\n\n{text_response}" + failure = Failure( + f"Response did not contain any text resembling JSON.\nResponse was\n\n{text_response}", + response_text=text_response, + ) + if not failure.has_value and last_failure is not None: + failure.value = last_failure.value + last_failure = failure if num_repairs_attempted >= self._max_repair_attempts: - return Failure(error_message) + return failure num_repairs_attempted += 1 messages.append({"role": "assistant", "content": text_response}) - messages.append({"role": "user", "content": self._create_repair_prompt(error_message)}) + messages.append({"role": "user", "content": self._create_repair_prompt(failure.message)}) def _create_request_prompt(self, intent: str) -> str: prompt = f""" diff --git a/python/src/typechat/_internal/validator.py b/python/src/typechat/_internal/validator.py index a1d17f1d..5ed87512 100644 --- a/python/src/typechat/_internal/validator.py +++ b/python/src/typechat/_internal/validator.py @@ -29,7 +29,7 @@ def validate_object(self, obj: object) -> Result[T]: Validates the given Python object according to the associated schema type. Returns a `Success[T]` object containing the object if validation was successful. - Otherwise, returns a `Failure` object with a `message` property describing the error. + Otherwise, returns a `Failure` with an error message and the rejected object in `value`. """ try: # TODO: Switch to `validate_python` when validation modes are exposed. @@ -42,10 +42,10 @@ def validate_object(self, obj: object) -> Result[T]: typed_dict = self._adapted_type.validate_json(json_str, strict=True) return Success(typed_dict) except pydantic.ValidationError as validation_error: - return _handle_error(validation_error) + return _handle_error(validation_error, obj) -def _handle_error(validation_error: pydantic.ValidationError) -> Failure: +def _handle_error(validation_error: pydantic.ValidationError, value: object) -> Failure: error_strings: list[str] = [] for error in validation_error.errors(include_url=False): error_string = "" @@ -64,4 +64,4 @@ def _handle_error(validation_error: pydantic.ValidationError) -> Failure: failure_message = "" failure_message += "\n".join(error_strings) - return Failure(failure_message) + return Failure(failure_message, value=value) diff --git a/python/tests/test_failure.py b/python/tests/test_failure.py new file mode 100644 index 00000000..86c283c8 --- /dev/null +++ b/python/tests/test_failure.py @@ -0,0 +1,243 @@ +import asyncio +from copy import deepcopy +from dataclasses import dataclass + +import pytest +from typing_extensions import assert_type, override + +from typechat import ( + Failure, + PromptSection, + Result, + Success, + TypeChatJsonTranslator, + TypeChatLanguageModel, + TypeChatValidator, +) + + +@dataclass +class Example: + value: int + + +class FixedResultModel(TypeChatLanguageModel): + def __init__(self, responses: list[Result[str]]) -> None: + super().__init__() + self._responses = iter(responses) + self.prompts: list[str | list[PromptSection]] = [] + + @override + async def complete(self, prompt: str | list[PromptSection]) -> Result[str]: + self.prompts.append(prompt.copy() if isinstance(prompt, list) else prompt) + return next(self._responses) + + +class NoRepairTranslator(TypeChatJsonTranslator[Example]): + _max_repair_attempts = 0 + + +class RejectingValidator(TypeChatValidator[Example]): + def __init__(self, failure: Failure) -> None: + super().__init__(Example) + self.failure = failure + + @override + def validate_object(self, obj: object) -> Result[Example]: + return self.failure + + +class CustomFailure(Failure): + def __init__(self, message: str, code: str) -> None: + super().__init__(message) + self.code = code + + +def translator_for( + responses: list[Result[str]], + validator: TypeChatValidator[Example] | None = None, + *, + repair: bool = True, +) -> tuple[TypeChatJsonTranslator[Example], FixedResultModel]: + model = FixedResultModel(responses) + validator = validator if validator is not None else TypeChatValidator(Example) + if repair: + translator = TypeChatJsonTranslator(model, validator, Example) + else: + translator = NoRepairTranslator(model, validator, Example) + return translator, model + + +def test_message_only_failure_has_no_value(): + result = Failure("failed") + assert not result.has_value + assert result.response_text is None + assert repr(result) == "Failure(message='failed')" + assert not deepcopy(result).has_value + + +@pytest.mark.parametrize("value", [None, False, 0, "", {}, []]) +def test_failure_retains_explicit_values(value: object): + result = Failure("failed", value=value, response_text="") + assert_type(result.value, object) + assert result.has_value + assert result.value is value + assert result.response_text == "" + assert "response_text" not in repr(result) + + +@pytest.mark.parametrize("value", [{"value": "invalid"}, None, False, 0, ""]) +def test_validator_retains_the_entire_rejected_value(value: object): + result = TypeChatValidator(Example).validate_object(value) + assert isinstance(result, Failure) + assert result.has_value + assert result.value is value + assert result.response_text is None + + +def test_translator_preserves_success(): + translator, model = translator_for([Success('{"value":42}')]) + result = asyncio.run(translator.translate("request")) + assert result == Success(Example(42)) + assert len(model.prompts) == 1 + + +def test_translator_retains_rejected_data_without_repair(): + response_text = 'before {"value":"invalid"} after' + translator, model = translator_for([Success(response_text)], repair=False) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, Failure) + assert result.has_value + assert result.value == {"value": "invalid"} + assert result.response_text == response_text + assert "failed for value" in result.message + assert len(model.prompts) == 1 + + +@pytest.mark.parametrize("value", [None, False, 0, ""]) +def test_translator_preserves_custom_failure_payloads(value: object): + original = Failure("custom", value=value, response_text="") + translator, _ = translator_for([Success('{"value":1}')], RejectingValidator(original), repair=False) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, Failure) + assert result.has_value + assert result.value is value + assert result.response_text == "" + assert result.message == "custom" + assert original == Failure("custom", value=value, response_text="") + + +def test_translator_does_not_mutate_custom_message_only_failures(): + original = Failure("custom") + translator, _ = translator_for([Success('{"value":1}')], RejectingValidator(original), repair=False) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, Failure) + assert result.has_value + assert result.value == {"value": 1} + assert result.response_text == '{"value":1}' + assert not original.has_value + assert original.response_text is None + + +def test_translator_preserves_failure_subclasses_and_metadata(): + original = CustomFailure("custom", "CUSTOM") + translator, _ = translator_for([Success('{"value":1}')], RejectingValidator(original), repair=False) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, CustomFailure) + assert result.code == "CUSTOM" + assert result.has_value + assert result.value == {"value": 1} + assert not original.has_value + + +@pytest.mark.parametrize("response_text", ["not JSON", 'before {"value":} after', ""]) +def test_translator_retains_unparseable_text_without_inventing_data(response_text: str): + translator, model = translator_for([Success(response_text)], repair=False) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, Failure) + assert not result.has_value + assert result.response_text == response_text + assert len(model.prompts) == 1 + + +def test_translator_propagates_initial_model_failures_unchanged(): + original = Failure("network failure") + translator, _ = translator_for([original]) + assert asyncio.run(translator.translate("request")) is original + + +def test_translator_preserves_successful_repair(): + first = '{"value":"invalid"}' + translator, model = translator_for([Success(first), Success('{"value":42}')]) + assert asyncio.run(translator.translate("request")) == Success(Example(42)) + assert len(model.prompts) == 2 + repair_prompt = model.prompts[1] + assert isinstance(repair_prompt, list) + assert repair_prompt[1]["content"] == first + assert "failed for value" in repair_prompt[2]["content"] + + +def test_translator_returns_latest_rejected_candidate_after_one_repair(): + response_text = '{"value":"second"}' + translator, model = translator_for([Success('{"value":"first"}'), Success(response_text)]) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, Failure) + assert result.has_value + assert result.value == {"value": "second"} + assert result.response_text == response_text + assert len(model.prompts) == 2 + + +def test_translator_retains_previous_candidate_when_repair_request_fails(): + response_text = '{"value":"invalid"}' + original = Failure("network failure") + translator, _ = translator_for([Success(response_text), original]) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, Failure) + assert result.message == "network failure" + assert result.has_value + assert result.value == {"value": "invalid"} + assert result.response_text == response_text + assert not original.has_value + assert original.response_text is None + + +def test_translator_preserves_context_supplied_by_the_model(): + original = Failure("model failure", value=None, response_text="") + translator, _ = translator_for([Success('{"value":"invalid"}'), original]) + assert asyncio.run(translator.translate("request")) == original + + +def test_translator_preserves_model_failure_subclasses(): + original = CustomFailure("network failure", "NETWORK") + translator, _ = translator_for([Success('{"value":"invalid"}'), original]) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, CustomFailure) + assert result.code == "NETWORK" + assert result.value == {"value": "invalid"} + assert not original.has_value + + +def test_translator_does_not_leak_failure_context_into_another_translation(): + original = Failure("new request failed") + translator, _ = translator_for([Success('{"value":"invalid"}'), Failure("repair failed"), original]) + asyncio.run(translator.translate("first")) + assert asyncio.run(translator.translate("second")) is original + + +@pytest.mark.parametrize("response_text", ["not JSON", '{"value":}']) +def test_translator_retains_previous_candidate_when_repair_cannot_be_parsed(response_text: str): + translator, model = translator_for([Success('{"value":"invalid"}'), Success(response_text)]) + result = asyncio.run(translator.translate("request")) + assert isinstance(result, Failure) + assert result.has_value + assert result.value == {"value": "invalid"} + assert result.response_text == response_text + assert len(model.prompts) == 2 + + +@pytest.mark.parametrize("response_text", ["not JSON", '{"value":}']) +def test_python_still_repairs_unparseable_output(response_text: str): + translator, model = translator_for([Success(response_text), Success('{"value":42}')]) + assert asyncio.run(translator.translate("request")) == Success(Example(42)) + assert len(model.prompts) == 2 diff --git a/site/src/docs/python/basic-usage.md b/site/src/docs/python/basic-usage.md index de556fdf..d4b0fa29 100644 --- a/site/src/docs/python/basic-usage.md +++ b/site/src/docs/python/basic-usage.md @@ -170,7 +170,7 @@ class TypeChatValidator(Generic[T]): Validates the given Python object according to the associated schema type. Returns a `Success[T]` object containing the object if validation was successful. - Otherwise, returns a `Failure` object with a `message` property describing the error. + Otherwise, returns a `Failure` with an error message and the rejected object in `value`. """ ... ``` @@ -263,6 +263,32 @@ access our well-typed `value` property! This will correspond to the type that we passed in when we created our translator object (i.e. `Sentiment`). +## Accessing Failed Output + +A `Failure` can also provide an unvalidated `value: object` and `response_text: str | None`. +Check `has_value` before using `value`; it distinguishes missing data from a rejected `None`, +`False`, `0`, or empty string. + +```py +result = await translator.translate(message) +if isinstance(result, Failure): + original_response = result.response_text + if result.has_value: + rejected = result.value + # Validate or repair this value before using it as a Sentiment. +``` + +Use `isinstance` with `Success` or `Failure` to distinguish outcomes, not the presence of `value`. +Validators also retain rejected inputs when used directly. +Custom validators can return `Failure(message, value=rejected, response_text=text)`, and the +translator preserves their payloads. + +If automatic repair fails, TypeChat retains the latest rejected candidate and available model text, +not a full attempt history. These can refer to different attempts if the repair response could not +be parsed. A network failure before any output has no candidate value. Repair limits and successful +results are unchanged, including Python's existing repair of malformed JSON. +Rejected values and response text may contain sensitive output; do not log or serialize them blindly. + That's it! You should now have a basic idea of TypeChat's APIs and how to get started with a new project. 🎉 diff --git a/site/src/docs/typescript/basic-usage.md b/site/src/docs/typescript/basic-usage.md index 9bf343b2..085a5c9c 100644 --- a/site/src/docs/typescript/basic-usage.md +++ b/site/src/docs/typescript/basic-usage.md @@ -166,7 +166,7 @@ export interface TypeChatJsonValidator { /** * Validates the given JSON object according to the associated TypeScript schema. Returns a * `Success` object containing the JSON object if validation was successful. Otherwise, returns - * an `Error` object with a `message` property describing the error. + * an `Error` object with a `message` describing the error and, when available, rejected `data`. * @param jsonText The JSON object to validate. * @returns The JSON object or an error message. */ @@ -318,6 +318,33 @@ If something goes wrong, TypeChat will retry requests up to a maximum specified However, if the initial request as well as all retries fail, `response.success` will be `false` and we'll be able to grab a `message` explaining what went wrong. In the ideal case, `response.success` will be `true` and we'll be able to access our well-typed `data` property! + +## Accessing Failed Output + +A failed response may also provide `data: unknown` and `responseText: string`. +The data is the rejected value, not a validated `SentimentResponse`. +The response text is the original model output, including when it cannot be parsed as JSON. + +```ts +const response = await translator.translate(request); +if (!response.success) { + const originalResponse = response.responseText; + if ("data" in response) { + const rejected: unknown = response.data; + // Validate or repair this value before using it as a SentimentResponse. + } +} +``` + +Use `success` to distinguish outcomes, not the presence of `data`. +TypeScript and Zod validators also retain rejected inputs when used directly. +Custom validators can return `error(message, { data, responseText })`, and the translator preserves +their payloads. To manage validation repairs yourself, set `translator.attemptRepair = false`. + +If automatic repair fails, TypeChat retains the latest rejected candidate and available model text, +not a full attempt history. These can refer to different attempts if the repair response could not +be parsed. A network failure before any output has no candidate data. +Rejected values and response text may contain sensitive output; do not log them automatically. This will correspond to the type that we passed in when we created our translator object (i.e. `SentimentResponse`). That's it! diff --git a/typescript/README.md b/typescript/README.md index 4ed66add..cc4df6b9 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -40,6 +40,34 @@ To see TypeChat in action, we recommend exploring the [TypeChat example projects To learn more about TypeChat, visit the [documentation](https://microsoft.github.io/TypeChat) which includes more information on TypeChat and how to get started. +## Accessing failed output + +Failed results include a `message` and may also include `data: unknown` and `responseText: string`. +Validators return the rejected input in `data`. Translators also retain the original model text, +including when that text is not valid JSON: + +```ts +const result = await translator.translate(request); +if (!result.success) { + const rejected: unknown = result.data; + const originalResponse = result.responseText; + // Validate or repair the rejected value before treating it as the target type. +} +``` + +On final failure, automatic repair retains the latest rejected candidate and available response text, +not a full attempt history. If a repair response cannot be parsed, `data` can come from an earlier +attempt than `responseText`. An initial network failure has no candidate data. Successful results, +error messages, and repair limits are unchanged; `getData` still throws on failure. + +Use `success` to distinguish successful and failed results, not the presence of `data`. +Custom validators can supply their own failure payloads, which the translator preserves. +To create such a failure, use `error(message, { data, responseText })`; `error(message)` still returns +the original message-only shape. A present `data` property can contain any value, including `null` +or `undefined`, so check property presence when that distinction matters. + +Rejected values and response text may contain sensitive output. Do not log them automatically. + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/typescript/package.json b/typescript/package.json index 1654f47e..0c58996a 100644 --- a/typescript/package.json +++ b/typescript/package.json @@ -21,7 +21,7 @@ }, "scripts": { "build": "tsc -p src", - "test": "npm run build && tsc -p test && node --test out/validate.test.js out/zod.test.js out/program.test.js tests/model.test.mjs tests/examples.test.mjs", + "test": "npm run build && tsc -p test && node --test out/validate.test.js out/zod.test.js out/program.test.js out/failure.test.js tests/model.test.mjs tests/examples.test.mjs", "build-all": "npm run build --workspaces", "prepare": "npm run build-all", "prepublishOnly": "node -e \"require('fs').copyFileSync('../SECURITY.md','SECURITY.md')\"", diff --git a/typescript/src/result.ts b/typescript/src/result.ts index 7ec8c078..d2b8f4c0 100644 --- a/typescript/src/result.ts +++ b/typescript/src/result.ts @@ -6,7 +6,20 @@ export type Success = { success: true, data: T }; /** * An object representing an operation that failed for the reason given in `message`. */ -export type Error = { success: false, message: string }; +export type Error = { + success: false; + message: string; + /** + * An unvalidated value associated with the failure, when available. A translation retains the + * latest rejected candidate, which may precede a repair response that couldn't be parsed. + */ + data?: unknown; + /** + * The latest model response text, when available. This may come from a different attempt than + * `data` if a repair response couldn't be parsed. + */ + responseText?: string; +}; /** * An object representing a successful or failed operation of type `T`. @@ -25,10 +38,21 @@ export function success(data: T): Success { /** * Returns an `Error` object. * @param message The value for the `message` property of the result. + * @param details Optional unvalidated data and original model response text. * @returns An `Error` object. */ -export function error(message: string): Error { - return { success: false, message }; +export function error(message: string, details?: Pick): Error { + const result: Error = { success: false, message }; + if (details) { + if ("data" in details) { + result.data = details.data; + } + const responseText = details.responseText; + if (responseText !== undefined) { + result.responseText = responseText; + } + } + return result; } /** diff --git a/typescript/src/ts/program.ts b/typescript/src/ts/program.ts index 5fc21b5d..c19d01f8 100644 --- a/typescript/src/ts/program.ts +++ b/typescript/src/ts/program.ts @@ -99,7 +99,7 @@ function isValidFunctionName(name: unknown): name is string { export function createModuleTextFromProgram(jsonObject: object): Result { const steps = (jsonObject as Program)["@steps"]; if (!(Array.isArray(steps) && steps.every(step => typeof step === "object" && step !== null && step.hasOwnProperty("@func")))) { - return error("JSON object is not a valid program"); + return error("JSON object is not a valid program", { data: jsonObject }); } let hasError = false; let functionBody = ""; @@ -109,7 +109,7 @@ export function createModuleTextFromProgram(jsonObject: object): Result currentStep++; } return hasError ? - error("JSON program contains an invalid expression") : + error("JSON program contains an invalid expression", { data: jsonObject }) : success(`import { API } from "./schema";\nfunction program(api: API) {\n${functionBody}}`); function exprToString(expr: unknown): string { diff --git a/typescript/src/ts/validate.ts b/typescript/src/ts/validate.ts index b2b0b9c5..be202b74 100644 --- a/typescript/src/ts/validate.ts +++ b/typescript/src/ts/validate.ts @@ -89,16 +89,21 @@ export function createTypeScriptJsonValidator(schema: }; return validator; - function validate(jsonObject: object) { + function validate(jsonObject: object): Result { const moduleResult = validator.createModuleTextFromJson(jsonObject); if (!moduleResult.success) { - return moduleResult; + return "data" in moduleResult ? moduleResult : { + ...moduleResult, + success: false, + message: moduleResult.message, + data: jsonObject + }; } writeJsonFile(jsonFileName, moduleResult.data); const snapshot = api.updateSnapshot({ fileChanges: { changed: [jsonFileName] } }); const project = snapshot.getProject(configFileName); if (!project) { - return error(`The TypeScript schema project '${configFileName}' couldn't be loaded.`); + return error(`The TypeScript schema project '${configFileName}' couldn't be loaded.`, { data: jsonObject }); } const program = project.program; const syntacticDiagnostics = program.getSyntacticDiagnostics(); @@ -115,7 +120,7 @@ export function createTypeScriptJsonValidator(schema: } return message; }).join("\n"); - return error(diagnostics); + return error(diagnostics, { data: jsonObject }); } return success(jsonObject as T); } diff --git a/typescript/src/typechat.ts b/typescript/src/typechat.ts index 74c67331..af7a6a8b 100644 --- a/typescript/src/typechat.ts +++ b/typescript/src/typechat.ts @@ -1,4 +1,4 @@ -import { Result, success, error } from "./result"; +import { Result, success, error, Error as TypeChatError } from "./result"; import { TypeChatLanguageModel, PromptSection, PromptContent } from "./model"; /** @@ -55,6 +55,8 @@ export interface TypeChatJsonTranslator { * the language model fails to validate and the `attemptRepair` property is `true`, a second * attempt to translate the request will be made. The prompt for the second attempt will include the * diagnostics produced for the first attempt. This often helps produce a valid instance. + * A failure retains the latest rejected candidate and available model response text. These may + * come from different attempts if a repair response couldn't be parsed. * @param request The natural language request. * @param promptPreamble An optional string or array of prompt sections to prepend to the generated * prompt. If a string is specified, it is converted into a single "user" role prompt section. @@ -78,7 +80,7 @@ export interface TypeChatJsonValidator { /** * Validates the given JSON object according to the associated TypeScript schema. Returns a * `Success` object containing the JSON object if validation was successful. Otherwise, returns - * an `Error` object with a `message` property describing the error. + * an `Error` object with a `message` describing the error and, when available, rejected `data`. * @param jsonText The JSON object to validate. * @returns The JSON object or an error message. */ @@ -121,20 +123,29 @@ export function createJsonTranslator(model: TypeChatLanguageMo `The following is a revised JSON object:\n`; } - async function translate(request: string, promptPreamble?: string | PromptSection[]) { + async function translate(request: string, promptPreamble?: string | PromptSection[]): Promise> { const preamble: PromptSection[] = typeof promptPreamble === "string" ? [{ role: "user", content: promptPreamble }] : promptPreamble ?? []; let prompt: PromptSection[] = [...preamble, { role: "user", content: typeChat.createRequestPrompt(request) }]; let attemptRepair = typeChat.attemptRepair; + let failureContext: (Pick & { responseText: string }) | undefined; while (true) { const response = await model.complete(prompt); if (!response.success) { - return response; + return failureContext ? { + ...failureContext, + ...response, + success: false, + message: response.message, + ...("data" in response ? { data: response.data } : {}), + responseText: response.responseText ?? failureContext.responseText + } : response; } const responseText = response.data; + failureContext = { ...failureContext, responseText }; const startIndex = responseText.indexOf("{"); const endIndex = responseText.lastIndexOf("}"); if (!(startIndex >= 0 && endIndex > startIndex)) { - return error(`Response is not JSON:\n${responseText}`); + return error(`Response is not JSON:\n${responseText}`, failureContext); } const jsonText = responseText.slice(startIndex, endIndex + 1); let jsonObject; @@ -142,7 +153,7 @@ export function createJsonTranslator(model: TypeChatLanguageMo jsonObject = JSON.parse(jsonText) as object; } catch (e) { - return error(e instanceof SyntaxError ? e.message : "JSON parse error"); + return error(e instanceof SyntaxError ? e.message : "JSON parse error", failureContext); } if (typeChat.stripNulls) { stripNulls(jsonObject); @@ -152,11 +163,19 @@ export function createJsonTranslator(model: TypeChatLanguageMo if (validation.success) { return validation; } + const failure: TypeChatError & { responseText: string } = { + ...validation, + success: false, + message: validation.message, + data: "data" in validation ? validation.data : schemaValidation.success ? schemaValidation.data : jsonObject, + responseText: validation.responseText ?? responseText + }; + failureContext = { data: failure.data, responseText: failure.responseText }; if (!attemptRepair) { - return error(`JSON validation failed: ${validation.message}\n${jsonText}`); + return { ...failure, message: `JSON validation failed: ${failure.message}\n${jsonText}` }; } prompt.push({ role: "assistant", content: responseText }); - prompt.push({ role: "user", content: typeChat.createRepairPrompt(validation.message) }); + prompt.push({ role: "user", content: typeChat.createRepairPrompt(failure.message) }); attemptRepair = false; } } diff --git a/typescript/src/zod/validate.ts b/typescript/src/zod/validate.ts index 02e5bb98..4babb5cc 100644 --- a/typescript/src/zod/validate.ts +++ b/typescript/src/zod/validate.ts @@ -23,7 +23,7 @@ export function createZodJsonValidator, K ex function validate(jsonObject: object) { const result = schema[typeName].safeParse(jsonObject); if (!result.success) { - return error(result.error.issues.map(({ path, message }) => `${path.map(key => `[${JSON.stringify(key)}]`).join("")}: ${message}`).join("\n")); + return error(result.error.issues.map(({ path, message }) => `${path.map(key => `[${JSON.stringify(key)}]`).join("")}: ${message}`).join("\n"), { data: jsonObject }); } return success(result.data as z.infer & object); } diff --git a/typescript/test/failure.test.ts b/typescript/test/failure.test.ts new file mode 100644 index 00000000..5a5666d1 --- /dev/null +++ b/typescript/test/failure.test.ts @@ -0,0 +1,323 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { z } from "zod"; +import { + createJsonTranslator, error, getData, success, + Result, TypeChatJsonValidator, TypeChatLanguageModel, PromptSection +} from "../dist/index.js"; +import { createModuleTextFromProgram, createTypeScriptJsonValidator } from "../dist/ts/index.js"; +import { createZodJsonValidator } from "../dist/zod/index.js"; + +type Example = { value: number }; +type IsUnknown = unknown extends T ? keyof T extends never ? true : false : false; + +class GetterFailure { + get success(): false { return false; } + get message() { return "custom"; } + get data(): unknown { return null; } + get responseText() { return ""; } +} + +function exampleValidator(): TypeChatJsonValidator { + return { + getSchemaText: () => "export interface Example { value: number; }", + getTypeName: () => "Example", + validate: value => "value" in value && typeof value.value === "number" ? + success({ value: value.value }) : error("value must be a number") + }; +} + +function translatorFor(responses: Result[], validator = exampleValidator()) { + const prompts: (string | PromptSection[])[] = []; + let index = 0; + const model: TypeChatLanguageModel = { + async complete(prompt) { + prompts.push(typeof prompt === "string" ? prompt : [...prompt]); + const response = responses[index++]; + if (!response) { + throw new Error("Unexpected model call"); + } + return response; + } + }; + return { translator: createJsonTranslator(model, validator), prompts }; +} + +describe("failure result contract", () => { + it("preserves the shape of message-only failures", () => { + assert.deepEqual(error("failed"), { success: false, message: "failed" }); + }); + + for (const data of [null, false, 0, "", undefined]) { + it(`retains an explicitly supplied ${String(data)} payload`, () => { + const result = error("failed", { data }); + assert.ok("data" in result); + assert.equal(result.data, data); + }); + } + + it("keeps failure data unknown and getData throwing", () => { + const result = error("invalid", { data: { value: "invalid" } }); + const isUnknown: IsUnknown = true; + assert.equal(isUnknown, true); + assert.throws(() => getData(result), /invalid/); + }); + + it("preserves typed successful results", () => { + const result = success({ value: 42 }); + const value: number = result.data.value; + assert.equal(value, 42); + assert.deepEqual(getData(result), { value: 42 }); + }); + + it("retains an empty response text", () => { + assert.equal(error("failed", { responseText: "" }).responseText, ""); + }); + + it("accepts failure details supplied through getters", () => { + assert.deepEqual(error("failed", new GetterFailure()), { + success: false, message: "failed", data: null, responseText: "" + }); + }); +}); + +describe("direct validator failure payloads", () => { + it("retains the TypeScript validator's rejected object", () => { + const validator = createTypeScriptJsonValidator( + "export interface Example { value: number; }", "Example" + ); + try { + const input = { value: "invalid" }; + const result = validator.validate(input); + assert.ok(!result.success); + assert.equal(result.data, input); + assert.ok(!("responseText" in result)); + assert.match(result.message, /not assignable/); + } + finally { + validator.close(); + } + }); + + it("preserves module-conversion failure metadata", () => { + const validator = createTypeScriptJsonValidator( + "export interface Example { value: number; }", "Example" + ); + const original = error("conversion failed", { data: null, responseText: "custom" }); + validator.createModuleTextFromJson = () => original; + try { + assert.deepEqual(validator.validate({ value: 1 }), original); + } + finally { + validator.close(); + } + }); + + it("adds the input to message-only module-conversion failures", () => { + const validator = createTypeScriptJsonValidator( + "export interface Example { value: number; }", "Example" + ); + validator.createModuleTextFromJson = () => error("conversion failed"); + try { + const input = { value: 1 }; + const result = validator.validate(input); + assert.ok(!result.success); + assert.equal(result.data, input); + } + finally { + validator.close(); + } + }); + + it("retains the Zod validator's rejected object", () => { + const validator = createZodJsonValidator({ Example: z.object({ value: z.number() }) }, "Example"); + const input = { value: "invalid" }; + const result = validator.validate(input); + assert.ok(!result.success); + assert.equal(result.data, input); + }); + + for (const input of [ + { "@steps": "invalid" }, + { "@steps": [{ "@func": "valid", "@args": [{ "@ref": -1 }] }] } + ]) { + it("retains a rejected program during module conversion", () => { + const result = createModuleTextFromProgram(input); + assert.ok(!result.success); + assert.equal(result.data, input); + }); + } +}); + +describe("translation failure context", () => { + it("preserves immediate successful results", async () => { + const { translator, prompts } = translatorFor([success('{"value":42}')]); + assert.deepEqual(await translator.translate("request"), success({ value: 42 })); + assert.equal(prompts.length, 1); + }); + + it("returns rejected data and the original response without repair", async () => { + const responseText = 'before {"value":"invalid"} after'; + const { translator, prompts } = translatorFor([success(responseText)]); + translator.attemptRepair = false; + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.deepEqual(result.data, { value: "invalid" }); + assert.equal(result.responseText, responseText); + assert.equal(result.message, 'JSON validation failed: value must be a number\n{"value":"invalid"}'); + assert.equal(prompts.length, 1); + }); + + for (const data of [null, false, 0, "", undefined]) { + it(`preserves a custom validator's ${String(data)} payload and metadata`, async () => { + const original = { ...error("custom", { data, responseText: "" }), code: "CUSTOM" }; + const validator = { ...exampleValidator(), validate: () => original }; + const { translator } = translatorFor([success('{"value":1}')], validator); + translator.attemptRepair = false; + const result = await translator.translate("request"); + assert.deepEqual(result, { ...original, message: 'JSON validation failed: custom\n{"value":1}' }); + assert.equal(original.message, "custom"); + }); + } + + it("does not mutate message-only custom failures", async () => { + const original = error("custom"); + const validator = { ...exampleValidator(), validate: () => original }; + const { translator } = translatorFor([success('{"value":1}')], validator); + translator.attemptRepair = false; + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.deepEqual(result.data, { value: 1 }); + assert.deepEqual(original, error("custom")); + }); + + it("preserves a custom validator's inherited failure fields", async () => { + const validator = { ...exampleValidator(), validate: () => new GetterFailure() }; + const { translator } = translatorFor([success('{"value":1}')], validator); + translator.attemptRepair = false; + assert.deepEqual(await translator.translate("request"), { + success: false, message: 'JSON validation failed: custom\n{"value":1}', data: null, responseText: "" + }); + }); + + it("retains the transformed instance rejected by additional validation", async () => { + const validator = createZodJsonValidator({ Example: z.object({ value: z.coerce.number() }) }, "Example"); + const responseText = '{"value":"42"}'; + const { translator } = translatorFor([success(responseText)], validator); + translator.attemptRepair = false; + translator.validateInstance = () => error("instance rejected"); + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.deepEqual(result.data, { value: 42 }); + assert.equal(result.responseText, responseText); + }); + + it("distinguishes preprocessed data from the original response", async () => { + const responseText = '{"value":"invalid","optional":null}'; + const { translator } = translatorFor([success(responseText)]); + translator.attemptRepair = false; + translator.stripNulls = true; + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.deepEqual(result.data, { value: "invalid" }); + assert.equal(result.responseText, responseText); + }); + + for (const responseText of ["not JSON", 'before {"value":} after', ""]) { + it("retains unparseable output without inventing data or changing repair behavior", async () => { + const { translator, prompts } = translatorFor([success(responseText)]); + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.ok(!("data" in result)); + assert.equal(result.responseText, responseText); + assert.equal(prompts.length, 1); + }); + } + + it("propagates an initial model failure unchanged", async () => { + const original = error("network failure"); + const { translator } = translatorFor([original]); + assert.equal(await translator.translate("request"), original); + }); + + it("does not convert validator exceptions to failures", async () => { + const validator = { + ...exampleValidator(), + validate: (): Result => { throw new Error("validator bug"); } + }; + const { translator } = translatorFor([success('{"value":1}')], validator); + await assert.rejects(translator.translate("request"), /validator bug/); + }); +}); + +describe("repair failure context", () => { + it("preserves a successful repair and the existing repair prompt", async () => { + const first = '{"value":"invalid"}'; + const { translator, prompts } = translatorFor([success(first), success('{"value":42}')]); + assert.deepEqual(await translator.translate("request"), success({ value: 42 })); + assert.equal(prompts.length, 2); + const repairPrompt = prompts[1]; + assert.ok(Array.isArray(repairPrompt)); + assert.equal(repairPrompt[1].content, first); + assert.match(String(repairPrompt[2].content), /value must be a number/); + }); + + it("returns the latest rejected candidate after exactly one repair", async () => { + const responseText = '{"value":"second"}'; + const { translator, prompts } = translatorFor([ + success('{"value":"first"}'), success(responseText) + ]); + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.deepEqual(result.data, { value: "second" }); + assert.equal(result.responseText, responseText); + assert.equal(prompts.length, 2); + }); + + it("retains the previous candidate when the repair request fails", async () => { + const responseText = '{"value":"invalid"}'; + const original = error("network failure"); + const { translator } = translatorFor([success(responseText), original]); + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.equal(result.message, "network failure"); + assert.deepEqual(result.data, { value: "invalid" }); + assert.equal(result.responseText, responseText); + assert.deepEqual(original, error("network failure")); + }); + + it("preserves context explicitly supplied by the model", async () => { + const original = error("model failure", { data: null, responseText: "" }); + const { translator } = translatorFor([success('{"value":"invalid"}'), original]); + assert.deepEqual(await translator.translate("request"), original); + }); + + it("preserves a model's inherited failure fields", async () => { + const { translator } = translatorFor([success('{"value":"invalid"}'), new GetterFailure()]); + assert.deepEqual(await translator.translate("request"), { + success: false, message: "custom", data: null, responseText: "" + }); + }); + + it("does not leak failure context into another translation", async () => { + const original = error("new request failed"); + const { translator } = translatorFor([ + success('{"value":"invalid"}'), error("repair failed"), original + ]); + await translator.translate("first"); + assert.equal(await translator.translate("second"), original); + }); + + for (const responseText of ["not JSON", '{"value":}']) { + it("retains the previous candidate when the repair response cannot be parsed", async () => { + const { translator, prompts } = translatorFor([ + success('{"value":"invalid"}'), success(responseText) + ]); + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.deepEqual(result.data, { value: "invalid" }); + assert.equal(result.responseText, responseText); + assert.equal(prompts.length, 2); + }); + } +});