diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 790aff0..87e4901 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,14 @@ on: # Also turn on "Require approval for all outside collaborators" in repo Actions # settings before making this repo public. jobs: + benchmark-tools: + name: benchmark tooling (unit tests) + if: github.event_name == 'push' || (github.event.pull_request.head.repo.full_name == github.repository && (github.base_ref == 'main' || github.base_ref == 'dev')) + runs-on: [self-hosted, macOS, ARM64, m4pro] + steps: + - uses: actions/checkout@v4 + - run: python3 -m unittest discover -s benchmarks/tests -v + base-convert: name: base-convert (build + test) if: github.event_name == 'push' || (github.event.pull_request.head.repo.full_name == github.repository && (github.base_ref == 'main' || github.base_ref == 'dev')) diff --git a/.github/workflows/serve-smoke.yml b/.github/workflows/serve-smoke.yml index a498f29..2887247 100644 --- a/.github/workflows/serve-smoke.yml +++ b/.github/workflows/serve-smoke.yml @@ -1,6 +1,6 @@ name: Server smoke -# Manually-triggered: start baseRT_serve from an engine release and assert it +# Manually-triggered: start basert-serve from an engine release and assert it # answers an OpenAI /v1/chat/completions request. Needs an engine release and a # .base model URL. @@ -10,6 +10,11 @@ on: model_url: description: "URL to a .base model" required: true + run_tool_state_smoke: + description: "Run sequential/concurrent tool-state regression checks" + required: false + type: boolean + default: false jobs: serve: @@ -20,18 +25,20 @@ jobs: run: | mkdir -p build gh release download --repo ${{ github.repository }} \ - --pattern 'baseRT-engine-macos-arm64*.tar.gz' -D build - tar -xzf build/baseRT-engine-macos-arm64*.tar.gz -C build + --pattern 'basert-engine-macos-arm64*.tar.gz' -D build + tar -xzf build/basert-engine-macos-arm64*.tar.gz -C build env: GH_TOKEN: ${{ github.token }} - name: Fetch model run: | mkdir -p models - curl -fsSL "${{ inputs.model_url }}" -o models/model.base - - name: Start server + assert a completion + curl -fsSL "$MODEL_URL" -o models/model.base + env: + MODEL_URL: ${{ inputs.model_url }} + - name: Start server + run smoke checks run: | KEY="ci-$RANDOM" - ./build/baseRT_serve --model models/model.base --api-key "$KEY" --port 8080 \ + ./build/basert-serve models/model.base --api-key "$KEY" --port 8080 \ > serve.log 2>&1 & SV=$! trap 'kill $SV 2>/dev/null || true' EXIT @@ -44,3 +51,11 @@ jobs: http://127.0.0.1:8080/v1/chat/completions) echo "$RESP" echo "$RESP" | python3 -c "import sys,json; c=json.load(sys.stdin)['choices'][0]['message']['content']; assert c, 'empty completion'; print('OK:', repr(c[:60]))" + if [[ "$RUN_TOOL_STATE_SMOKE" == "true" ]]; then + python3 benchmarks/scripts/tool_state_smoke.py \ + --base-url http://127.0.0.1:8080/v1 \ + --model model.base \ + --api-key "$KEY" + fi + env: + RUN_TOOL_STATE_SMOKE: ${{ inputs.run_tool_state_smoke }} diff --git a/benchmarks/README.md b/benchmarks/README.md index 7c0d7c1..0b84c4a 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -23,6 +23,36 @@ Results are written to `benchmarks/results/_baseRT.csv` with columns throughput at prompt length N; `tgN` rows are decode (token-generation) throughput. +## Tool-state regression smoke + +`tool_state_smoke.py` checks whether repeated tool schemas preserve isolated +arguments between requests. It sends distinct `lookup_key` sentinels in 10 +sequential requests, then concurrent batches of 2 and 4 by default. Failures are +consistent with cross-request state leakage, but this diagnostic does not prove +an engine root cause by itself. It is opt-in for tool-capable models; plain chat +models are expected to fail it: + +```sh +python3 benchmarks/scripts/tool_state_smoke.py \ + --base-url http://127.0.0.1:8080/v1 \ + --model model.base \ + --sequential 10 --concurrency 2 4 \ + --timeout 300 --max-tokens 2048 +``` + +Set `BASERT_API_KEY` in the environment when the server requires a bearer +token. The harness uses only the Python standard library. Each mismatch is +printed as a `FAIL` JSON record, followed by a `SUMMARY` JSON record, and any +mismatch makes the process exit nonzero. + +The manual **Server smoke** workflow can run this check after its basic chat +completion by enabling `run_tool_state_smoke`. Pure unit tests use a local fake +HTTP server and do not require an engine or model: + +```sh +python3 -m unittest benchmarks.tests.test_tool_state_smoke -v +``` + ## Example results `results/m4-pro_baseRT.csv` and `results/m3-base_baseRT.csv` hold reference diff --git a/benchmarks/scripts/tool_state_smoke.py b/benchmarks/scripts/tool_state_smoke.py new file mode 100644 index 0000000..214b99d --- /dev/null +++ b/benchmarks/scripts/tool_state_smoke.py @@ -0,0 +1,402 @@ +#!/usr/bin/env python3 +"""Exercise OpenAI-compatible tool calls across reused and concurrent requests.""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import socket +import sys +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +from concurrent.futures import ThreadPoolExecutor +from dataclasses import asdict, dataclass +from typing import Any, Mapping, Sequence + +_TOOL_NAME = "lookup_key" +_DEFAULT_BASE_URL = "http://127.0.0.1:8080/v1" + + +@dataclass(frozen=True) +class Validation: + errors: tuple[str, ...] + finish_reason: Any = None + tool_call_count: int | None = None + function_name: Any = None + arguments: Any = None + + @property + def ok(self) -> bool: + return not self.errors + + +@dataclass(frozen=True) +class CaseResult: + phase: str + index: int + expected_key: str + ok: bool + elapsed_ms: int + http_status: int | None + finish_reason: Any + tool_call_count: int | None + function_name: Any + arguments: Any + errors: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + result = asdict(self) + result["errors"] = list(self.errors) + return result + + +@dataclass(frozen=True) +class Config: + base_url: str + model: str + api_key: str | None + sequential: int + concurrency: tuple[int, ...] + timeout: float + max_tokens: int + + +def build_request(model: str, sentinel: str, max_tokens: int) -> dict[str, Any]: + """Build the repeated-schema request used to expose cross-request state leaks.""" + return { + "model": model, + "messages": [ + { + "role": "user", + "content": f"Call {_TOOL_NAME} with key {sentinel}.", + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": _TOOL_NAME, + "description": "Lookup a key", + "parameters": { + "type": "object", + "properties": {"key": {"type": "string"}}, + "required": ["key"], + "additionalProperties": False, + }, + }, + } + ], + "max_tokens": max_tokens, + "temperature": 0, + } + + +def validate_response(payload: Any, expected_key: str) -> Validation: + """Validate one complete non-streaming chat-completion response.""" + errors: list[str] = [] + if not isinstance(payload, Mapping): + return Validation(("response must be a JSON object",)) + + choices = payload.get("choices") + if not isinstance(choices, list) or not choices: + return Validation(("response must contain at least one choice",)) + + choice = choices[0] + if not isinstance(choice, Mapping): + return Validation(("choice must be a JSON object",)) + + finish_reason = choice.get("finish_reason") + if finish_reason != "tool_calls": + errors.append( + f"expected finish_reason 'tool_calls', got {finish_reason!r}" + ) + + message = choice.get("message") + if not isinstance(message, Mapping): + errors.append("choice.message must be a JSON object") + return Validation(tuple(errors), finish_reason=finish_reason) + + tool_calls = message.get("tool_calls") + call_count = len(tool_calls) if isinstance(tool_calls, list) else 0 + if not isinstance(tool_calls, list) or call_count != 1: + errors.append(f"expected exactly one tool call, got {call_count}") + return Validation( + tuple(errors), + finish_reason=finish_reason, + tool_call_count=call_count, + ) + + call = tool_calls[0] + if not isinstance(call, Mapping): + errors.append("tool call must be a JSON object") + return Validation( + tuple(errors), + finish_reason=finish_reason, + tool_call_count=call_count, + ) + + function = call.get("function") + if not isinstance(function, Mapping): + errors.append("tool call function must be a JSON object") + return Validation( + tuple(errors), + finish_reason=finish_reason, + tool_call_count=call_count, + ) + + function_name = function.get("name") + if function_name != _TOOL_NAME: + errors.append( + f"expected function name {_TOOL_NAME!r}, got {function_name!r}" + ) + + raw_arguments = function.get("arguments") + arguments: Any = None + arguments_decoded = False + if not isinstance(raw_arguments, str): + errors.append("tool arguments must be a JSON-encoded string") + else: + try: + arguments = json.loads(raw_arguments) + arguments_decoded = True + except json.JSONDecodeError as exc: + errors.append(f"tool arguments are not valid JSON: {exc.msg}") + + if arguments_decoded: + if not isinstance(arguments, Mapping): + errors.append("decoded tool arguments must be a JSON object") + elif arguments != {"key": expected_key}: + errors.append( + f"expected arguments {{'key': {expected_key!r}}}, got {arguments!r}" + ) + + return Validation( + tuple(errors), + finish_reason=finish_reason, + tool_call_count=call_count, + function_name=function_name, + arguments=arguments, + ) + + +def _positive_int(value: str) -> int: + parsed = int(value) + if parsed <= 0: + raise argparse.ArgumentTypeError("must be greater than zero") + return parsed + + +def _nonnegative_int(value: str) -> int: + parsed = int(value) + if parsed < 0: + raise argparse.ArgumentTypeError("must be zero or greater") + return parsed + + +def _positive_float(value: str) -> float: + parsed = float(value) + if not math.isfinite(parsed) or parsed <= 0: + raise argparse.ArgumentTypeError("must be a finite number greater than zero") + return parsed + + +def _http_base_url(value: str) -> str: + parsed = urllib.parse.urlparse(value) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise argparse.ArgumentTypeError("must be an absolute HTTP or HTTPS URL") + return value + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Check that sequential shared-prefix and concurrent tool requests " + "preserve isolated arguments." + ) + ) + parser.add_argument( + "--base-url", + type=_http_base_url, + default=os.environ.get("BASERT_BASE_URL", _DEFAULT_BASE_URL), + help=f"OpenAI-compatible API base URL (default: {_DEFAULT_BASE_URL})", + ) + parser.add_argument( + "--model", + default=os.environ.get("BASERT_MODEL", "model.base"), + help="model name sent in requests (default: model.base)", + ) + parser.add_argument( + "--api-key", + default=os.environ.get("BASERT_API_KEY"), + help="Bearer token; defaults to BASERT_API_KEY when set", + ) + parser.add_argument( + "--sequential", + type=_nonnegative_int, + default=10, + help="number of sequential requests (default: 10)", + ) + parser.add_argument( + "--concurrency", + type=_positive_int, + nargs="+", + default=[2, 4], + metavar="N", + help="concurrent batch sizes (default: 2 4)", + ) + parser.add_argument( + "--timeout", + type=_positive_float, + default=300.0, + help="per-request timeout in seconds (default: 300)", + ) + parser.add_argument( + "--max-tokens", + type=_positive_int, + default=2048, + help="maximum completion tokens per request (default: 2048)", + ) + return parser + + +def _endpoint(base_url: str) -> str: + base_url = base_url.rstrip("/") + if base_url.endswith("/chat/completions"): + return base_url + return f"{base_url}/chat/completions" + + +def _run_case(config: Config, phase: str, index: int, sentinel: str) -> CaseResult: + body = json.dumps( + build_request(config.model, sentinel, config.max_tokens), + separators=(",", ":"), + ).encode("utf-8") + headers = {"Content-Type": "application/json"} + if config.api_key: + headers["Authorization"] = f"Bearer {config.api_key}" + request = urllib.request.Request( + _endpoint(config.base_url), data=body, headers=headers, method="POST" + ) + + started = time.monotonic() + status: int | None = None + try: + with urllib.request.urlopen(request, timeout=config.timeout) as response: + status = response.status + response_body = response.read() + try: + payload = json.loads(response_body) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + validation = Validation((f"response is not valid JSON: {exc}",)) + else: + validation = validate_response(payload, sentinel) + except urllib.error.HTTPError as exc: + status = exc.code + detail = exc.read().decode("utf-8", errors="replace")[:500] + validation = Validation((f"HTTP {exc.code}: {detail}",)) + except (urllib.error.URLError, TimeoutError, socket.timeout) as exc: + validation = Validation((f"request failed: {exc}",)) + except OSError as exc: + validation = Validation((f"request failed: {exc}",)) + + elapsed_ms = round((time.monotonic() - started) * 1000) + return CaseResult( + phase=phase, + index=index, + expected_key=sentinel, + ok=validation.ok, + elapsed_ms=elapsed_ms, + http_status=status, + finish_reason=validation.finish_reason, + tool_call_count=validation.tool_call_count, + function_name=validation.function_name, + arguments=validation.arguments, + errors=validation.errors, + ) + + +def _run_concurrent(config: Config, size: int) -> list[CaseResult]: + phase = f"concurrency-{size}" + barrier = threading.Barrier(size) + + def worker(index: int) -> CaseResult: + try: + barrier.wait(timeout=config.timeout) + except threading.BrokenBarrierError: + return CaseResult( + phase=phase, + index=index, + expected_key=f"c{size}-{index:04d}", + ok=False, + elapsed_ms=0, + http_status=None, + finish_reason=None, + tool_call_count=None, + function_name=None, + arguments=None, + errors=("concurrency barrier timed out",), + ) + return _run_case(config, phase, index, f"c{size}-{index:04d}") + + with ThreadPoolExecutor(max_workers=size) as executor: + futures = [executor.submit(worker, index) for index in range(size)] + return [future.result() for future in futures] + + +def _print_phase(phase: str, results: Sequence[CaseResult]) -> None: + failed = [result for result in results if not result.ok] + for result in failed: + print("FAIL " + json.dumps(result.to_dict(), sort_keys=True)) + passed = len(results) - len(failed) + label = "PASS" if not failed else "FAILED" + print(f"{label} {phase}: {passed}/{len(results)} requests passed") + + +def run(config: Config) -> list[CaseResult]: + all_results: list[CaseResult] = [] + + sequential_results = [ + _run_case(config, "sequential", index, f"seq-{index:04d}") + for index in range(config.sequential) + ] + _print_phase("sequential", sequential_results) + all_results.extend(sequential_results) + + for size in config.concurrency: + concurrent_results = _run_concurrent(config, size) + _print_phase(f"concurrency-{size}", concurrent_results) + all_results.extend(concurrent_results) + + return all_results + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + config = Config( + base_url=args.base_url, + model=args.model, + api_key=args.api_key, + sequential=args.sequential, + concurrency=tuple(args.concurrency), + timeout=args.timeout, + max_tokens=args.max_tokens, + ) + results = run(config) + failed = sum(not result.ok for result in results) + summary = { + "ok": failed == 0, + "total": len(results), + "passed": len(results) - failed, + "failed": failed, + } + print("SUMMARY " + json.dumps(summary, sort_keys=True)) + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/tests/test_tool_state_smoke.py b/benchmarks/tests/test_tool_state_smoke.py new file mode 100644 index 0000000..52b366e --- /dev/null +++ b/benchmarks/tests/test_tool_state_smoke.py @@ -0,0 +1,344 @@ +from __future__ import annotations + +import contextlib +import io +import json +import sys +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +_SCRIPTS_DIR = Path(__file__).resolve().parents[1] / "scripts" +if str(_SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPTS_DIR)) + +import tool_state_smoke # type: ignore[import-not-found] # noqa: E402 + + +def _response(key: str, *, name: str = "lookup_key", finish_reason: str = "tool_calls") -> dict: + return { + "choices": [ + { + "finish_reason": finish_reason, + "message": { + "tool_calls": [ + { + "id": "call_test", + "type": "function", + "function": { + "name": name, + "arguments": json.dumps({"key": key}), + }, + } + ] + }, + } + ] + } + + +class ToolResponseValidationTests(unittest.TestCase): + def test_accepts_one_exact_lookup_key_call(self) -> None: + result = tool_state_smoke.validate_response(_response("seq-0000"), "seq-0000") + + self.assertTrue(result.ok) + self.assertEqual(result.errors, ()) + self.assertEqual(result.arguments, {"key": "seq-0000"}) + + def test_rejects_non_exact_key_value(self) -> None: + result = tool_state_smoke.validate_response( + _response("prefix-seq-0000-suffix"), "seq-0000" + ) + + self.assertFalse(result.ok) + self.assertTrue(any("expected arguments" in error for error in result.errors)) + + def test_rejects_extra_argument_properties(self) -> None: + payload = _response("seq-0000") + payload["choices"][0]["message"]["tool_calls"][0]["function"][ + "arguments" + ] = json.dumps({"key": "seq-0000", "extra": "stale-c4-0001"}) + + result = tool_state_smoke.validate_response(payload, "seq-0000") + + self.assertFalse(result.ok) + self.assertTrue(any("expected arguments" in error for error in result.errors)) + + def test_rejects_missing_key_property(self) -> None: + payload = _response("seq-0000") + payload["choices"][0]["message"]["tool_calls"][0]["function"][ + "arguments" + ] = "{}" + + result = tool_state_smoke.validate_response(payload, "seq-0000") + + self.assertFalse(result.ok) + self.assertTrue(any("expected arguments" in error for error in result.errors)) + + def test_rejects_wrong_function_name(self) -> None: + result = tool_state_smoke.validate_response( + _response("seq-0000", name="other_lookup"), "seq-0000" + ) + + self.assertFalse(result.ok) + self.assertTrue(any("function name" in error for error in result.errors)) + + def test_rejects_multiple_tool_calls(self) -> None: + payload = _response("seq-0000") + payload["choices"][0]["message"]["tool_calls"].append( + payload["choices"][0]["message"]["tool_calls"][0] + ) + + result = tool_state_smoke.validate_response(payload, "seq-0000") + + self.assertFalse(result.ok) + self.assertTrue(any("exactly one tool call" in error for error in result.errors)) + + def test_rejects_arguments_that_are_not_json(self) -> None: + payload = _response("seq-0000") + payload["choices"][0]["message"]["tool_calls"][0]["function"][ + "arguments" + ] = '{"key":' + + result = tool_state_smoke.validate_response(payload, "seq-0000") + + self.assertFalse(result.ok) + self.assertTrue(any("valid JSON" in error for error in result.errors)) + + def test_rejects_json_null_arguments(self) -> None: + payload = _response("seq-0000") + payload["choices"][0]["message"]["tool_calls"][0]["function"][ + "arguments" + ] = "null" + + result = tool_state_smoke.validate_response(payload, "seq-0000") + + self.assertFalse(result.ok) + self.assertTrue(any("JSON object" in error for error in result.errors)) + + def test_rejects_non_tool_finish_reason(self) -> None: + result = tool_state_smoke.validate_response( + _response("seq-0000", finish_reason="length"), "seq-0000" + ) + + self.assertFalse(result.ok) + self.assertTrue(any("finish_reason" in error for error in result.errors)) + + def test_rejects_malformed_response_shape_without_raising(self) -> None: + result = tool_state_smoke.validate_response({"choices": []}, "seq-0000") + + self.assertFalse(result.ok) + self.assertTrue(result.errors) + + +class RequestTests(unittest.TestCase): + def test_request_uses_lookup_schema_and_exact_sentinel(self) -> None: + payload = tool_state_smoke.build_request("model.base", "seq-0007", 321) + + self.assertEqual(payload["model"], "model.base") + self.assertEqual(payload["max_tokens"], 321) + self.assertEqual(payload["temperature"], 0) + self.assertEqual( + payload["messages"], + [{"role": "user", "content": "Call lookup_key with key seq-0007."}], + ) + function = payload["tools"][0]["function"] + self.assertEqual(function["name"], "lookup_key") + self.assertEqual(function["parameters"]["required"], ["key"]) + self.assertFalse(function["parameters"]["additionalProperties"]) + self.assertEqual( + function["parameters"]["properties"]["key"], {"type": "string"} + ) + + def test_defaults_cover_sequential_and_concurrent_regressions(self) -> None: + args = tool_state_smoke.build_parser().parse_args([]) + + self.assertEqual(args.sequential, 10) + self.assertEqual(args.concurrency, [2, 4]) + + def test_rejects_nonfinite_timeout_and_malformed_base_url(self) -> None: + parser = tool_state_smoke.build_parser() + for value in ("nan", "inf"): + with self.subTest(timeout=value), self.assertRaises(SystemExit): + parser.parse_args(["--timeout", value]) + with self.assertRaises(SystemExit): + parser.parse_args(["--timeout=-inf"]) + for value in ("", "localhost:8080/v1", "ftp://example.com/v1"): + with self.subTest(base_url=value), self.assertRaises(SystemExit): + parser.parse_args(["--base-url", value]) + + +class _FakeState: + def __init__(self, corrupt: bool = False, http_error: bool = False) -> None: + self.corrupt = corrupt + self.http_error = http_error + self.requests: list[dict] = [] + self.authorization: list[str | None] = [] + self.active = 0 + self.max_active = 0 + self.lock = threading.Lock() + + +@contextlib.contextmanager +def _fake_server(*, corrupt: bool = False, http_error: bool = False): + state = _FakeState(corrupt=corrupt, http_error=http_error) + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - stdlib callback name + length = int(self.headers["Content-Length"]) + payload = json.loads(self.rfile.read(length)) + with state.lock: + state.requests.append(payload) + state.authorization.append(self.headers.get("Authorization")) + state.active += 1 + state.max_active = max(state.max_active, state.active) + + try: + time.sleep(0.05) + if state.http_error: + body = b'{"error":{"message":"synthetic failure"}}' + self.send_response(500) + else: + prompt = payload["messages"][0]["content"] + key = prompt.removeprefix("Call lookup_key with key ").removesuffix(".") + if state.corrupt: + key = f"corrupt-{key}" + body = json.dumps(_response(key)).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + finally: + with state.lock: + state.active -= 1 + + def log_message(self, format: str, *args: object) -> None: + del format, args + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield state, f"http://127.0.0.1:{server.server_port}/v1" + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + +class HarnessIntegrationTests(unittest.TestCase): + def test_harness_passes_and_sends_concurrent_requests(self) -> None: + with _fake_server() as (state, base_url), contextlib.redirect_stdout( + io.StringIO() + ) as stdout: + exit_code = tool_state_smoke.main( + [ + "--base-url", + base_url, + "--model", + "model.base", + "--api-key", + "test-key", + "--sequential", + "2", + "--concurrency", + "2", + "--timeout", + "2", + "--max-tokens", + "64", + ] + ) + + self.assertEqual(exit_code, 0) + self.assertEqual(len(state.requests), 4) + sentinels = [ + request["messages"][0]["content"].removeprefix( + "Call lookup_key with key " + ).removesuffix(".") + for request in state.requests + ] + self.assertEqual(len(sentinels), len(set(sentinels))) + self.assertGreaterEqual(state.max_active, 2) + self.assertEqual(state.authorization, ["Bearer test-key"] * 4) + self.assertIn('"ok": true', stdout.getvalue()) + + def test_harness_returns_nonzero_with_machine_readable_failure(self) -> None: + with _fake_server(corrupt=True) as (_state, base_url), contextlib.redirect_stdout( + io.StringIO() + ) as stdout: + exit_code = tool_state_smoke.main( + [ + "--base-url", + base_url, + "--model", + "model.base", + "--sequential", + "1", + "--concurrency", + "2", + "--timeout", + "2", + ] + ) + + self.assertEqual(exit_code, 1) + failure_lines = [ + line.removeprefix("FAIL ") + for line in stdout.getvalue().splitlines() + if line.startswith("FAIL ") + ] + self.assertEqual(len(failure_lines), 3) + failure = json.loads(failure_lines[0]) + self.assertEqual(failure["expected_key"], "seq-0000") + self.assertFalse(failure["ok"]) + self.assertIn("errors", failure) + + def test_http_errors_are_reported_without_a_traceback(self) -> None: + with _fake_server(http_error=True) as (_state, base_url), contextlib.redirect_stdout( + io.StringIO() + ) as stdout: + exit_code = tool_state_smoke.main( + [ + "--base-url", + base_url, + "--model", + "model.base", + "--sequential", + "1", + "--concurrency", + "2", + "--timeout", + "2", + ] + ) + + self.assertEqual(exit_code, 1) + self.assertIn("HTTP 500", stdout.getvalue()) + self.assertNotIn("Traceback", stdout.getvalue()) + + +class WorkflowIntegrationTests(unittest.TestCase): + def test_manual_workflow_can_opt_into_harness(self) -> None: + workflow = ( + Path(__file__).resolve().parents[2] + / ".github" + / "workflows" + / "serve-smoke.yml" + ).read_text() + + self.assertIn("run_tool_state_smoke:", workflow) + self.assertIn("type: boolean", workflow) + self.assertIn("benchmarks/scripts/tool_state_smoke.py", workflow) + self.assertIn("'basert-engine-macos-arm64*.tar.gz'", workflow) + self.assertIn("./build/basert-serve models/model.base", workflow) + self.assertNotIn("baseRT_serve", workflow) + self.assertIn("--base-url http://127.0.0.1:8080/v1", workflow) + self.assertIn("--model model.base", workflow) + + +if __name__ == "__main__": + unittest.main()