diff --git a/.gitignore b/.gitignore index 2f368a4..454f5e5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,9 @@ dist/ build/ *.egg-info/ artifacts/ + +# Node / Playwright UI test artifacts (tests/ui) +node_modules/ +playwright-report/ +test-results/ +.playwright/ diff --git a/src/cas_evals/reference_product.py b/src/cas_evals/reference_product.py index 49ec38b..165c4d2 100644 --- a/src/cas_evals/reference_product.py +++ b/src/cas_evals/reference_product.py @@ -2,6 +2,7 @@ from __future__ import annotations +import concurrent.futures import hashlib import json from collections.abc import Callable @@ -122,9 +123,8 @@ def evaluate_reference_suite( suite = json.loads(fixture_path.read_text(encoding="utf-8")) released_at = suite.get("releasedAt", DEFAULT_RELEASED_AT) invoke = transport or _http_transport(endpoint, timeout_seconds) - evaluated = [] - for source_case in suite["cases"]: + def process_case(source_case: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: envelope = _build_envelope(source_case, suite["suiteId"], released_at) output, events = _validate_response(invoke(envelope), envelope) live_case = {**source_case, "response": output} @@ -142,17 +142,18 @@ def evaluate_reference_suite( "normalization": "fixture-observed", }, } - evaluated.append( - _evaluate_case_with_evidence( - live_case, - suite["suiteId"], - released_at, - source_case=source_case, - metadata=envelope, - execution_evidence=evidence, - ) + return _evaluate_case_with_evidence( + live_case, + suite["suiteId"], + released_at, + source_case=source_case, + metadata=envelope, + execution_evidence=evidence, ) + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + evaluated = list(executor.map(process_case, suite["cases"])) + results = [result for result, _ in evaluated] return { "schemaVersion": "0.2.0", diff --git a/tests/ui/package-lock.json b/tests/ui/package-lock.json new file mode 100644 index 0000000..c1961bf --- /dev/null +++ b/tests/ui/package-lock.json @@ -0,0 +1,96 @@ +{ + "name": "cas-ui-evals", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cas-ui-evals", + "version": "1.0.0", + "devDependencies": { + "@playwright/test": "^1.42.0", + "@types/node": "^20.0.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "20.19.43", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz", + "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +}