|
| 1 | +"""Check the HTTP exchanges recorded during the test run against the spec. |
| 2 | +
|
| 3 | +``O2jConformanceRecorder`` (a test-scope servlet filter in every generated service) writes one |
| 4 | +JSON line per exchange to ``target/o2j-conformance/exchanges.jsonl``. For each exchange that |
| 5 | +belongs to a spec operation this module checks: |
| 6 | +
|
| 7 | +* the status is declared for the operation (exact, ``4XX``-style range, or ``default``); |
| 8 | +* a non-empty body has a content type the response declares; |
| 9 | +* the body is valid against that media type's schema (JSON Schema 2020-12, which is the |
| 10 | + dialect of the canonical 3.1 document, with formats checked). |
| 11 | +
|
| 12 | +This is what turns "the tests pass" into "what the service actually returned matches the |
| 13 | +contract". Line coverage cannot see a renamed JSON property or an undeclared 500; this can. |
| 14 | +""" |
| 15 | + |
| 16 | +from __future__ import annotations |
| 17 | + |
| 18 | +import json |
| 19 | +import re |
| 20 | +from dataclasses import dataclass, field |
| 21 | +from pathlib import Path |
| 22 | +from typing import Any |
| 23 | + |
| 24 | +from o2j.samples import Sampler |
| 25 | + |
| 26 | +EXCHANGES = Path("target") / "o2j-conformance" / "exchanges.jsonl" |
| 27 | + |
| 28 | + |
| 29 | +@dataclass |
| 30 | +class OperationConformance: |
| 31 | + exchanges: int = 0 |
| 32 | + conforming_success: int = 0 # 2xx exchanges that passed every check |
| 33 | + violations: list[str] = field(default_factory=list) |
| 34 | + |
| 35 | + |
| 36 | +@dataclass |
| 37 | +class ConformanceResult: |
| 38 | + found: bool |
| 39 | + exchanges: int = 0 |
| 40 | + checked: int = 0 |
| 41 | + ignored: int = 0 # exchanges outside the spec (actuator, refused unknown routes) |
| 42 | + by_operation: dict[str, OperationConformance] = field(default_factory=dict) |
| 43 | + |
| 44 | + @property |
| 45 | + def violations(self) -> list[str]: |
| 46 | + return [v for op in self.by_operation.values() for v in op.violations] |
| 47 | + |
| 48 | + |
| 49 | +@dataclass(frozen=True) |
| 50 | +class _Route: |
| 51 | + operation_id: str |
| 52 | + method: str |
| 53 | + path: str |
| 54 | + regex: re.Pattern[str] |
| 55 | + |
| 56 | + |
| 57 | +def check(project: Path, spec: dict[str, Any], base_path: str) -> ConformanceResult: |
| 58 | + path = project / EXCHANGES |
| 59 | + if not path.is_file(): |
| 60 | + return ConformanceResult(found=False) |
| 61 | + |
| 62 | + sampler = Sampler(spec) |
| 63 | + routes = _routes(spec, base_path) |
| 64 | + result = ConformanceResult(found=True) |
| 65 | + |
| 66 | + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): |
| 67 | + if not line.strip(): |
| 68 | + continue |
| 69 | + result.exchanges += 1 |
| 70 | + try: |
| 71 | + exchange = json.loads(line) |
| 72 | + except json.JSONDecodeError: |
| 73 | + result.ignored += 1 |
| 74 | + continue |
| 75 | + route = _match(routes, str(exchange.get("method", "")), str(exchange.get("path", ""))) |
| 76 | + if route is None: |
| 77 | + result.ignored += 1 |
| 78 | + continue |
| 79 | + result.checked += 1 |
| 80 | + stats = result.by_operation.setdefault(route.operation_id, OperationConformance()) |
| 81 | + stats.exchanges += 1 |
| 82 | + problems = _check_exchange(sampler, spec, route, exchange) |
| 83 | + where = f"{route.operation_id} {exchange.get('method')} {exchange.get('path')}" |
| 84 | + status = exchange.get("status") |
| 85 | + stats.violations += [f"{where} -> {status}: {p} (exchange {number})" for p in problems] |
| 86 | + if not problems and isinstance(status, int) and 200 <= status < 300: |
| 87 | + stats.conforming_success += 1 |
| 88 | + return result |
| 89 | + |
| 90 | + |
| 91 | +def _routes(spec: dict[str, Any], base_path: str) -> list[_Route]: |
| 92 | + out: list[_Route] = [] |
| 93 | + for path, item in (spec.get("paths") or {}).items(): |
| 94 | + if not isinstance(item, dict): |
| 95 | + continue |
| 96 | + full = (base_path.rstrip("/") + "/" + path.lstrip("/")) if base_path else path |
| 97 | + pattern = "^" + re.sub(r"\\\{[^/]+?\\\}", "[^/]+", re.escape(full)) + "/?$" |
| 98 | + for method, op in item.items(): |
| 99 | + if isinstance(op, dict) and "responses" in op: |
| 100 | + out.append( |
| 101 | + _Route( |
| 102 | + operation_id=str(op.get("operationId") or f"{method} {path}"), |
| 103 | + method=method.upper(), |
| 104 | + path=path, |
| 105 | + regex=re.compile(pattern), |
| 106 | + ) |
| 107 | + ) |
| 108 | + # Literal segments before templated ones, as Spring MVC matches. |
| 109 | + return sorted(out, key=lambda r: (r.path.count("{"), r.path)) |
| 110 | + |
| 111 | + |
| 112 | +def _match(routes: list[_Route], method: str, path: str) -> _Route | None: |
| 113 | + for route in routes: |
| 114 | + if route.method == method.upper() and route.regex.match(path): |
| 115 | + return route |
| 116 | + return None |
| 117 | + |
| 118 | + |
| 119 | +def _check_exchange( |
| 120 | + sampler: Sampler, spec: dict[str, Any], route: _Route, exchange: dict[str, Any] |
| 121 | +) -> list[str]: |
| 122 | + status = exchange.get("status") |
| 123 | + if not isinstance(status, int): |
| 124 | + return ["recorded exchange has no status"] |
| 125 | + if status == 501: |
| 126 | + return ["501 Not Implemented: the operation is still a stub"] |
| 127 | + |
| 128 | + operation = spec["paths"][route.path][route.method.lower()] |
| 129 | + responses = operation.get("responses", {}) |
| 130 | + response = ( |
| 131 | + responses.get(str(status)) |
| 132 | + or responses.get(f"{str(status)[0]}XX") |
| 133 | + or responses.get("default") |
| 134 | + ) |
| 135 | + if response is None: |
| 136 | + declared = ", ".join(sorted(str(k) for k in responses)) |
| 137 | + return [f"status {status} is not declared (declared: {declared})"] |
| 138 | + response = sampler.resolve(response) |
| 139 | + |
| 140 | + body = exchange.get("body") or "" |
| 141 | + if not body.strip(): |
| 142 | + return [] |
| 143 | + |
| 144 | + content = response.get("content") or {} |
| 145 | + if not content: |
| 146 | + return ["a body was returned but the response declares no content"] |
| 147 | + media = _media_type(exchange.get("contentType")) |
| 148 | + if media not in content: |
| 149 | + return [f"content type {media or 'none'} is not declared (declared: {', '.join(content)})"] |
| 150 | + schema = content[media].get("schema") |
| 151 | + if not isinstance(schema, dict) or not (media.endswith("json")): |
| 152 | + return [] |
| 153 | + try: |
| 154 | + value = json.loads(body) |
| 155 | + except json.JSONDecodeError: |
| 156 | + return [f"body is not valid JSON although {media} was declared"] |
| 157 | + |
| 158 | + errors = sorted(sampler.validator(schema).iter_errors(value), key=lambda e: list(e.path)) |
| 159 | + return [ |
| 160 | + f"body does not match the schema at /{'/'.join(str(p) for p in e.path)}: {e.message}" |
| 161 | + for e in errors[:5] |
| 162 | + ] |
| 163 | + |
| 164 | + |
| 165 | +def _media_type(content_type: Any) -> str | None: |
| 166 | + if not isinstance(content_type, str) or not content_type: |
| 167 | + return None |
| 168 | + return content_type.split(";", 1)[0].strip().lower() |
0 commit comments