From 0f2a5a80d9f8a7ed970fb6dee8576a77b31388df Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 20 Apr 2026 09:07:34 +0100 Subject: [PATCH 01/11] Updated openAPI schema to remove callback (not currently added) and add versioning --- schemas/muEd/openapi-v0_1_0.yml | 199 +++++++++++++++++++++----------- 1 file changed, 133 insertions(+), 66 deletions(-) diff --git a/schemas/muEd/openapi-v0_1_0.yml b/schemas/muEd/openapi-v0_1_0.yml index 844028a..4ec5199 100644 --- a/schemas/muEd/openapi-v0_1_0.yml +++ b/schemas/muEd/openapi-v0_1_0.yml @@ -18,12 +18,13 @@ paths: summary: Evaluate a submission and generate feedback operationId: evaluateSubmission description: | - Generates a list of feedback items for a given student submission. The request can optionally include the task context, user information, criteria to evaluate on, pre-submission feedback options, configuration, and a callback URL for asynchronous result delivery. + Generates a list of feedback items for a given student submission. The request can optionally include the task context, user information, criteria to evaluate on, pre-submission feedback options, and configuration. tags: - evaluate parameters: - $ref: '#/components/parameters/Authorization' - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' requestBody: required: true content: @@ -60,19 +61,6 @@ paths: llm: model: gpt-5.2 temperature: 0.4 - asyncCallbackExample: - summary: Asynchronous processing via callback URL - value: - submission: - submissionId: sub-async-001 - taskId: task-42 - type: TEXT - format: plain - content: - text: Detailed essay answer that may require longer processing. - submittedAt: '2025-12-16T09:45:00Z' - version: 1 - callbackUrl: https://learning-platform.example.com/hooks/evaluate-result withTaskAndExtras: summary: With task context, user, criteria and configuration value: @@ -340,6 +328,10 @@ paths: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -370,12 +362,12 @@ paths: - feedbackId: fb-2 title: Overall structure message: The overall structure of your answer is clear and easy to follow. - '202': - $ref: '#/components/responses/202-Accepted' '400': $ref: '#/components/responses/400-BadRequest' '403': $ref: '#/components/responses/403-Forbidden' + '406': + $ref: '#/components/responses/406-VersionNotSupported' '500': $ref: '#/components/responses/500-InternalError' '501': @@ -390,9 +382,19 @@ paths: - evaluate parameters: - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' responses: '200': description: Evaluate service is reachable and reporting capabilities. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -427,6 +429,10 @@ paths: supportedLanguages: - en - de + supportedVersions: + - 0.1.0 + '406': + $ref: '#/components/responses/406-VersionNotSupported' '501': description: The server does not implement the health endpoint for evaluate. $ref: '#/components/responses/501-NotImplemented' @@ -443,6 +449,7 @@ paths: parameters: - $ref: '#/components/parameters/Authorization' - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' requestBody: required: true content: @@ -567,6 +574,10 @@ paths: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -616,6 +627,8 @@ paths: $ref: '#/components/responses/400-BadRequest-2' '403': $ref: '#/components/responses/403-Forbidden' + '406': + $ref: '#/components/responses/406-VersionNotSupported' '500': $ref: '#/components/responses/500-InternalError-2' '501': @@ -630,9 +643,19 @@ paths: - chat parameters: - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' responses: '200': description: Chat service is reachable and reporting capabilities. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -655,6 +678,10 @@ paths: supportedModels: - gpt-4o - llama-3 + supportedVersions: + - 0.1.0 + '406': + $ref: '#/components/responses/406-VersionNotSupported' '501': description: The server does not implement the health endpoint for chat. $ref: '#/components/responses/501-NotImplemented-2' @@ -675,6 +702,15 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + in: header + name: X-Api-Version + description: | + The µEd API version the client is targeting (e.g. "0.1.0"). If omitted, the server will use the latest version it supports. If the requested version cannot be served, the server returns 406 Version Not Supported. + required: false + schema: + type: string + example: 0.1.0 schemas: Task: type: object @@ -759,10 +795,6 @@ components: additionalProperties: true description: | Logical representation of the submission content. The expected structure depends on the artefact type: - TEXT: { text: string } or { markdown: string } - CODE: { code: string } or { files: [{ path: string, content: string }], entryPoint?: string } - MATH: { expression: string } - MODEL: { model: string | object, notation?: string } - supplementaryContent: - type: object - additionalProperties: true - description: Optional additional content for the learner's submission. This could include workings for math tasks, or raw source code for compiled binaries. submittedAt: type: - string @@ -1203,7 +1235,7 @@ components: EvaluateRequest: type: object description: | - Input for task evaluate service. The submission is mandatory; task, user, criteria, pre-submission feedback options, and configuration, and callback URL are optional. + Input for task evaluate service. The submission is mandatory; task, user, criteria, pre-submission feedback options, and configuration are optional. required: - submission properties: @@ -1239,13 +1271,6 @@ components: - 'null' allOf: - $ref: '#/components/schemas/PreSubmissionFeedback' - callbackUrl: - type: - - string - - 'null' - format: uri - description: | - Optional HTTPS callback URL for asynchronous processing. If provided, the service may return 202 Accepted immediately and deliver feedback results to this URL once processing is complete. configuration: description: | Optional key-value configuration dictionary for provider-specific or experimental parameters. Not standardized. @@ -1339,26 +1364,6 @@ components: description: Optional target reference inside the submission. allOf: - $ref: '#/components/schemas/FeedbackTarget' - EvaluateAcceptedResponse: - type: object - description: Acknowledgement that evaluation was accepted for asynchronous processing. - required: - - status - - requestId - properties: - status: - type: string - enum: - - ACCEPTED - description: Indicates that the request has been accepted for asynchronous processing. - requestId: - type: string - description: Identifier to correlate this accepted request with callback delivery. - message: - type: - - string - - 'null' - description: Optional human-readable message about asynchronous processing. ErrorResponse: type: object description: Standard error response returned by µEd API services. @@ -1505,6 +1510,14 @@ components: description: Optional list of supported language codes (e.g., 'en', 'de'). items: type: string + supportedAPIVersions: + type: + - array + - 'null' + description: | + Optional list of µEd API versions supported by this service implementation (e.g., ["0.1.0"]). Clients can use this to select a compatible X-Api-Version. + items: + type: string EvaluateHealthResponse: type: object description: Health status and capabilities of the evaluate service. @@ -1658,6 +1671,14 @@ components: description: Optional list of supported models. items: type: string + supportedAPIVersions: + type: + - array + - 'null' + description: | + Optional list of µEd API versions supported by this service implementation (e.g., ["0.1.0"]). Clients can use this to select a compatible X-Api-Version. + items: + type: string ChatHealthResponse: type: object description: Health status and capabilities of the chat service. @@ -1680,24 +1701,6 @@ components: capabilities: $ref: '#/components/schemas/ChatCapabilities' responses: - 202-Accepted: - description: Request accepted for asynchronous evaluation processing. - headers: - X-Request-Id: - description: Request id for tracing this request across services. - schema: - type: string - content: - application/json: - schema: - $ref: '#/components/schemas/EvaluateAcceptedResponse' - examples: - asyncAccepted: - summary: Example accepted async request - value: - status: ACCEPTED - requestId: req-7c193f38 - message: Evaluation queued. Results will be sent to callbackUrl. 400-BadRequest: description: Invalid request (e.g. missing content or invalid schema). headers: @@ -1705,6 +1708,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1726,6 +1733,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1741,6 +1752,34 @@ components: details: resource: submission required_permission: write + 406-VersionNotSupported: + description: | + The requested API version (supplied via X-Api-Version) is not supported by this service. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + versionNotSupported: + summary: Example version not supported error + value: + title: API version not supported + message: 'The requested API version ''0.0'' is not supported. Supported versions are: [''0.1.0''].' + code: VERSION_NOT_SUPPORTED + trace: null + details: + requestedVersion: '0.0' + supportedVersions: + - 0.1.0 500-InternalError: description: Internal server error. headers: @@ -1748,6 +1787,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1770,6 +1813,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1790,6 +1837,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1826,6 +1877,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1847,6 +1902,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1869,6 +1928,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1889,6 +1952,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: From c599e3e05fea14ac8eaef4305783c2c700ac301a Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 20 Apr 2026 09:11:49 +0100 Subject: [PATCH 02/11] Updated healthcheck test to use `/evaluate/health` endpoint --- tests/mued_handling_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mued_handling_test.py b/tests/mued_handling_test.py index 4744dcc..a6fd516 100644 --- a/tests/mued_handling_test.py +++ b/tests/mued_handling_test.py @@ -98,7 +98,7 @@ def test_evaluate_bodyless_event_returns_error(self): ) def test_healthcheck(self): - event = {"path": "/health"} + event = {"path": "/evaluate/health"} response = handler(event) From 39f3c6528c96064ff724ad9b9eae19fbef8cd889 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 20 Apr 2026 09:24:33 +0100 Subject: [PATCH 03/11] Updated healthcheck to use `healthcheck_muEd` and added test coverage for supported API versions. --- handler.py | 3 ++- tests/commands_test.py | 9 +++++++++ tests/mued_handling_test.py | 6 ++++-- tools/commands.py | 24 ++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/handler.py b/handler.py index 9e893a1..ac2ad7a 100755 --- a/handler.py +++ b/handler.py @@ -81,7 +81,8 @@ def handle_muEd_command(event: JsonType, command: str) -> HandlerResponse: validate.body(response, MuEdResBodyValidators.EVALUATION) elif command == "healthcheck": - response = commands.healthcheck() + response = commands.healthcheck_muEd() + validate.body(response, MuEdResBodyValidators.HEALTHCHECK) else: response = Response( diff --git a/tests/commands_test.py b/tests/commands_test.py index 18a4a28..4fe4752 100644 --- a/tests/commands_test.py +++ b/tests/commands_test.py @@ -346,6 +346,15 @@ def test_healthcheck(self): self.assertIn("tests_passed", result) + def test_healthcheck_muEd(self): + response = commands.healthcheck_muEd() + + self.assertIn(response["status"], ("OK", "DEGRADED", "UNAVAILABLE")) + capabilities = response["capabilities"] + self.assertIn("supportedAPIVersions", capabilities) + self.assertIsInstance(capabilities["supportedAPIVersions"], list) + self.assertIn("0.1.0", capabilities["supportedAPIVersions"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/mued_handling_test.py b/tests/mued_handling_test.py index a6fd516..a7e2e55 100644 --- a/tests/mued_handling_test.py +++ b/tests/mued_handling_test.py @@ -102,8 +102,10 @@ def test_healthcheck(self): response = handler(event) - self.assertEqual(response.get("command"), "healthcheck") - self.assertIn("result", response) + self.assertIn(response.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) + capabilities = response.get("capabilities", {}) + self.assertIn("supportedAPIVersions", capabilities) + self.assertIn("0.1.0", capabilities["supportedAPIVersions"]) def test_unknown_path_falls_back_to_legacy(self): event = { diff --git a/tools/commands.py b/tools/commands.py index c5d54f6..a0ae15d 100644 --- a/tools/commands.py +++ b/tools/commands.py @@ -58,6 +58,9 @@ class CaseResult(NamedTuple): warning: Optional[CaseWarning] = None +SUPPORTED_MUED_VERSIONS: list[str] = ["0.1.0"] + + def healthcheck() -> Response: """Run the healthcheck command for the evaluation function. @@ -68,6 +71,27 @@ def healthcheck() -> Response: return Response(command="healthcheck", result=result) +def healthcheck_muEd() -> Dict: + """Run the healthcheck command and return a muEd EvaluateHealthResponse. + + Returns: + Dict: A spec-compliant EvaluateHealthResponse. + """ + result = health.healthcheck() + status = "OK" if result["tests_passed"] else "DEGRADED" + return { + "status": status, + "capabilities": { + "supportsEvaluate": True, + "supportsPreSubmissionFeedback": True, + "supportsFormativeFeedback": True, + "supportsSummativeFeedback": False, + "supportsDataPolicy": "NOT_SUPPORTED", + "supportedAPIVersions": SUPPORTED_MUED_VERSIONS, + }, + } + + def preview( body: JsonType, fnc: Optional[PreviewFunctionType] = None ) -> Response: From d7b68002cbf7431c7e4cc485020371daca1d36bf Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 20 Apr 2026 09:47:40 +0100 Subject: [PATCH 04/11] Added version compatibility checks for `X-Api-Version` header in `/evaluate` and `/evaluate/health` endpoints, including tests for supported, unsupported, and absent versions. --- handler.py | 33 +++++++++++++++++++++++++++++++++ tests/mued_handling_test.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/handler.py b/handler.py index ac2ad7a..28f4472 100755 --- a/handler.py +++ b/handler.py @@ -2,6 +2,8 @@ from .tools import commands, docs, parse, validate from .tools.parse import ParseError +from typing import Optional + from .tools.utils import ErrorResponse, HandlerResponse, JsonType, Response from .tools.validate import ( LegacyReqBodyValidators, @@ -58,6 +60,33 @@ def handle_legacy_command(event: JsonType, command: str) -> HandlerResponse: return response +def check_muEd_version(event: JsonType) -> Optional[HandlerResponse]: + """Check the X-Api-Version header against supported muEd versions. + + Args: + event (JsonType): The AWS Lambda event received by the handler. + + Returns: + Optional[HandlerResponse]: A version-not-supported error response if + the requested version is unsupported, otherwise None. + """ + version = (event.get("headers") or {}).get("X-Api-Version") + if version and version not in commands.SUPPORTED_MUED_VERSIONS: + return { + "title": "API version not supported", + "message": ( + f"The requested API version '{version}' is not supported. " + f"Supported versions are: {commands.SUPPORTED_MUED_VERSIONS}." + ), + "code": "VERSION_NOT_SUPPORTED", + "details": { + "requestedVersion": version, + "supportedVersions": commands.SUPPORTED_MUED_VERSIONS, + }, + } + return None + + def handle_muEd_command(event: JsonType, command: str) -> HandlerResponse: """Switch case for handling different command options using muEd schemas. @@ -68,6 +97,10 @@ def handle_muEd_command(event: JsonType, command: str) -> HandlerResponse: Returns: HandlerResponse: The response object returned by the handler. """ + version_error = check_muEd_version(event) + if version_error: + return version_error + if command == "eval": body = parse.body(event) validate.body(body, MuEdReqBodyValidators.EVALUATION) diff --git a/tests/mued_handling_test.py b/tests/mued_handling_test.py index a7e2e55..d8a46f0 100644 --- a/tests/mued_handling_test.py +++ b/tests/mued_handling_test.py @@ -107,6 +107,37 @@ def test_healthcheck(self): self.assertIn("supportedAPIVersions", capabilities) self.assertIn("0.1.0", capabilities["supportedAPIVersions"]) + def test_supported_version_header_is_accepted(self): + event = { + "path": "/evaluate/health", + "headers": {"X-Api-Version": "0.1.0"}, + } + + response = handler(event) + + self.assertIn(response.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) + + def test_unsupported_version_header_returns_error(self): + event = { + "path": "/evaluate", + "headers": {"X-Api-Version": "99.0.0"}, + "body": {"submission": {"type": "TEXT", "content": {}}}, + } + + response = handler(event) + + self.assertEqual(response.get("code"), "VERSION_NOT_SUPPORTED") + self.assertIn("details", response) + self.assertEqual(response["details"]["requestedVersion"], "99.0.0") + self.assertIn("0.1.0", response["details"]["supportedVersions"]) + + def test_absent_version_header_proceeds_normally(self): + event = {"path": "/evaluate/health"} + + response = handler(event) + + self.assertNotIn("code", response) + def test_unknown_path_falls_back_to_legacy(self): event = { "path": "/unknown", From a9d6e25dffa99e75554fb8c20070534c046e213d Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 20 Apr 2026 10:07:34 +0100 Subject: [PATCH 05/11] Refactored tests and handler logic to standardize response structure using `wrap_muEd_response`, ensuring consistent JSON serialization and inclusion of `X-Api-Version` across all endpoints. Updated tests accordingly. --- handler.py | 33 ++++++++++++-- tests/mued_handling_test.py | 90 ++++++++++++++++++++++--------------- 2 files changed, 83 insertions(+), 40 deletions(-) diff --git a/handler.py b/handler.py index 28f4472..36e28ed 100755 --- a/handler.py +++ b/handler.py @@ -1,10 +1,11 @@ +import json from evaluation_function_utils.errors import EvaluationException from .tools import commands, docs, parse, validate from .tools.parse import ParseError -from typing import Optional +from typing import Any, Optional -from .tools.utils import ErrorResponse, HandlerResponse, JsonType, Response +from .tools.utils import DocsResponse, ErrorResponse, HandlerResponse, JsonType, Response from .tools.validate import ( LegacyReqBodyValidators, LegacyResBodyValidators, @@ -60,6 +61,30 @@ def handle_legacy_command(event: JsonType, command: str) -> HandlerResponse: return response +def wrap_muEd_response(body: Any, event: JsonType, status_code: int = 200) -> DocsResponse: + """Wrap a muEd response body in Lambda proxy format with X-Api-Version header. + + Args: + body: The response body to serialise. + event (JsonType): The incoming event (used to resolve the served version). + status_code (int): The HTTP status code. Defaults to 200. + + Returns: + DocsResponse: Proxy-format response with X-Api-Version header set. + """ + requested = (event.get("headers") or {}).get("X-Api-Version") + if requested and requested in commands.SUPPORTED_MUED_VERSIONS: + version = requested + else: + version = commands.SUPPORTED_MUED_VERSIONS[-1] + return DocsResponse( + statusCode=status_code, + headers={"X-Api-Version": version}, + body=json.dumps(body), + isBase64Encoded=False, + ) + + def check_muEd_version(event: JsonType) -> Optional[HandlerResponse]: """Check the X-Api-Version header against supported muEd versions. @@ -99,7 +124,7 @@ def handle_muEd_command(event: JsonType, command: str) -> HandlerResponse: """ version_error = check_muEd_version(event) if version_error: - return version_error + return wrap_muEd_response(version_error, event, 406) if command == "eval": body = parse.body(event) @@ -122,7 +147,7 @@ def handle_muEd_command(event: JsonType, command: str) -> HandlerResponse: error=ErrorResponse(message=f"Unknown command '{command}'.") ) - return response + return wrap_muEd_response(response, event) def handler(event: JsonType, _=None) -> HandlerResponse: diff --git a/tests/mued_handling_test.py b/tests/mued_handling_test.py index d8a46f0..be83ccf 100644 --- a/tests/mued_handling_test.py +++ b/tests/mued_handling_test.py @@ -1,3 +1,4 @@ +import json import os import unittest from pathlib import Path @@ -33,9 +34,11 @@ def test_evaluate_returns_feedback_list(self): response = handler(event) - self.assertIsInstance(response, list) - self.assertEqual(len(response), 1) - self.assertIn("awardedPoints", response[0]) + self.assertEqual(response["statusCode"], 200) + body = json.loads(response["body"]) + self.assertIsInstance(body, list) + self.assertEqual(len(body), 1) + self.assertIn("awardedPoints", body[0]) def test_evaluate_feedback_message(self): event = { @@ -45,9 +48,9 @@ def test_evaluate_feedback_message(self): response = handler(event) - self.assertIsInstance(response, list) - self.assertEqual(response[0]["message"], "Well done.") - self.assertEqual(response[0]["awardedPoints"], True) + body = json.loads(response["body"]) + self.assertEqual(body[0]["message"], "Well done.") + self.assertEqual(body[0]["awardedPoints"], True) def test_evaluate_with_task(self): event = { @@ -63,7 +66,8 @@ def test_evaluate_with_task(self): response = handler(event) - self.assertIsInstance(response, list) + body = json.loads(response["body"]) + self.assertIsInstance(body, list) def test_evaluate_missing_submission_returns_error(self): event = { @@ -102,8 +106,11 @@ def test_healthcheck(self): response = handler(event) - self.assertIn(response.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) - capabilities = response.get("capabilities", {}) + self.assertEqual(response["statusCode"], 200) + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertIn(body.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) + capabilities = body.get("capabilities", {}) self.assertIn("supportedAPIVersions", capabilities) self.assertIn("0.1.0", capabilities["supportedAPIVersions"]) @@ -115,7 +122,10 @@ def test_supported_version_header_is_accepted(self): response = handler(event) - self.assertIn(response.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) + self.assertEqual(response["statusCode"], 200) + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertIn(body.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) def test_unsupported_version_header_returns_error(self): event = { @@ -126,17 +136,22 @@ def test_unsupported_version_header_returns_error(self): response = handler(event) - self.assertEqual(response.get("code"), "VERSION_NOT_SUPPORTED") - self.assertIn("details", response) - self.assertEqual(response["details"]["requestedVersion"], "99.0.0") - self.assertIn("0.1.0", response["details"]["supportedVersions"]) + self.assertEqual(response["statusCode"], 406) + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertEqual(body.get("code"), "VERSION_NOT_SUPPORTED") + self.assertIn("details", body) + self.assertEqual(body["details"]["requestedVersion"], "99.0.0") + self.assertIn("0.1.0", body["details"]["supportedVersions"]) def test_absent_version_header_proceeds_normally(self): event = {"path": "/evaluate/health"} response = handler(event) - self.assertNotIn("code", response) + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertNotIn("code", body) def test_unknown_path_falls_back_to_legacy(self): event = { @@ -179,10 +194,10 @@ def test_math_submission_extracts_expression(self): "task": {"title": "T", "referenceSolution": {"expression": "x+1"}}, }, } - result = handler(event) + result = json.loads(handler(event)["body"]) self.assertEqual(self.captured["response"], "x+1") self.assertEqual(self.captured["answer"], "x+1") - self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + self.assertEqual(result[0]["awardedPoints"], True) def test_text_submission_extracts_text(self): event = { @@ -192,10 +207,10 @@ def test_text_submission_extracts_text(self): "task": {"title": "T", "referenceSolution": {"text": "hello"}}, }, } - result = handler(event) + result = json.loads(handler(event)["body"]) self.assertEqual(self.captured["response"], "hello") self.assertEqual(self.captured["answer"], "hello") - self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + self.assertEqual(result[0]["awardedPoints"], True) def test_configuration_params_forwarded(self): event = { @@ -205,9 +220,9 @@ def test_configuration_params_forwarded(self): "configuration": {"params": {"strict_syntax": False}}, }, } - result = handler(event) + result = json.loads(handler(event)["body"]) self.assertEqual(self.captured["params"], {"strict_syntax": False}) - self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + self.assertEqual(result[0]["awardedPoints"], True) def test_no_task_answer_is_none(self): event = { @@ -216,9 +231,9 @@ def test_no_task_answer_is_none(self): "submission": {"type": "MATH", "content": {"expression": "x+1"}}, }, } - result = handler(event) + result = json.loads(handler(event)["body"]) self.assertIsNone(self.captured["answer"]) - self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + self.assertEqual(result[0]["awardedPoints"], True) class TestMuEdPreviewHandlerFunction(unittest.TestCase): @@ -242,8 +257,10 @@ def test_preview_returns_feedback_list(self): response = handler(event) - self.assertIsInstance(response, list) - self.assertEqual(len(response), 1) + self.assertEqual(response["statusCode"], 200) + body = json.loads(response["body"]) + self.assertIsInstance(body, list) + self.assertEqual(len(body), 1) def test_preview_feedback_id_is_preSubmissionFeedback(self): event = { @@ -251,9 +268,9 @@ def test_preview_feedback_id_is_preSubmissionFeedback(self): "body": {"submission": {"type": "MATH", "content": {"expression": "x+1"}}}, } - response = handler(event) + body = json.loads(handler(event)["body"]) - self.assertNotIn("feedbackId", response[0]) # type: ignore + self.assertNotIn("feedbackId", body[0]) def test_preview_contains_preSubmissionFeedback_field(self): event = { @@ -261,9 +278,9 @@ def test_preview_contains_preSubmissionFeedback_field(self): "body": {"submission": {"type": "MATH", "content": {"expression": "x+1"}}}, } - response = handler(event) + body = json.loads(handler(event)["body"]) - self.assertIn("preSubmissionFeedback", response[0]) # type: ignore + self.assertIn("preSubmissionFeedback", body[0]) def test_preview_preSubmissionFeedback_has_latex_and_sympy(self): event = { @@ -271,9 +288,9 @@ def test_preview_preSubmissionFeedback_has_latex_and_sympy(self): "body": {"submission": {"type": "MATH", "content": {"expression": "x+1"}}}, } - response = handler(event) + body = json.loads(handler(event)["body"]) - preview = response[0]["preSubmissionFeedback"] # type: ignore + preview = body[0]["preSubmissionFeedback"] self.assertIn("latex", preview) self.assertIn("sympy", preview) @@ -375,7 +392,8 @@ def test_no_task_required(self): response = handler(event) - self.assertIsInstance(response, list) + body = json.loads(response["body"]) + self.assertIsInstance(body, list) self.assertEqual(self.captured["response"], "sin(x)") def test_preview_result_propagated(self): @@ -386,11 +404,11 @@ def test_preview_result_propagated(self): }, } - response = handler(event) + body = json.loads(handler(event)["body"]) - self.assertEqual(response[0]["preSubmissionFeedback"]["latex"], "captured") # type: ignore - self.assertEqual(response[0]["preSubmissionFeedback"]["sympy"], "x+1") # type: ignore + self.assertEqual(body[0]["preSubmissionFeedback"]["latex"], "captured") + self.assertEqual(body[0]["preSubmissionFeedback"]["sympy"], "x+1") if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() From 4c8cfb7b2f46fa607e17e3676d001c52b9ed66bb Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 20 Apr 2026 10:09:10 +0100 Subject: [PATCH 06/11] Fixed test leakage --- tests/mued_handling_test.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/mued_handling_test.py b/tests/mued_handling_test.py index be83ccf..852594c 100644 --- a/tests/mued_handling_test.py +++ b/tests/mued_handling_test.py @@ -18,12 +18,13 @@ class TestMuEdHandlerFunction(unittest.TestCase): def setUp(self) -> None: os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_eval = commands.evaluation_function commands.evaluation_function = evaluation_function return super().setUp() def tearDown(self) -> None: os.environ.pop("SCHEMA_DIR", None) - commands.evaluation_function = None + commands.evaluation_function = self._orig_eval return super().tearDown() def test_evaluate_returns_feedback_list(self): @@ -169,6 +170,7 @@ def test_unknown_path_falls_back_to_legacy(self): class TestMuEdEvaluateExtraction(unittest.TestCase): def setUp(self) -> None: os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_eval = commands.evaluation_function self.captured: dict = {} captured = self.captured @@ -183,7 +185,7 @@ def capturing_eval(response, answer, params): def tearDown(self) -> None: os.environ.pop("SCHEMA_DIR", None) - commands.evaluation_function = None + commands.evaluation_function = self._orig_eval return super().tearDown() def test_math_submission_extracts_expression(self): @@ -239,6 +241,7 @@ def test_no_task_answer_is_none(self): class TestMuEdPreviewHandlerFunction(unittest.TestCase): def setUp(self) -> None: os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_preview = commands.preview_function commands.preview_function = lambda response, params: { "preview": {"latex": f"\\text{{{response}}}", "sympy": response} } @@ -246,7 +249,7 @@ def setUp(self) -> None: def tearDown(self) -> None: os.environ.pop("SCHEMA_DIR", None) - commands.preview_function = None + commands.preview_function = self._orig_preview return super().tearDown() def test_preview_returns_feedback_list(self): @@ -329,6 +332,7 @@ def test_preview_invalid_submission_type_returns_error(self): class TestMuEdPreviewExtraction(unittest.TestCase): def setUp(self) -> None: os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_preview = commands.preview_function self.captured: dict = {} captured = self.captured @@ -342,7 +346,7 @@ def capturing_preview(response, params): def tearDown(self) -> None: os.environ.pop("SCHEMA_DIR", None) - commands.preview_function = None + commands.preview_function = self._orig_preview return super().tearDown() def test_math_submission_extracts_expression(self): From 9909032902daa8df9c589ea737f6d6d9677f24d6 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 20 Apr 2026 09:07:34 +0100 Subject: [PATCH 07/11] Updated openAPI schema to remove callback (not currently added) and add versioning --- schemas/muEd/openapi-v0_1_0.yml | 199 +++++++++++++++++++++----------- 1 file changed, 133 insertions(+), 66 deletions(-) diff --git a/schemas/muEd/openapi-v0_1_0.yml b/schemas/muEd/openapi-v0_1_0.yml index 844028a..4ec5199 100644 --- a/schemas/muEd/openapi-v0_1_0.yml +++ b/schemas/muEd/openapi-v0_1_0.yml @@ -18,12 +18,13 @@ paths: summary: Evaluate a submission and generate feedback operationId: evaluateSubmission description: | - Generates a list of feedback items for a given student submission. The request can optionally include the task context, user information, criteria to evaluate on, pre-submission feedback options, configuration, and a callback URL for asynchronous result delivery. + Generates a list of feedback items for a given student submission. The request can optionally include the task context, user information, criteria to evaluate on, pre-submission feedback options, and configuration. tags: - evaluate parameters: - $ref: '#/components/parameters/Authorization' - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' requestBody: required: true content: @@ -60,19 +61,6 @@ paths: llm: model: gpt-5.2 temperature: 0.4 - asyncCallbackExample: - summary: Asynchronous processing via callback URL - value: - submission: - submissionId: sub-async-001 - taskId: task-42 - type: TEXT - format: plain - content: - text: Detailed essay answer that may require longer processing. - submittedAt: '2025-12-16T09:45:00Z' - version: 1 - callbackUrl: https://learning-platform.example.com/hooks/evaluate-result withTaskAndExtras: summary: With task context, user, criteria and configuration value: @@ -340,6 +328,10 @@ paths: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -370,12 +362,12 @@ paths: - feedbackId: fb-2 title: Overall structure message: The overall structure of your answer is clear and easy to follow. - '202': - $ref: '#/components/responses/202-Accepted' '400': $ref: '#/components/responses/400-BadRequest' '403': $ref: '#/components/responses/403-Forbidden' + '406': + $ref: '#/components/responses/406-VersionNotSupported' '500': $ref: '#/components/responses/500-InternalError' '501': @@ -390,9 +382,19 @@ paths: - evaluate parameters: - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' responses: '200': description: Evaluate service is reachable and reporting capabilities. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -427,6 +429,10 @@ paths: supportedLanguages: - en - de + supportedVersions: + - 0.1.0 + '406': + $ref: '#/components/responses/406-VersionNotSupported' '501': description: The server does not implement the health endpoint for evaluate. $ref: '#/components/responses/501-NotImplemented' @@ -443,6 +449,7 @@ paths: parameters: - $ref: '#/components/parameters/Authorization' - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' requestBody: required: true content: @@ -567,6 +574,10 @@ paths: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -616,6 +627,8 @@ paths: $ref: '#/components/responses/400-BadRequest-2' '403': $ref: '#/components/responses/403-Forbidden' + '406': + $ref: '#/components/responses/406-VersionNotSupported' '500': $ref: '#/components/responses/500-InternalError-2' '501': @@ -630,9 +643,19 @@ paths: - chat parameters: - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' responses: '200': description: Chat service is reachable and reporting capabilities. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -655,6 +678,10 @@ paths: supportedModels: - gpt-4o - llama-3 + supportedVersions: + - 0.1.0 + '406': + $ref: '#/components/responses/406-VersionNotSupported' '501': description: The server does not implement the health endpoint for chat. $ref: '#/components/responses/501-NotImplemented-2' @@ -675,6 +702,15 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + in: header + name: X-Api-Version + description: | + The µEd API version the client is targeting (e.g. "0.1.0"). If omitted, the server will use the latest version it supports. If the requested version cannot be served, the server returns 406 Version Not Supported. + required: false + schema: + type: string + example: 0.1.0 schemas: Task: type: object @@ -759,10 +795,6 @@ components: additionalProperties: true description: | Logical representation of the submission content. The expected structure depends on the artefact type: - TEXT: { text: string } or { markdown: string } - CODE: { code: string } or { files: [{ path: string, content: string }], entryPoint?: string } - MATH: { expression: string } - MODEL: { model: string | object, notation?: string } - supplementaryContent: - type: object - additionalProperties: true - description: Optional additional content for the learner's submission. This could include workings for math tasks, or raw source code for compiled binaries. submittedAt: type: - string @@ -1203,7 +1235,7 @@ components: EvaluateRequest: type: object description: | - Input for task evaluate service. The submission is mandatory; task, user, criteria, pre-submission feedback options, and configuration, and callback URL are optional. + Input for task evaluate service. The submission is mandatory; task, user, criteria, pre-submission feedback options, and configuration are optional. required: - submission properties: @@ -1239,13 +1271,6 @@ components: - 'null' allOf: - $ref: '#/components/schemas/PreSubmissionFeedback' - callbackUrl: - type: - - string - - 'null' - format: uri - description: | - Optional HTTPS callback URL for asynchronous processing. If provided, the service may return 202 Accepted immediately and deliver feedback results to this URL once processing is complete. configuration: description: | Optional key-value configuration dictionary for provider-specific or experimental parameters. Not standardized. @@ -1339,26 +1364,6 @@ components: description: Optional target reference inside the submission. allOf: - $ref: '#/components/schemas/FeedbackTarget' - EvaluateAcceptedResponse: - type: object - description: Acknowledgement that evaluation was accepted for asynchronous processing. - required: - - status - - requestId - properties: - status: - type: string - enum: - - ACCEPTED - description: Indicates that the request has been accepted for asynchronous processing. - requestId: - type: string - description: Identifier to correlate this accepted request with callback delivery. - message: - type: - - string - - 'null' - description: Optional human-readable message about asynchronous processing. ErrorResponse: type: object description: Standard error response returned by µEd API services. @@ -1505,6 +1510,14 @@ components: description: Optional list of supported language codes (e.g., 'en', 'de'). items: type: string + supportedAPIVersions: + type: + - array + - 'null' + description: | + Optional list of µEd API versions supported by this service implementation (e.g., ["0.1.0"]). Clients can use this to select a compatible X-Api-Version. + items: + type: string EvaluateHealthResponse: type: object description: Health status and capabilities of the evaluate service. @@ -1658,6 +1671,14 @@ components: description: Optional list of supported models. items: type: string + supportedAPIVersions: + type: + - array + - 'null' + description: | + Optional list of µEd API versions supported by this service implementation (e.g., ["0.1.0"]). Clients can use this to select a compatible X-Api-Version. + items: + type: string ChatHealthResponse: type: object description: Health status and capabilities of the chat service. @@ -1680,24 +1701,6 @@ components: capabilities: $ref: '#/components/schemas/ChatCapabilities' responses: - 202-Accepted: - description: Request accepted for asynchronous evaluation processing. - headers: - X-Request-Id: - description: Request id for tracing this request across services. - schema: - type: string - content: - application/json: - schema: - $ref: '#/components/schemas/EvaluateAcceptedResponse' - examples: - asyncAccepted: - summary: Example accepted async request - value: - status: ACCEPTED - requestId: req-7c193f38 - message: Evaluation queued. Results will be sent to callbackUrl. 400-BadRequest: description: Invalid request (e.g. missing content or invalid schema). headers: @@ -1705,6 +1708,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1726,6 +1733,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1741,6 +1752,34 @@ components: details: resource: submission required_permission: write + 406-VersionNotSupported: + description: | + The requested API version (supplied via X-Api-Version) is not supported by this service. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + versionNotSupported: + summary: Example version not supported error + value: + title: API version not supported + message: 'The requested API version ''0.0'' is not supported. Supported versions are: [''0.1.0''].' + code: VERSION_NOT_SUPPORTED + trace: null + details: + requestedVersion: '0.0' + supportedVersions: + - 0.1.0 500-InternalError: description: Internal server error. headers: @@ -1748,6 +1787,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1770,6 +1813,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1790,6 +1837,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1826,6 +1877,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1847,6 +1902,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1869,6 +1928,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1889,6 +1952,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: From 09f78c646ced80fdf7f4bbcbc50d3ed94449ac00 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 20 Apr 2026 09:24:33 +0100 Subject: [PATCH 08/11] Updated healthcheck to use `healthcheck_muEd` and added test coverage for supported API versions. --- handler.py | 3 ++- tests/commands_test.py | 9 +++++++++ tests/mued_handling_test.py | 6 ++++-- tools/commands.py | 24 ++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/handler.py b/handler.py index aa1b322..7a6361e 100755 --- a/handler.py +++ b/handler.py @@ -75,7 +75,8 @@ def handle_muEd_command(event: JsonType, command: str) -> HandlerResponse: validate.body(response, MuEdResBodyValidators.EVALUATION) elif command == "healthcheck": - response = commands.healthcheck() + response = commands.healthcheck_muEd() + validate.body(response, MuEdResBodyValidators.HEALTHCHECK) else: response = Response( diff --git a/tests/commands_test.py b/tests/commands_test.py index 18a4a28..4fe4752 100644 --- a/tests/commands_test.py +++ b/tests/commands_test.py @@ -346,6 +346,15 @@ def test_healthcheck(self): self.assertIn("tests_passed", result) + def test_healthcheck_muEd(self): + response = commands.healthcheck_muEd() + + self.assertIn(response["status"], ("OK", "DEGRADED", "UNAVAILABLE")) + capabilities = response["capabilities"] + self.assertIn("supportedAPIVersions", capabilities) + self.assertIsInstance(capabilities["supportedAPIVersions"], list) + self.assertIn("0.1.0", capabilities["supportedAPIVersions"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/mued_handling_test.py b/tests/mued_handling_test.py index 078ce5d..da29663 100644 --- a/tests/mued_handling_test.py +++ b/tests/mued_handling_test.py @@ -102,8 +102,10 @@ def test_healthcheck(self): response = handler(event) - self.assertEqual(response.get("command"), "healthcheck") - self.assertIn("result", response) + self.assertIn(response.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) + capabilities = response.get("capabilities", {}) + self.assertIn("supportedAPIVersions", capabilities) + self.assertIn("0.1.0", capabilities["supportedAPIVersions"]) def test_unknown_path_falls_back_to_legacy(self): event = { diff --git a/tools/commands.py b/tools/commands.py index 34af3dd..b44d33f 100644 --- a/tools/commands.py +++ b/tools/commands.py @@ -58,6 +58,9 @@ class CaseResult(NamedTuple): warning: Optional[CaseWarning] = None +SUPPORTED_MUED_VERSIONS: list[str] = ["0.1.0"] + + def healthcheck() -> Response: """Run the healthcheck command for the evaluation function. @@ -68,6 +71,27 @@ def healthcheck() -> Response: return Response(command="healthcheck", result=result) +def healthcheck_muEd() -> Dict: + """Run the healthcheck command and return a muEd EvaluateHealthResponse. + + Returns: + Dict: A spec-compliant EvaluateHealthResponse. + """ + result = health.healthcheck() + status = "OK" if result["tests_passed"] else "DEGRADED" + return { + "status": status, + "capabilities": { + "supportsEvaluate": True, + "supportsPreSubmissionFeedback": True, + "supportsFormativeFeedback": True, + "supportsSummativeFeedback": False, + "supportsDataPolicy": "NOT_SUPPORTED", + "supportedAPIVersions": SUPPORTED_MUED_VERSIONS, + }, + } + + def preview( body: JsonType, fnc: Optional[PreviewFunctionType] = None ) -> Response: From b05d5bec7f9c4d73e0231abc8c2cfd74080032f1 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 20 Apr 2026 09:47:40 +0100 Subject: [PATCH 09/11] Added version compatibility checks for `X-Api-Version` header in `/evaluate` and `/evaluate/health` endpoints, including tests for supported, unsupported, and absent versions. --- handler.py | 33 +++++++++++++++++++++++++++++++++ tests/mued_handling_test.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/handler.py b/handler.py index 7a6361e..74fdb16 100755 --- a/handler.py +++ b/handler.py @@ -2,6 +2,8 @@ from .tools import commands, docs, parse, validate from .tools.parse import ParseError +from typing import Optional + from .tools.utils import ErrorResponse, HandlerResponse, JsonType, Response from .tools.validate import ( LegacyReqBodyValidators, @@ -58,6 +60,33 @@ def handle_legacy_command(event: JsonType, command: str) -> HandlerResponse: return response +def check_muEd_version(event: JsonType) -> Optional[HandlerResponse]: + """Check the X-Api-Version header against supported muEd versions. + + Args: + event (JsonType): The AWS Lambda event received by the handler. + + Returns: + Optional[HandlerResponse]: A version-not-supported error response if + the requested version is unsupported, otherwise None. + """ + version = (event.get("headers") or {}).get("X-Api-Version") + if version and version not in commands.SUPPORTED_MUED_VERSIONS: + return { + "title": "API version not supported", + "message": ( + f"The requested API version '{version}' is not supported. " + f"Supported versions are: {commands.SUPPORTED_MUED_VERSIONS}." + ), + "code": "VERSION_NOT_SUPPORTED", + "details": { + "requestedVersion": version, + "supportedVersions": commands.SUPPORTED_MUED_VERSIONS, + }, + } + return None + + def handle_muEd_command(event: JsonType, command: str) -> HandlerResponse: """Switch case for handling different command options using muEd schemas. @@ -68,6 +97,10 @@ def handle_muEd_command(event: JsonType, command: str) -> HandlerResponse: Returns: HandlerResponse: The response object returned by the handler. """ + version_error = check_muEd_version(event) + if version_error: + return version_error + if command == "eval": body = parse.body(event) validate.body(body, MuEdReqBodyValidators.EVALUATION) diff --git a/tests/mued_handling_test.py b/tests/mued_handling_test.py index da29663..f9cc45b 100644 --- a/tests/mued_handling_test.py +++ b/tests/mued_handling_test.py @@ -107,6 +107,37 @@ def test_healthcheck(self): self.assertIn("supportedAPIVersions", capabilities) self.assertIn("0.1.0", capabilities["supportedAPIVersions"]) + def test_supported_version_header_is_accepted(self): + event = { + "path": "/evaluate/health", + "headers": {"X-Api-Version": "0.1.0"}, + } + + response = handler(event) + + self.assertIn(response.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) + + def test_unsupported_version_header_returns_error(self): + event = { + "path": "/evaluate", + "headers": {"X-Api-Version": "99.0.0"}, + "body": {"submission": {"type": "TEXT", "content": {}}}, + } + + response = handler(event) + + self.assertEqual(response.get("code"), "VERSION_NOT_SUPPORTED") + self.assertIn("details", response) + self.assertEqual(response["details"]["requestedVersion"], "99.0.0") + self.assertIn("0.1.0", response["details"]["supportedVersions"]) + + def test_absent_version_header_proceeds_normally(self): + event = {"path": "/evaluate/health"} + + response = handler(event) + + self.assertNotIn("code", response) + def test_unknown_path_falls_back_to_legacy(self): event = { "path": "/unknown", From 93a10690f849430d191f9af635ce1f5f8a1dc25f Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 20 Apr 2026 10:07:34 +0100 Subject: [PATCH 10/11] Refactored tests and handler logic to standardize response structure using `wrap_muEd_response`, ensuring consistent JSON serialization and inclusion of `X-Api-Version` across all endpoints. Updated tests accordingly. --- handler.py | 33 ++++++++++++-- tests/mued_handling_test.py | 90 ++++++++++++++++++++++--------------- 2 files changed, 83 insertions(+), 40 deletions(-) diff --git a/handler.py b/handler.py index 74fdb16..bec16e8 100755 --- a/handler.py +++ b/handler.py @@ -1,10 +1,11 @@ +import json from evaluation_function_utils.errors import EvaluationException from .tools import commands, docs, parse, validate from .tools.parse import ParseError -from typing import Optional +from typing import Any, Optional -from .tools.utils import ErrorResponse, HandlerResponse, JsonType, Response +from .tools.utils import DocsResponse, ErrorResponse, HandlerResponse, JsonType, Response from .tools.validate import ( LegacyReqBodyValidators, LegacyResBodyValidators, @@ -60,6 +61,30 @@ def handle_legacy_command(event: JsonType, command: str) -> HandlerResponse: return response +def wrap_muEd_response(body: Any, event: JsonType, status_code: int = 200) -> DocsResponse: + """Wrap a muEd response body in Lambda proxy format with X-Api-Version header. + + Args: + body: The response body to serialise. + event (JsonType): The incoming event (used to resolve the served version). + status_code (int): The HTTP status code. Defaults to 200. + + Returns: + DocsResponse: Proxy-format response with X-Api-Version header set. + """ + requested = (event.get("headers") or {}).get("X-Api-Version") + if requested and requested in commands.SUPPORTED_MUED_VERSIONS: + version = requested + else: + version = commands.SUPPORTED_MUED_VERSIONS[-1] + return DocsResponse( + statusCode=status_code, + headers={"X-Api-Version": version}, + body=json.dumps(body), + isBase64Encoded=False, + ) + + def check_muEd_version(event: JsonType) -> Optional[HandlerResponse]: """Check the X-Api-Version header against supported muEd versions. @@ -99,7 +124,7 @@ def handle_muEd_command(event: JsonType, command: str) -> HandlerResponse: """ version_error = check_muEd_version(event) if version_error: - return version_error + return wrap_muEd_response(version_error, event, 406) if command == "eval": body = parse.body(event) @@ -116,7 +141,7 @@ def handle_muEd_command(event: JsonType, command: str) -> HandlerResponse: error=ErrorResponse(message=f"Unknown command '{command}'.") ) - return response + return wrap_muEd_response(response, event) def handler(event: JsonType, _=None) -> HandlerResponse: diff --git a/tests/mued_handling_test.py b/tests/mued_handling_test.py index f9cc45b..66c15f5 100644 --- a/tests/mued_handling_test.py +++ b/tests/mued_handling_test.py @@ -1,3 +1,4 @@ +import json import os import unittest from pathlib import Path @@ -33,9 +34,11 @@ def test_evaluate_returns_feedback_list(self): response = handler(event) - self.assertIsInstance(response, list) - self.assertEqual(len(response), 1) - self.assertIn("awardedPoints", response[0]) + self.assertEqual(response["statusCode"], 200) + body = json.loads(response["body"]) + self.assertIsInstance(body, list) + self.assertEqual(len(body), 1) + self.assertIn("awardedPoints", body[0]) def test_evaluate_feedback_message(self): event = { @@ -45,9 +48,9 @@ def test_evaluate_feedback_message(self): response = handler(event) - self.assertIsInstance(response, list) - self.assertEqual(response[0]["message"], "Well done.") - self.assertEqual(response[0]["awardedPoints"], True) + body = json.loads(response["body"]) + self.assertEqual(body[0]["message"], "Well done.") + self.assertEqual(body[0]["awardedPoints"], True) def test_evaluate_with_task(self): event = { @@ -63,7 +66,8 @@ def test_evaluate_with_task(self): response = handler(event) - self.assertIsInstance(response, list) + body = json.loads(response["body"]) + self.assertIsInstance(body, list) def test_evaluate_missing_submission_returns_error(self): event = { @@ -102,8 +106,11 @@ def test_healthcheck(self): response = handler(event) - self.assertIn(response.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) - capabilities = response.get("capabilities", {}) + self.assertEqual(response["statusCode"], 200) + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertIn(body.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) + capabilities = body.get("capabilities", {}) self.assertIn("supportedAPIVersions", capabilities) self.assertIn("0.1.0", capabilities["supportedAPIVersions"]) @@ -115,7 +122,10 @@ def test_supported_version_header_is_accepted(self): response = handler(event) - self.assertIn(response.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) + self.assertEqual(response["statusCode"], 200) + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertIn(body.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) def test_unsupported_version_header_returns_error(self): event = { @@ -126,17 +136,22 @@ def test_unsupported_version_header_returns_error(self): response = handler(event) - self.assertEqual(response.get("code"), "VERSION_NOT_SUPPORTED") - self.assertIn("details", response) - self.assertEqual(response["details"]["requestedVersion"], "99.0.0") - self.assertIn("0.1.0", response["details"]["supportedVersions"]) + self.assertEqual(response["statusCode"], 406) + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertEqual(body.get("code"), "VERSION_NOT_SUPPORTED") + self.assertIn("details", body) + self.assertEqual(body["details"]["requestedVersion"], "99.0.0") + self.assertIn("0.1.0", body["details"]["supportedVersions"]) def test_absent_version_header_proceeds_normally(self): event = {"path": "/evaluate/health"} response = handler(event) - self.assertNotIn("code", response) + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertNotIn("code", body) def test_unknown_path_falls_back_to_legacy(self): event = { @@ -179,10 +194,10 @@ def test_math_submission_extracts_expression(self): "task": {"title": "T", "referenceSolution": {"expression": "x+1"}}, }, } - result = handler(event) + result = json.loads(handler(event)["body"]) self.assertEqual(self.captured["response"], "x+1") self.assertEqual(self.captured["answer"], "x+1") - self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + self.assertEqual(result[0]["awardedPoints"], True) def test_text_submission_extracts_text(self): event = { @@ -192,10 +207,10 @@ def test_text_submission_extracts_text(self): "task": {"title": "T", "referenceSolution": {"text": "hello"}}, }, } - result = handler(event) + result = json.loads(handler(event)["body"]) self.assertEqual(self.captured["response"], "hello") self.assertEqual(self.captured["answer"], "hello") - self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + self.assertEqual(result[0]["awardedPoints"], True) def test_configuration_params_forwarded(self): event = { @@ -205,9 +220,9 @@ def test_configuration_params_forwarded(self): "configuration": {"params": {"strict_syntax": False}}, }, } - result = handler(event) + result = json.loads(handler(event)["body"]) self.assertEqual(self.captured["params"], {"strict_syntax": False}) - self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + self.assertEqual(result[0]["awardedPoints"], True) def test_no_task_answer_is_none(self): event = { @@ -216,9 +231,9 @@ def test_no_task_answer_is_none(self): "submission": {"type": "MATH", "content": {"expression": "x+1"}}, }, } - result = handler(event) + result = json.loads(handler(event)["body"]) self.assertIsNone(self.captured["answer"]) - self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + self.assertEqual(result[0]["awardedPoints"], True) class TestMuEdPreviewHandlerFunction(unittest.TestCase): @@ -249,8 +264,10 @@ def test_preview_returns_feedback_list(self): response = handler(event) - self.assertIsInstance(response, list) - self.assertEqual(len(response), 1) + self.assertEqual(response["statusCode"], 200) + body = json.loads(response["body"]) + self.assertIsInstance(body, list) + self.assertEqual(len(body), 1) def test_preview_feedback_id_is_preSubmissionFeedback(self): event = { @@ -261,9 +278,9 @@ def test_preview_feedback_id_is_preSubmissionFeedback(self): }, } - response = handler(event) + body = json.loads(handler(event)["body"]) - self.assertNotIn("feedbackId", response[0]) # type: ignore + self.assertNotIn("feedbackId", body[0]) def test_preview_contains_preSubmissionFeedback_field(self): event = { @@ -274,9 +291,9 @@ def test_preview_contains_preSubmissionFeedback_field(self): }, } - response = handler(event) + body = json.loads(handler(event)["body"]) - self.assertIn("preSubmissionFeedback", response[0]) # type: ignore + self.assertIn("preSubmissionFeedback", body[0]) def test_preview_preSubmissionFeedback_has_latex_and_sympy(self): event = { @@ -287,9 +304,9 @@ def test_preview_preSubmissionFeedback_has_latex_and_sympy(self): }, } - response = handler(event) + body = json.loads(handler(event)["body"]) - preview = response[0]["preSubmissionFeedback"] # type: ignore + preview = body[0]["preSubmissionFeedback"] self.assertIn("latex", preview) self.assertIn("sympy", preview) @@ -416,7 +433,8 @@ def test_no_task_required(self): response = handler(event) - self.assertIsInstance(response, list) + body = json.loads(response["body"]) + self.assertIsInstance(body, list) self.assertEqual(self.captured["response"], "sin(x)") def test_preview_result_propagated(self): @@ -428,11 +446,11 @@ def test_preview_result_propagated(self): }, } - response = handler(event) + body = json.loads(handler(event)["body"]) - self.assertEqual(response[0]["preSubmissionFeedback"]["latex"], "captured") # type: ignore - self.assertEqual(response[0]["preSubmissionFeedback"]["sympy"], "x+1") # type: ignore + self.assertEqual(body[0]["preSubmissionFeedback"]["latex"], "captured") + self.assertEqual(body[0]["preSubmissionFeedback"]["sympy"], "x+1") if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() From 85c1a8cfe4f7be37a030d1b1fc74fcf82f36ac68 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 20 Apr 2026 10:09:10 +0100 Subject: [PATCH 11/11] Fixed test leakage --- tests/mued_handling_test.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/mued_handling_test.py b/tests/mued_handling_test.py index 66c15f5..e834b3a 100644 --- a/tests/mued_handling_test.py +++ b/tests/mued_handling_test.py @@ -18,12 +18,13 @@ class TestMuEdHandlerFunction(unittest.TestCase): def setUp(self) -> None: os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_eval = commands.evaluation_function commands.evaluation_function = evaluation_function return super().setUp() def tearDown(self) -> None: os.environ.pop("SCHEMA_DIR", None) - commands.evaluation_function = None + commands.evaluation_function = self._orig_eval return super().tearDown() def test_evaluate_returns_feedback_list(self): @@ -169,6 +170,7 @@ def test_unknown_path_falls_back_to_legacy(self): class TestMuEdEvaluateExtraction(unittest.TestCase): def setUp(self) -> None: os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_eval = commands.evaluation_function self.captured: dict = {} captured = self.captured @@ -183,7 +185,7 @@ def capturing_eval(response, answer, params): def tearDown(self) -> None: os.environ.pop("SCHEMA_DIR", None) - commands.evaluation_function = None + commands.evaluation_function = self._orig_eval return super().tearDown() def test_math_submission_extracts_expression(self): @@ -239,6 +241,8 @@ def test_no_task_answer_is_none(self): class TestMuEdPreviewHandlerFunction(unittest.TestCase): def setUp(self) -> None: os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_preview = commands.preview_function + self._orig_eval = commands.evaluation_function commands.preview_function = lambda response, params: { "preview": {"latex": f"\\text{{{response}}}", "sympy": response} } @@ -249,8 +253,8 @@ def setUp(self) -> None: def tearDown(self) -> None: os.environ.pop("SCHEMA_DIR", None) - commands.preview_function = None - commands.evaluation_function = None + commands.preview_function = self._orig_preview + commands.evaluation_function = self._orig_eval return super().tearDown() def test_preview_returns_feedback_list(self): @@ -358,14 +362,16 @@ def test_presubmission_disabled_runs_normal_evaluation(self): response = handler(event) - self.assertIsInstance(response, list) - self.assertIn("awardedPoints", response[0]) # type: ignore - self.assertNotIn("preSubmissionFeedback", response[0]) # type: ignore + body = json.loads(response["body"]) + self.assertIsInstance(body, list) + self.assertIn("awardedPoints", body[0]) + self.assertNotIn("preSubmissionFeedback", body[0]) class TestMuEdPreviewExtraction(unittest.TestCase): def setUp(self) -> None: os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_preview = commands.preview_function self.captured: dict = {} captured = self.captured @@ -379,7 +385,7 @@ def capturing_preview(response, params): def tearDown(self) -> None: os.environ.pop("SCHEMA_DIR", None) - commands.preview_function = None + commands.preview_function = self._orig_preview return super().tearDown() def test_math_submission_extracts_expression(self):