diff --git a/handler.py b/handler.py index aa1b322..0773461 100755 --- a/handler.py +++ b/handler.py @@ -1,8 +1,11 @@ +import json from evaluation_function_utils.errors import EvaluationException from .tools import commands, docs, parse, validate from .tools.parse import ParseError -from .tools.utils import ErrorResponse, HandlerResponse, JsonType, Response +from typing import Any, Optional + +from .tools.utils import DocsResponse, ErrorResponse, HandlerResponse, JsonType, Response from .tools.validate import ( LegacyReqBodyValidators, LegacyResBodyValidators, @@ -58,6 +61,57 @@ def handle_legacy_command(event: JsonType, command: str) -> HandlerResponse: return response +def wrap_muEd_response(body: Any, event: JsonType, status_code: int = 200) -> DocsResponse: + """Wrap a muEd response body in Lambda proxy format with X-Api-Version header. + + Args: + body: The response body to serialise. + event (JsonType): The incoming event (used to resolve the served version). + status_code (int): The HTTP status code. Defaults to 200. + + Returns: + DocsResponse: Proxy-format response with X-Api-Version header set. + """ + requested = (event.get("headers") or {}).get("X-Api-Version") + if requested and requested in commands.SUPPORTED_MUED_VERSIONS: + version = requested + else: + version = commands.SUPPORTED_MUED_VERSIONS[-1] + return DocsResponse( + statusCode=status_code, + headers={"X-Api-Version": version}, + body=json.dumps(body), + isBase64Encoded=False, + ) + + +def check_muEd_version(event: JsonType) -> Optional[HandlerResponse]: + """Check the X-Api-Version header against supported muEd versions. + + Args: + event (JsonType): The AWS Lambda event received by the handler. + + Returns: + Optional[HandlerResponse]: A version-not-supported error response if + the requested version is unsupported, otherwise None. + """ + version = (event.get("headers") or {}).get("X-Api-Version") + if version and version not in commands.SUPPORTED_MUED_VERSIONS: + return { + "title": "API version not supported", + "message": ( + f"The requested API version '{version}' is not supported. " + f"Supported versions are: {commands.SUPPORTED_MUED_VERSIONS}." + ), + "code": "VERSION_NOT_SUPPORTED", + "details": { + "requestedVersion": version, + "supportedVersions": commands.SUPPORTED_MUED_VERSIONS, + }, + } + return None + + def handle_muEd_command(event: JsonType, command: str) -> HandlerResponse: """Switch case for handling different command options using muEd schemas. @@ -68,21 +122,51 @@ def handle_muEd_command(event: JsonType, command: str) -> HandlerResponse: Returns: HandlerResponse: The response object returned by the handler. """ - if command == "eval": - body = parse.body(event) - validate.body(body, MuEdReqBodyValidators.EVALUATION) - response = commands.evaluate_muEd(body) - validate.body(response, MuEdResBodyValidators.EVALUATION) + try: + version_error = check_muEd_version(event) + if version_error: + return wrap_muEd_response(version_error, event, 406) + + if command == "eval": + body = parse.body(event) + validate.body(body, MuEdReqBodyValidators.EVALUATION) + response = commands.evaluate_muEd(body) + validate.body(response, MuEdResBodyValidators.EVALUATION) + + elif command == "healthcheck": + response = commands.healthcheck_muEd() + validate.body(response, MuEdResBodyValidators.HEALTHCHECK) + status_code = 503 if response.get("status") == "UNAVAILABLE" else 200 + return wrap_muEd_response(response, event, status_code) - elif command == "healthcheck": - response = commands.healthcheck() + else: + error = { + "title": "Not implemented", + "message": f"Unknown command '{command}'.", + "code": "NOT_IMPLEMENTED", + } + return wrap_muEd_response(error, event, 501) - else: - response = Response( - error=ErrorResponse(message=f"Unknown command '{command}'.") - ) + return wrap_muEd_response(response, event) - return response + except (ParseError, ValidationError) as e: + error = { + "title": "Bad request", + "message": e.message, + "code": "VALIDATION_ERROR", + "details": {"error": str(e.error_thrown)} if e.error_thrown else None, + } + return wrap_muEd_response(error, event, 400) + + except EvaluationException as e: + detail = str(e) if str(e) else repr(e) + error = {"title": "Internal server error", "message": detail, "code": "INTERNAL_ERROR"} + return wrap_muEd_response(error, event, 500) + + except Exception as e: + detail = str(e) if str(e) else repr(e) + error = {"title": "Internal server error", "message": detail, "code": "INTERNAL_ERROR"} + return wrap_muEd_response(error, event, 500) def handler(event: JsonType, _=None) -> HandlerResponse: diff --git a/schemas/muEd/openapi-v0_1_0.yml b/schemas/muEd/openapi-v0_1_0.yml index 28a8377..9004264 100644 --- a/schemas/muEd/openapi-v0_1_0.yml +++ b/schemas/muEd/openapi-v0_1_0.yml @@ -18,12 +18,13 @@ paths: summary: Evaluate a submission and generate feedback operationId: evaluateSubmission description: | - Generates a list of feedback items for a given student submission. The request can optionally include the task context, user information, criteria to evaluate on, pre-submission feedback options, configuration, and a callback URL for asynchronous result delivery. + Generates a list of feedback items for a given student submission. The request can optionally include the task context, user information, criteria to evaluate on, pre-submission feedback options, and configuration. tags: - evaluate parameters: - $ref: '#/components/parameters/Authorization' - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' requestBody: required: true content: @@ -60,19 +61,6 @@ paths: llm: model: gpt-5.2 temperature: 0.4 - asyncCallbackExample: - summary: Asynchronous processing via callback URL - value: - submission: - submissionId: sub-async-001 - taskId: task-42 - type: TEXT - format: plain - content: - text: Detailed essay answer that may require longer processing. - submittedAt: '2025-12-16T09:45:00Z' - version: 1 - callbackUrl: https://learning-platform.example.com/hooks/evaluate-result withTaskAndExtras: summary: With task context, user, criteria and configuration value: @@ -340,6 +328,10 @@ paths: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -370,12 +362,12 @@ paths: - feedbackId: fb-2 title: Overall structure message: The overall structure of your answer is clear and easy to follow. - '202': - $ref: '#/components/responses/202-Accepted' '400': $ref: '#/components/responses/400-BadRequest' '403': $ref: '#/components/responses/403-Forbidden' + '406': + $ref: '#/components/responses/406-VersionNotSupported' '500': $ref: '#/components/responses/500-InternalError' '501': @@ -390,9 +382,19 @@ paths: - evaluate parameters: - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' responses: '200': description: Evaluate service is reachable and reporting capabilities. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -427,6 +429,10 @@ paths: supportedLanguages: - en - de + supportedVersions: + - 0.1.0 + '406': + $ref: '#/components/responses/406-VersionNotSupported' '501': description: The server does not implement the health endpoint for evaluate. $ref: '#/components/responses/501-NotImplemented' @@ -443,6 +449,7 @@ paths: parameters: - $ref: '#/components/parameters/Authorization' - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' requestBody: required: true content: @@ -567,6 +574,10 @@ paths: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -616,6 +627,8 @@ paths: $ref: '#/components/responses/400-BadRequest-2' '403': $ref: '#/components/responses/403-Forbidden' + '406': + $ref: '#/components/responses/406-VersionNotSupported' '500': $ref: '#/components/responses/500-InternalError-2' '501': @@ -630,9 +643,19 @@ paths: - chat parameters: - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' responses: '200': description: Chat service is reachable and reporting capabilities. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -655,6 +678,10 @@ paths: supportedModels: - gpt-4o - llama-3 + supportedVersions: + - 0.1.0 + '406': + $ref: '#/components/responses/406-VersionNotSupported' '501': description: The server does not implement the health endpoint for chat. $ref: '#/components/responses/501-NotImplemented-2' @@ -675,6 +702,15 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + in: header + name: X-Api-Version + description: | + The µEd API version the client is targeting (e.g. "0.1.0"). If omitted, the server will use the latest version it supports. If the requested version cannot be served, the server returns 406 Version Not Supported. + required: false + schema: + type: string + example: 0.1.0 schemas: Task: type: object @@ -757,10 +793,6 @@ components: additionalProperties: true description: | Logical representation of the submission content. The expected structure depends on the artefact type: - TEXT: { text: string } or { markdown: string } - CODE: { code: string } or { files: [{ path: string, content: string }], entryPoint?: string } - MATH: { expression: string } - MODEL: { model: string | object, notation?: string } - supplementaryContent: - type: object - additionalProperties: true - description: Optional additional content for the learner's submission. This could include workings for math tasks, or raw source code for compiled binaries. submittedAt: type: - string @@ -1201,7 +1233,7 @@ components: EvaluateRequest: type: object description: | - Input for task evaluate service. The submission is mandatory; task, user, criteria, pre-submission feedback options, and configuration, and callback URL are optional. + Input for task evaluate service. The submission is mandatory; task, user, criteria, pre-submission feedback options, and configuration are optional. required: - submission properties: @@ -1237,13 +1269,6 @@ components: - 'null' allOf: - $ref: '#/components/schemas/PreSubmissionFeedback' - callbackUrl: - type: - - string - - 'null' - format: uri - description: | - Optional HTTPS callback URL for asynchronous processing. If provided, the service may return 202 Accepted immediately and deliver feedback results to this URL once processing is complete. configuration: description: | Optional key-value configuration dictionary for provider-specific or experimental parameters. Not standardized. @@ -1337,26 +1362,6 @@ components: description: Optional target reference inside the submission. allOf: - $ref: '#/components/schemas/FeedbackTarget' - EvaluateAcceptedResponse: - type: object - description: Acknowledgement that evaluation was accepted for asynchronous processing. - required: - - status - - requestId - properties: - status: - type: string - enum: - - ACCEPTED - description: Indicates that the request has been accepted for asynchronous processing. - requestId: - type: string - description: Identifier to correlate this accepted request with callback delivery. - message: - type: - - string - - 'null' - description: Optional human-readable message about asynchronous processing. ErrorResponse: type: object description: Standard error response returned by µEd API services. @@ -1503,6 +1508,14 @@ components: description: Optional list of supported language codes (e.g., 'en', 'de'). items: type: string + supportedAPIVersions: + type: + - array + - 'null' + description: | + Optional list of µEd API versions supported by this service implementation (e.g., ["0.1.0"]). Clients can use this to select a compatible X-Api-Version. + items: + type: string EvaluateHealthResponse: type: object description: Health status and capabilities of the evaluate service. @@ -1656,6 +1669,14 @@ components: description: Optional list of supported models. items: type: string + supportedAPIVersions: + type: + - array + - 'null' + description: | + Optional list of µEd API versions supported by this service implementation (e.g., ["0.1.0"]). Clients can use this to select a compatible X-Api-Version. + items: + type: string ChatHealthResponse: type: object description: Health status and capabilities of the chat service. @@ -1678,24 +1699,6 @@ components: capabilities: $ref: '#/components/schemas/ChatCapabilities' responses: - 202-Accepted: - description: Request accepted for asynchronous evaluation processing. - headers: - X-Request-Id: - description: Request id for tracing this request across services. - schema: - type: string - content: - application/json: - schema: - $ref: '#/components/schemas/EvaluateAcceptedResponse' - examples: - asyncAccepted: - summary: Example accepted async request - value: - status: ACCEPTED - requestId: req-7c193f38 - message: Evaluation queued. Results will be sent to callbackUrl. 400-BadRequest: description: Invalid request (e.g. missing content or invalid schema). headers: @@ -1703,6 +1706,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1724,6 +1731,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1739,6 +1750,34 @@ components: details: resource: submission required_permission: write + 406-VersionNotSupported: + description: | + The requested API version (supplied via X-Api-Version) is not supported by this service. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + versionNotSupported: + summary: Example version not supported error + value: + title: API version not supported + message: 'The requested API version ''0.0'' is not supported. Supported versions are: [''0.1.0''].' + code: VERSION_NOT_SUPPORTED + trace: null + details: + requestedVersion: '0.0' + supportedVersions: + - 0.1.0 500-InternalError: description: Internal server error. headers: @@ -1746,6 +1785,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1768,6 +1811,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1788,6 +1835,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1824,6 +1875,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1845,6 +1900,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1867,6 +1926,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: @@ -1887,6 +1950,10 @@ components: description: Request id for tracing this request across services. schema: type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string content: application/json: schema: diff --git a/tests/commands_test.py b/tests/commands_test.py index a9f249a..29cc992 100644 --- a/tests/commands_test.py +++ b/tests/commands_test.py @@ -365,6 +365,15 @@ def test_healthcheck(self): self.assertIn("tests_passed", result) + def test_healthcheck_muEd(self): + response = commands.healthcheck_muEd() + + self.assertIn(response["status"], ("OK", "DEGRADED", "UNAVAILABLE")) + capabilities = response["capabilities"] + self.assertIn("supportedAPIVersions", capabilities) + self.assertIsInstance(capabilities["supportedAPIVersions"], list) + self.assertIn("0.1.0", capabilities["supportedAPIVersions"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/mued_handling_test.py b/tests/mued_handling_test.py index 263c3e5..d077414 100644 --- a/tests/mued_handling_test.py +++ b/tests/mued_handling_test.py @@ -1,3 +1,4 @@ +import json import os import unittest from pathlib import Path @@ -17,12 +18,13 @@ class TestMuEdHandlerFunction(unittest.TestCase): def setUp(self) -> None: os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_eval = commands.evaluation_function commands.evaluation_function = evaluation_function return super().setUp() def tearDown(self) -> None: os.environ.pop("SCHEMA_DIR", None) - commands.evaluation_function = None + commands.evaluation_function = self._orig_eval return super().tearDown() def test_evaluate_returns_feedback_list(self): @@ -33,9 +35,11 @@ def test_evaluate_returns_feedback_list(self): response = handler(event) - self.assertIsInstance(response, list) - self.assertEqual(len(response), 1) - self.assertIn("awardedPoints", response[0]) + self.assertEqual(response["statusCode"], 200) + body = json.loads(response["body"]) + self.assertIsInstance(body, list) + self.assertEqual(len(body), 1) + self.assertIn("awardedPoints", body[0]) def test_evaluate_feedback_message(self): event = { @@ -45,9 +49,9 @@ def test_evaluate_feedback_message(self): response = handler(event) - self.assertIsInstance(response, list) - self.assertEqual(response[0]["message"], "Well done.") - self.assertEqual(response[0]["awardedPoints"], True) + body = json.loads(response["body"]) + self.assertEqual(body[0]["message"], "Well done.") + self.assertEqual(body[0]["awardedPoints"], True) def test_evaluate_with_task(self): event = { @@ -63,7 +67,8 @@ def test_evaluate_with_task(self): response = handler(event) - self.assertIsInstance(response, list) + body = json.loads(response["body"]) + self.assertIsInstance(body, list) def test_evaluate_missing_submission_returns_error(self): event = { @@ -73,8 +78,9 @@ def test_evaluate_missing_submission_returns_error(self): response = handler(event) - self.assertIn("error", response) - self.assertIn("submission", response["error"]["detail"]) # type: ignore + self.assertEqual(response["statusCode"], 400) + body = json.loads(response["body"]) + self.assertEqual(body["code"], "VALIDATION_ERROR") def test_evaluate_invalid_submission_type_returns_error(self): event = { @@ -84,26 +90,71 @@ def test_evaluate_invalid_submission_type_returns_error(self): response = handler(event) - self.assertIn("error", response) + self.assertEqual(response["statusCode"], 400) + body = json.loads(response["body"]) + self.assertEqual(body["code"], "VALIDATION_ERROR") def test_evaluate_bodyless_event_returns_error(self): event = {"path": "/evaluate", "random": "metadata"} response = handler(event) - self.assertIn("error", response) - self.assertEqual( - response["error"]["message"], # type: ignore - "No data supplied in request body.", - ) + self.assertEqual(response["statusCode"], 400) + body = json.loads(response["body"]) + self.assertEqual(body["code"], "VALIDATION_ERROR") + self.assertEqual(body["message"], "No data supplied in request body.") def test_healthcheck(self): event = {"path": "/evaluate/health"} response = handler(event) - self.assertEqual(response.get("command"), "healthcheck") - self.assertIn("result", response) + self.assertEqual(response["statusCode"], 200) + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertIn(body.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) + capabilities = body.get("capabilities", {}) + self.assertIn("supportedAPIVersions", capabilities) + self.assertIn("0.1.0", capabilities["supportedAPIVersions"]) + + def test_supported_version_header_is_accepted(self): + event = { + "path": "/evaluate/health", + "headers": {"X-Api-Version": "0.1.0"}, + } + + response = handler(event) + + self.assertEqual(response["statusCode"], 200) + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertIn(body.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) + + def test_unsupported_version_header_returns_error(self): + event = { + "path": "/evaluate", + "headers": {"X-Api-Version": "99.0.0"}, + "body": {"submission": {"type": "TEXT", "content": {}}}, + } + + response = handler(event) + + self.assertEqual(response["statusCode"], 406) + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertEqual(body.get("code"), "VERSION_NOT_SUPPORTED") + self.assertIn("details", body) + self.assertEqual(body["details"]["requestedVersion"], "99.0.0") + self.assertIn("0.1.0", body["details"]["supportedVersions"]) + + def test_absent_version_header_proceeds_normally(self): + event = {"path": "/evaluate/health"} + + response = handler(event) + + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertNotIn("code", body) def test_unknown_path_falls_back_to_legacy(self): event = { @@ -149,6 +200,7 @@ def test_evaluate_response_latex_and_simplified_populated_when_returned(self): class TestMuEdEvaluateExtraction(unittest.TestCase): def setUp(self) -> None: os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_eval = commands.evaluation_function self.captured: dict = {} captured = self.captured @@ -163,7 +215,7 @@ def capturing_eval(response, answer, params): def tearDown(self) -> None: os.environ.pop("SCHEMA_DIR", None) - commands.evaluation_function = None + commands.evaluation_function = self._orig_eval return super().tearDown() def test_math_submission_extracts_expression(self): @@ -174,10 +226,10 @@ def test_math_submission_extracts_expression(self): "task": {"title": "T", "referenceSolution": {"expression": "x+1"}}, }, } - result = handler(event) + result = json.loads(handler(event)["body"]) self.assertEqual(self.captured["response"], "x+1") self.assertEqual(self.captured["answer"], "x+1") - self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + self.assertEqual(result[0]["awardedPoints"], True) def test_text_submission_extracts_text(self): event = { @@ -187,10 +239,10 @@ def test_text_submission_extracts_text(self): "task": {"title": "T", "referenceSolution": {"text": "hello"}}, }, } - result = handler(event) + result = json.loads(handler(event)["body"]) self.assertEqual(self.captured["response"], "hello") self.assertEqual(self.captured["answer"], "hello") - self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + self.assertEqual(result[0]["awardedPoints"], True) def test_other_submission_extracts_value(self): event = { @@ -236,9 +288,9 @@ def test_configuration_params_forwarded(self): "configuration": {"params": {"strict_syntax": False}}, }, } - result = handler(event) + result = json.loads(handler(event)["body"]) self.assertEqual(self.captured["params"], {"strict_syntax": False}) - self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + self.assertEqual(result[0]["awardedPoints"], True) def test_no_task_answer_is_none(self): event = { @@ -247,14 +299,16 @@ def test_no_task_answer_is_none(self): "submission": {"type": "MATH", "content": {"expression": "x+1"}}, }, } - result = handler(event) + result = json.loads(handler(event)["body"]) self.assertIsNone(self.captured["answer"]) - self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + self.assertEqual(result[0]["awardedPoints"], True) class TestMuEdPreviewHandlerFunction(unittest.TestCase): def setUp(self) -> None: os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_preview = commands.preview_function + self._orig_eval = commands.evaluation_function commands.preview_function = lambda response, params: { "preview": {"latex": f"\\text{{{response}}}", "sympy": response} } @@ -265,8 +319,8 @@ def setUp(self) -> None: def tearDown(self) -> None: os.environ.pop("SCHEMA_DIR", None) - commands.preview_function = None - commands.evaluation_function = None + commands.preview_function = self._orig_preview + commands.evaluation_function = self._orig_eval return super().tearDown() def test_preview_returns_feedback_list(self): @@ -280,8 +334,10 @@ def test_preview_returns_feedback_list(self): response = handler(event) - self.assertIsInstance(response, list) - self.assertEqual(len(response), 1) + self.assertEqual(response["statusCode"], 200) + body = json.loads(response["body"]) + self.assertIsInstance(body, list) + self.assertEqual(len(body), 1) def test_preview_feedback_id_is_preSubmissionFeedback(self): event = { @@ -292,9 +348,9 @@ def test_preview_feedback_id_is_preSubmissionFeedback(self): }, } - response = handler(event) + body = json.loads(handler(event)["body"]) - self.assertNotIn("feedbackId", response[0]) # type: ignore + self.assertNotIn("feedbackId", body[0]) def test_preview_contains_preSubmissionFeedback_field(self): event = { @@ -305,9 +361,9 @@ def test_preview_contains_preSubmissionFeedback_field(self): }, } - response = handler(event) + body = json.loads(handler(event)["body"]) - self.assertIn("preSubmissionFeedback", response[0]) # type: ignore + self.assertIn("preSubmissionFeedback", body[0]) def test_preview_preSubmissionFeedback_has_latex_and_sympy(self): event = { @@ -318,9 +374,9 @@ def test_preview_preSubmissionFeedback_has_latex_and_sympy(self): }, } - response = handler(event) + body = json.loads(handler(event)["body"]) - preview = response[0]["preSubmissionFeedback"] # type: ignore + preview = body[0]["preSubmissionFeedback"] self.assertIn("latex", preview) self.assertIn("sympy", preview) @@ -335,18 +391,19 @@ def test_preview_missing_submission_returns_error(self): response = handler(event) - self.assertIn("error", response) + self.assertEqual(response["statusCode"], 400) + body = json.loads(response["body"]) + self.assertEqual(body["code"], "VALIDATION_ERROR") def test_preview_bodyless_event_returns_error(self): event = {"path": "/evaluate", "random": "metadata"} response = handler(event) - self.assertIn("error", response) - self.assertEqual( - response["error"]["message"], # type: ignore - "No data supplied in request body.", - ) + self.assertEqual(response["statusCode"], 400) + body = json.loads(response["body"]) + self.assertEqual(body["code"], "VALIDATION_ERROR") + self.assertEqual(body["message"], "No data supplied in request body.") def test_preview_invalid_submission_type_returns_error(self): event = { @@ -359,7 +416,9 @@ def test_preview_invalid_submission_type_returns_error(self): response = handler(event) - self.assertIn("error", response) + self.assertEqual(response["statusCode"], 400) + body = json.loads(response["body"]) + self.assertEqual(body["code"], "VALIDATION_ERROR") def test_presubmission_disabled_runs_normal_evaluation(self): event = { @@ -372,14 +431,16 @@ def test_presubmission_disabled_runs_normal_evaluation(self): response = handler(event) - self.assertIsInstance(response, list) - self.assertIn("awardedPoints", response[0]) # type: ignore - self.assertNotIn("preSubmissionFeedback", response[0]) # type: ignore + body = json.loads(response["body"]) + self.assertIsInstance(body, list) + self.assertIn("awardedPoints", body[0]) + self.assertNotIn("preSubmissionFeedback", body[0]) class TestMuEdPreviewExtraction(unittest.TestCase): def setUp(self) -> None: os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_preview = commands.preview_function self.captured: dict = {} captured = self.captured @@ -393,7 +454,7 @@ def capturing_preview(response, params): def tearDown(self) -> None: os.environ.pop("SCHEMA_DIR", None) - commands.preview_function = None + commands.preview_function = self._orig_preview return super().tearDown() def test_math_submission_extracts_expression(self): @@ -473,7 +534,8 @@ def test_no_task_required(self): response = handler(event) - self.assertIsInstance(response, list) + body = json.loads(response["body"]) + self.assertIsInstance(body, list) self.assertEqual(self.captured["response"], "sin(x)") def test_preview_result_propagated(self): @@ -485,10 +547,10 @@ def test_preview_result_propagated(self): }, } - response = handler(event) + body = json.loads(handler(event)["body"]) - self.assertEqual(response[0]["preSubmissionFeedback"]["latex"], "captured") # type: ignore - self.assertEqual(response[0]["preSubmissionFeedback"]["sympy"], "x+1") # type: ignore + self.assertEqual(body[0]["preSubmissionFeedback"]["latex"], "captured") + self.assertEqual(body[0]["preSubmissionFeedback"]["sympy"], "x+1") class TestMuEdMatchedCase(unittest.TestCase): @@ -645,4 +707,4 @@ def test_matched_case_with_mark_override(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tools/commands.py b/tools/commands.py index 0252042..fac46da 100644 --- a/tools/commands.py +++ b/tools/commands.py @@ -55,9 +55,13 @@ class CaseResult(NamedTuple): is_correct: bool = False feedback: str = "" + warning: Optional[CaseWarning] = None +SUPPORTED_MUED_VERSIONS: list[str] = ["0.1.0"] + + def healthcheck() -> Response: """Run the healthcheck command for the evaluation function. @@ -68,6 +72,27 @@ def healthcheck() -> Response: return Response(command="healthcheck", result=result) +def healthcheck_muEd() -> Dict: + """Run the healthcheck command and return a muEd EvaluateHealthResponse. + + Returns: + Dict: A spec-compliant EvaluateHealthResponse. + """ + result = health.healthcheck() + status = "OK" if result["tests_passed"] else "DEGRADED" + return { + "status": status, + "capabilities": { + "supportsEvaluate": True, + "supportsPreSubmissionFeedback": True, + "supportsFormativeFeedback": True, + "supportsSummativeFeedback": False, + "supportsDataPolicy": "NOT_SUPPORTED", + "supportedAPIVersions": SUPPORTED_MUED_VERSIONS, + }, + } + + def preview( body: JsonType, fnc: Optional[PreviewFunctionType] = None ) -> Response: