diff --git a/docs/development/code.md b/docs/development/code.md index e625892..70b53f0 100644 --- a/docs/development/code.md +++ b/docs/development/code.md @@ -87,6 +87,12 @@ async def tag_dataset( 11. Log statements give observability to what's happening in the REST API (see [Logging](#logging)). 12. Return values are coerced into JSON form by FastAPI. +## Database sessions +The `expdb_session` and `userdb_session` dependencies commit the transaction when the endpoint finishes. +Always declare them (and `fetch_user`, which uses a session) with `scope="function"`, for example `Depends(expdb_session, scope="function")`. +Without it, FastAPI commits *after* the response is sent, so a client can get a success response for data that is not saved yet. +A test checks that every router follows this rule. + ## Logging We use [`loguru`](https://loguru.readthedocs.io/en/stable/) for logging. A [middleware](https://fastapi.tiangolo.com/reference/middleware/) layer automatically provides the following context to each log entry (see `core/logging.py::add_request_context_to_log`): diff --git a/pyproject.toml b/pyproject.toml index 3ff160e..ccc718a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "fastapi", + "fastapi>=0.121", "loguru", "cryptography", "pydantic", diff --git a/src/routers/datasets.py b/src/routers/datasets.py index 177be32..b91be4e 100644 --- a/src/routers/datasets.py +++ b/src/routers/datasets.py @@ -94,7 +94,7 @@ async def tag_dataset( data_id: Annotated[Identifier, Body()], tag: Annotated[TagString, Body()], user: Annotated[User, Depends(fetch_user_or_raise)], - expdb_db: Annotated[AsyncSession, Depends(expdb_session)], + expdb_db: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> dict[str, dict[str, Any]]: """Add a tag to the dataset, this tag is publicly visible to all users.""" try: @@ -120,7 +120,7 @@ async def untag_dataset_like_php( data_id: Annotated[Identifier, Body()], tag: Annotated[TagString, Body()], user: Annotated[User, Depends(fetch_user_or_raise)], - expdb_db: Annotated[AsyncSession, Depends(expdb_session)], + expdb_db: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> dict[Literal["data_untag"], TagInfo]: """Remove a tag from the dataset with a response similar to the old PHP server.""" await untag_dataset(data_id, tag, user, expdb_db) @@ -138,7 +138,7 @@ async def untag_dataset( identifier: Identifier, tag: Annotated[TagString, Query()], user: Annotated[User, Depends(fetch_user_or_raise)], - expdb_db: Annotated[AsyncSession, Depends(expdb_session)], + expdb_db: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> None: """Remove a tag that you added to a dataset, or from a dataset you uploaded.""" dataset_tag = await database.datasets.get_tag(identifier, tag, expdb_db) @@ -186,7 +186,7 @@ def _quality_clause(quality: str, range_: str | None) -> str: @router.get(path="/list") async def list_datasets( # noqa: PLR0913, C901 *, - expdb_db: Annotated[AsyncSession, Depends(expdb_session)], + expdb_db: Annotated[AsyncSession, Depends(expdb_session, scope="function")], pagination: Annotated[Pagination, Body(default_factory=Pagination)], data_name: Annotated[CasualString128 | None, Body()] = None, tag: Annotated[TagString | None, Body()] = None, @@ -210,7 +210,7 @@ async def list_datasets( # noqa: PLR0913, C901 number_classes: Annotated[IntegerRange | None, Body()] = None, number_missing_values: Annotated[IntegerRange | None, Body()] = None, status: Annotated[DatasetStatusFilter, Body()] = DatasetStatusFilter.ACTIVE, - user: Annotated[User | None, Depends(fetch_user)] = None, + user: Annotated[User | None, Depends(fetch_user, scope="function")] = None, ) -> list[dict[str, Any]]: """List all datasets that match the filters.""" status_subquery = text( @@ -388,8 +388,8 @@ async def _get_dataset_raise_otherwise( @router.get("/features/{dataset_id}", response_model_exclude_none=True) async def get_dataset_features( dataset_id: Identifier, - expdb: Annotated[AsyncSession, Depends(expdb_session)], - user: Annotated[User | None, Depends(fetch_user)] = None, + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], + user: Annotated[User | None, Depends(fetch_user, scope="function")] = None, ) -> list[Feature]: """Return metadata for each feature (column) in the dataset.""" assert expdb is not None # noqa: S101 @@ -432,7 +432,7 @@ async def update_dataset_status( dataset_id: Annotated[Identifier, Body()], status: Annotated[Literal[DatasetStatus.ACTIVE, DatasetStatus.DEACTIVATED], Body()], user: Annotated[User, Depends(fetch_user_or_raise)], - expdb: Annotated[AsyncSession, Depends(expdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> dict[str, str | int]: """Update the status of the dataset. Can be used to deactivate a dataset.""" dataset = await _get_dataset_raise_otherwise(dataset_id, user, expdb) @@ -485,9 +485,9 @@ async def update_dataset_status( ) async def get_dataset( dataset_id: Identifier, - userdb_session: Annotated[AsyncSession, Depends(userdb_session)], - expdb_session: Annotated[AsyncSession, Depends(expdb_session)], - user: Annotated[User | None, Depends(fetch_user)] = None, + userdb_session: Annotated[AsyncSession, Depends(userdb_session, scope="function")], + expdb_session: Annotated[AsyncSession, Depends(expdb_session, scope="function")], + user: Annotated[User | None, Depends(fetch_user, scope="function")] = None, ) -> DatasetMetadata: """Get the user-provided metadata for a dataset.""" dataset = await _get_dataset_raise_otherwise(dataset_id, user, expdb_session) diff --git a/src/routers/dependencies.py b/src/routers/dependencies.py index 2ed817f..bb91857 100644 --- a/src/routers/dependencies.py +++ b/src/routers/dependencies.py @@ -30,7 +30,7 @@ async def userdb_session() -> AsyncIterator[AsyncSession]: async def fetch_user( api_key: APIKey | None = None, - user_data: Annotated[AsyncSession | None, Depends(userdb_session)] = None, + user_data: Annotated[AsyncSession | None, Depends(userdb_session, scope="function")] = None, ) -> AsyncGenerator[User | None]: if not (api_key and user_data): yield None @@ -53,7 +53,7 @@ async def fetch_user( def fetch_user_or_raise( - user: Annotated[User | None, Depends(fetch_user)] = None, + user: Annotated[User | None, Depends(fetch_user, scope="function")] = None, ) -> User: if user is None: logger.info("Unauthenticated user tried to access endpoint that requires authentication.") diff --git a/src/routers/estimation_procedure.py b/src/routers/estimation_procedure.py index 48904f5..97590e8 100644 --- a/src/routers/estimation_procedure.py +++ b/src/routers/estimation_procedure.py @@ -16,7 +16,7 @@ @router.get("/list", response_model_exclude_none=True) async def get_estimation_procedures( - expdb: Annotated[AsyncSession, Depends(expdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> list[EstimationProcedure]: """Return a list with descriptions of estimation procedures. diff --git a/src/routers/evaluations.py b/src/routers/evaluations.py index a0a73be..ce56158 100644 --- a/src/routers/evaluations.py +++ b/src/routers/evaluations.py @@ -15,7 +15,7 @@ @router.get("/list") async def get_evaluation_measures( - expdb: Annotated[AsyncSession, Depends(expdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> list[str]: """Return a list of evaluation measure names. diff --git a/src/routers/flows.py b/src/routers/flows.py index 7dc1521..18eb049 100644 --- a/src/routers/flows.py +++ b/src/routers/flows.py @@ -21,7 +21,7 @@ async def flow_exists( name: str, external_version: str, - expdb: Annotated[AsyncSession, Depends(expdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> dict[Literal["flow_id"], int]: """Check if a Flow with the name and version exists, if so, return the flow id.""" flow = await database.flows.get_by_name( @@ -38,7 +38,7 @@ async def flow_exists( @router.get("/{flow_id}") async def get_flow( flow_id: Identifier, - expdb: Annotated[AsyncSession, Depends(expdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> Flow: """Get a Flow by its identifier.""" flow = await database.flows.get(flow_id, expdb) diff --git a/src/routers/qualities.py b/src/routers/qualities.py index 24d3389..f2c5aba 100644 --- a/src/routers/qualities.py +++ b/src/routers/qualities.py @@ -30,7 +30,7 @@ @router.get("/qualities/list") async def list_qualities( - expdb: Annotated[AsyncSession, Depends(expdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> dict[Literal["data_qualities_list"], dict[Literal["quality"], list[str]]]: """List names of all computed qualities (dataset metafeatures).""" qualities = await database.qualities.list_all_qualities(session=expdb) @@ -44,8 +44,8 @@ async def list_qualities( @router.get("/qualities/{dataset_id}") async def get_qualities( dataset_id: Identifier, - user: Annotated[User | None, Depends(fetch_user)], - expdb: Annotated[AsyncSession, Depends(expdb_session)], + user: Annotated[User | None, Depends(fetch_user, scope="function")], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> list[Quality]: """Get computed qualities (metafeatures) for a dataset.""" dataset = await database.datasets.get(dataset_id, expdb) diff --git a/src/routers/runs.py b/src/routers/runs.py index f23fd59..97b130b 100644 --- a/src/routers/runs.py +++ b/src/routers/runs.py @@ -36,7 +36,7 @@ @router.get("/trace/{run_id}") async def get_run_trace( run_id: Identifier, - expdb: Annotated[AsyncSession, Depends(expdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> RunTrace: """Get trace data for a run by run ID.""" if not await database.runs.exist(run_id, expdb): @@ -135,8 +135,8 @@ def _normalise_value(v: object) -> object: @router.get("/{run_id}", response_model_exclude_none=True) async def get_run( run_id: Identifier, - expdb: Annotated[AsyncSession, Depends(expdb_session)], - userdb: Annotated[AsyncSession, Depends(userdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], + userdb: Annotated[AsyncSession, Depends(userdb_session, scope="function")], ) -> Run: """Get full metadata for a run by ID.""" # Authentication is not required because runs are always public. diff --git a/src/routers/setups.py b/src/routers/setups.py index 04373d1..3dbf46a 100644 --- a/src/routers/setups.py +++ b/src/routers/setups.py @@ -27,7 +27,7 @@ @router.get(path="/{setup_id}", response_model_exclude_none=True) async def get_setup( setup_id: Annotated[Identifier, Path()], - expdb_session: Annotated[AsyncSession, Depends(expdb_session)], + expdb_session: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> SetupResponse: """Get setup by id.""" setup = await database.setups.get(setup_id, expdb_session) @@ -51,7 +51,7 @@ async def tag_setup( setup_id: Annotated[Identifier, Body()], tag: Annotated[TagString, Body()], user: Annotated[User, Depends(fetch_user_or_raise)], - expdb_session: Annotated[AsyncSession, Depends(expdb_session)], + expdb_session: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> dict[str, dict[str, str | list[str]]]: """Add a tag to the setup, this tag is publicly visible to all users.""" try: @@ -75,7 +75,7 @@ async def untag_setup( setup_id: Annotated[Identifier, Body()], tag: Annotated[TagString, Body()], user: Annotated[User, Depends(fetch_user_or_raise)], - expdb_session: Annotated[AsyncSession, Depends(expdb_session)], + expdb_session: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> dict[str, dict[str, str | list[str]]]: """Remove tag `tag` from setup with id `setup_id`.""" # Setups don't really have an owner, they are associated with runs. diff --git a/src/routers/study.py b/src/routers/study.py index 23a6d60..c67a1e2 100644 --- a/src/routers/study.py +++ b/src/routers/study.py @@ -71,7 +71,7 @@ async def attach_to_study( study_id: Annotated[Identifier, Body()], entity_ids: Annotated[list[Identifier], Body()], user: Annotated[User, Depends(fetch_user_or_raise)], - expdb: Annotated[AsyncSession, Depends(expdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> AttachDetachResponse: """Add runs or tasks to a study which is in preparation.""" if user is None: @@ -125,7 +125,7 @@ async def attach_to_study( async def create_study( study: CreateStudy, user: Annotated[User, Depends(fetch_user_or_raise)], - expdb: Annotated[AsyncSession, Depends(expdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> dict[Literal["study_id"], int]: """Create a new study.""" if study.main_entity_type == StudyType.RUN and study.tasks: @@ -161,8 +161,8 @@ async def create_study( @router.get("/{alias_or_id}") async def get_study( alias_or_id: Identifier | str, - expdb: Annotated[AsyncSession, Depends(expdb_session)], - user: Annotated[User | None, Depends(fetch_user)] = None, + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], + user: Annotated[User | None, Depends(fetch_user, scope="function")] = None, ) -> Study: """Get a study by id or alias.""" study = await _get_study_raise_otherwise(alias_or_id, user, expdb) diff --git a/src/routers/tasks.py b/src/routers/tasks.py index a57e3de..73d2bb1 100644 --- a/src/routers/tasks.py +++ b/src/routers/tasks.py @@ -40,7 +40,7 @@ async def tag_task( task_id: Annotated[Identifier, Body()], tag: Annotated[TagString, Body()], user: Annotated[User, Depends(fetch_user_or_raise)], - expdb_session: Annotated[AsyncSession, Depends(expdb_session)], + expdb_session: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> dict[str, dict[str, Any]]: """Add a tag to the task, this tag is publicly visible to all users.""" try: @@ -259,7 +259,7 @@ def _quality_clause(quality: str, range_: str | None) -> str: @router.get(path="/list") async def list_tasks( # noqa: PLR0913, PLR0912, C901, PLR0915 *, - expdb: Annotated[AsyncSession, Depends(expdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], pagination: Annotated[Pagination, Body(default_factory=Pagination)], task_type_id: Annotated[Identifier | None, Body(description="Filter by task type id.")] = None, tag: Annotated[TagString | None, Body()] = None, @@ -450,7 +450,7 @@ async def list_tasks( # noqa: PLR0913, PLR0912, C901, PLR0915 @router.get("/{task_id}") async def get_task( task_id: Identifier, - expdb: Annotated[AsyncSession, Depends(expdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> Task: """Get a task by identifier.""" if not (task := await database.tasks.get(task_id, expdb)): diff --git a/src/routers/tasktype.py b/src/routers/tasktype.py index 4e86369..0909786 100644 --- a/src/routers/tasktype.py +++ b/src/routers/tasktype.py @@ -37,7 +37,7 @@ def _normalize_task_type(task_type: Row[Any]) -> dict[str, str | list[Any] | Non @router.get(path="/list") async def list_task_types( - expdb: Annotated[AsyncSession, Depends(expdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> dict[ Literal["task_types"], dict[Literal["task_type"], list[dict[str, str | list[Any] | None]]], @@ -52,7 +52,7 @@ async def list_task_types( @router.get(path="/{task_type_id}") async def get_task_type( task_type_id: Identifier, - expdb: Annotated[AsyncSession, Depends(expdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], ) -> dict[Literal["task_type"], dict[str, str | list[str] | list[dict[str, str]] | None]]: """Return a detailed description for the given task type, including expected inputs.""" task_type_record = await db_get_task_type(task_type_id, expdb) diff --git a/src/routers/users.py b/src/routers/users.py index aba293e..1dcbe32 100644 --- a/src/routers/users.py +++ b/src/routers/users.py @@ -36,8 +36,8 @@ async def delete_user_account( user_id: Annotated[int, Path(description="Numeric user id to delete.", gt=0)], current_user: Annotated[User, Depends(fetch_user_or_raise)], - expdb: Annotated[AsyncSession, Depends(expdb_session)], - userdb: Annotated[AsyncSession, Depends(userdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session, scope="function")], + userdb: Annotated[AsyncSession, Depends(userdb_session, scope="function")], ) -> Response: """Delete the user account if they have no associated resources. diff --git a/tests/dependencies/session_commit_test.py b/tests/dependencies/session_commit_test.py new file mode 100644 index 0000000..c774870 --- /dev/null +++ b/tests/dependencies/session_commit_test.py @@ -0,0 +1,174 @@ +"""Make sure database transactions are committed *before* the response is sent. + +Since FastAPI 0.118 the exit code of a `yield` dependency runs after the response is sent, +unless the dependency is declared with `scope="function"`. +Without it, clients may get a success response for data that is not (yet) saved. +""" + +import ast +from collections.abc import AsyncIterator, Callable, MutableMapping +from pathlib import Path +from types import TracebackType +from typing import Annotated, Any, Self + +import httpx +import pytest +from fastapi import Depends, FastAPI + +import routers.dependencies +from routers.dependencies import expdb_session, fetch_user, userdb_session + +SESSION_DEPENDENCIES = (expdb_session, userdb_session, fetch_user) +SOURCE_DIRECTORY = Path(__file__).parents[2] / "src" + + +class _FakeTransaction: + def __init__(self, events: list[str], *, fail_commit: bool) -> None: + self._events = events + self._fail_commit = fail_commit + + async def __aenter__(self) -> Self: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + if exc_type is not None: + self._events.append("rollback") + return + if self._fail_commit: + self._events.append("commit failed") + msg = "commit failed" + raise RuntimeError(msg) + self._events.append("commit") + + +class _FakeSession: + events: list[str] + fail_commit: bool = False + + def __init__(self, _engine: object) -> None: + pass + + async def __aenter__(self) -> Self: + return self + + async def __aexit__(self, *_: object) -> None: + return None + + def begin(self) -> _FakeTransaction: + return _FakeTransaction(self.events, fail_commit=self.fail_commit) + + +def _make_app(events: list[str]) -> FastAPI: + app = FastAPI() + + @app.post("/write") + async def write( + _session: Annotated[object, Depends(expdb_session, scope="function")], + ) -> dict[str, str]: + events.append("handler") + return {"status": "ok"} + + @app.post("/fail") + async def fail( + _session: Annotated[object, Depends(expdb_session, scope="function")], + ) -> None: + msg = "boom" + raise ValueError(msg) + + return app + + +def _record_response_start( + app: FastAPI, + events: list[str], +) -> Callable[..., Any]: + async def spy( + scope: MutableMapping[str, Any], + receive: Callable[..., Any], + send: Callable[..., Any], + ) -> None: + async def send_(message: MutableMapping[str, Any]) -> None: + if message["type"] == "http.response.start": + events.append("response sent") + await send(message) + + await app(scope, receive, send_) + + return spy + + +@pytest.fixture +def events(monkeypatch: pytest.MonkeyPatch) -> list[str]: + recorded: list[str] = [] + _FakeSession.events = recorded + _FakeSession.fail_commit = False + monkeypatch.setattr(routers.dependencies, "AsyncSession", _FakeSession) + monkeypatch.setattr(routers.dependencies, "expdb_database", object) + return recorded + + +@pytest.fixture +async def client(events: list[str]) -> AsyncIterator[httpx.AsyncClient]: + app = _make_app(events) + transport = httpx.ASGITransport( + app=_record_response_start(app, events), + raise_app_exceptions=False, + ) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: + yield client + + +async def test_commit_happens_before_response( + client: httpx.AsyncClient, + events: list[str], +) -> None: + response = await client.post("/write") + assert response.status_code == httpx.codes.OK + assert events == ["handler", "commit", "response sent"] + + +async def test_failing_commit_is_not_reported_as_success( + client: httpx.AsyncClient, + events: list[str], +) -> None: + _FakeSession.fail_commit = True + response = await client.post("/write") + assert response.status_code == httpx.codes.INTERNAL_SERVER_ERROR + assert "commit failed" in events + + +async def test_error_rolls_back(client: httpx.AsyncClient, events: list[str]) -> None: + response = await client.post("/fail") + assert response.status_code == httpx.codes.INTERNAL_SERVER_ERROR + assert "rollback" in events + assert "commit" not in events + + +def test_session_dependencies_are_function_scoped() -> None: + """Every `Depends` on a session (or user) dependency must have `scope="function"`. + + Use e.g. `Depends(expdb_session, scope="function")`, otherwise the transaction + is committed after the response is sent. + """ + names = {dependency.__name__ for dependency in SESSION_DEPENDENCIES} + offenders = [] + for path in SOURCE_DIRECTORY.rglob("*.py"): + for node in ast.walk(ast.parse(path.read_text())): + if not ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "Depends" + and node.args + and isinstance(node.args[0], ast.Name) + and node.args[0].id in names + ): + continue + scope = next((kw.value for kw in node.keywords if kw.arg == "scope"), None) + if not (isinstance(scope, ast.Constant) and scope.value == "function"): + offenders.append(f"{path.relative_to(SOURCE_DIRECTORY)}:{node.lineno}") + assert not offenders