From 368f615b912cd2b7ac364318de56a74eabd33842 Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Mon, 9 Nov 2026 16:13:53 +0000 Subject: [PATCH 01/11] Add a diff coverage action This uses the popular `5monkeys/coberatura-action` to show coverage in PRs. It doesn't rely on external services. I hope it will be better than our current solution, as I think it is able to check coverage of the diff rather than just global coverage. I added an xml coverage report to make this work. Rather than have an ever-growing list of reports in `pyproject.toml` I have removed those options from the pytest section. Instead, coverage reports should now be specified when they're needed. We only really need the xml report here, but I've left the others in just in case they're useful in the future. I've switched to using the VSCode tests panel as it puts coverage bars in my editor. Other developers are welcome to switch on whatever coverage reports they prefer. Turning off coverage when I run it at the command line takes tests from ~30s to ~20s on my system. --- .github/workflows/test.yml | 91 +++++--------------------------------- pyproject.toml | 5 --- 2 files changed, 11 insertions(+), 85 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 48553e77..873b9639 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,33 +4,6 @@ on: - pull_request jobs: - base_coverage: - continue-on-error: true - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ github.base_ref }} - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: 3.12 - - - name: Install Dependencies - run: pip install -e . -r dev-requirements.txt - - - name: Test with pytest - run: | - pytest - mv coverage.lcov base-coverage.lcov - - - name: Upload code coverage for base branch - uses: actions/upload-artifact@v7 - with: - name: base-coverage.lcov - path: ./base-coverage.lcov - pip_audit: runs-on: ubuntu-latest steps: @@ -77,7 +50,7 @@ jobs: run: flake8 src - name: Test with pytest - run: pytest + run: pytest --cov="src" --cov-report="xml" --cov-report="lcov" --cov-report="json" - name: Upload code coverage uses: actions/upload-artifact@v7 @@ -86,6 +59,7 @@ jobs: path: | ./coverage.lcov ./coverage.json + ./coverage.xml - name: Analyse with MyPy run: mypy @@ -130,62 +104,19 @@ jobs: run: bash ./docs/source/quickstart/test_quickstart_example.sh coverage: + needs: [test] runs-on: ubuntu-latest - needs: [base_coverage, test] + permissions: + contents: read + pull-requests: write steps: - name: Download code coverage report uses: actions/download-artifact@v8 with: name: coverage-3.12 - - - name: Parse coverage percentage - id: get-coverage - run: | - COVERAGE=$(jq -r '.totals.percent_covered_display' coverage.json) - echo "coverage=$COVERAGE" >> $GITHUB_OUTPUT - - - name: Update Coverage Badge - # Only run this on the main branch so PRs don't overwrite the main badge - if: github.ref == 'refs/heads/main' - uses: Schneegans/dynamic-badges-action@v1.7.0 - with: - auth: ${{ secrets.GIST_SECRET }} - gistID: 936bcec8e1815a49d1e7f947924ffa3f - filename: labthings-fastapi-coverage.json - label: coverage - message: ${{ steps.get-coverage.outputs.coverage }}% - valColorRange: ${{ steps.get-coverage.outputs.coverage }} - maxColorRange: 90 - minColorRange: 50 - - - name: Download code coverage report for base branch - id: download-base-coverage - continue-on-error: true - uses: actions/download-artifact@v8 - with: - name: base-coverage.lcov - - - name: Generate Code Coverage report - # Note, due to continue on error (to make job pass) we need to check the - # Status of the step directly not just use success() or failure() - if: steps.download-base-coverage.outcome == 'success' - id: code-coverage - uses: barecheck/code-coverage-action@v1 - with: - barecheck-github-app-token: ${{ secrets.BARECHECK_GITHUB_APP_TOKEN }} - lcov-file: "./coverage.lcov" - base-lcov-file: "./base-coverage.lcov" - minimum-ratio: 0 - send-summary-comment: true - show-annotations: "warning" - - - name: Generate Code Coverage report if base job fails - if: steps.download-base-coverage.outcome == 'failure' - id: code-coverage-without-base - uses: barecheck/code-coverage-action@v1 + - uses: 5monkeys/cobertura-action@master with: - barecheck-github-app-token: ${{ secrets.BARECHECK_GITHUB_APP_TOKEN }} - lcov-file: "./coverage.lcov" - minimum-ratio: 0 - send-summary-comment: true - show-annotations: "warning" + path: coverage.xml + minimum_coverage: 100 + fail_below_threshold: true + only_changed_files: true diff --git a/pyproject.toml b/pyproject.toml index 906911a1..6bb58b16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,11 +68,6 @@ artifacts = ["src/*.json", "src/*.jinja"] artifacts = ["src/*.json", "src/*.jinja"] [tool.pytest.ini_options] -addopts = [ - "--cov=src", - "--cov-report=lcov", - "--cov-report=json", -] markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", ] From e16b3fc7f6003480527e59cf3634947c9220fb76 Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Mon, 9 Nov 2026 16:31:16 +0000 Subject: [PATCH 02/11] pointless python change. --- src/labthings_fastapi/thing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/labthings_fastapi/thing.py b/src/labthings_fastapi/thing.py index f2de6ddd..c487c91b 100644 --- a/src/labthings_fastapi/thing.py +++ b/src/labthings_fastapi/thing.py @@ -125,6 +125,7 @@ def __init__(self, thing_server_interface: ThingServerInterface) -> None: # they have been loaded from the settings file (which would overwrite # the file with default values). self._disable_saving_settings: bool = True + self._pointless: bool = True def __init_subclass__(cls, **kwargs: Any) -> None: r"""Validate the class settings. From b022669387e0414f2a6ecc8108eb3bdea54668f1 Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Mon, 9 Nov 2026 16:35:40 +0000 Subject: [PATCH 03/11] Add permission for checks --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 873b9639..b91b6360 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -109,6 +109,7 @@ jobs: permissions: contents: read pull-requests: write + checks: write steps: - name: Download code coverage report uses: actions/download-artifact@v8 From 8d079bd235cfa1c6ea6639f9811113725c1410da Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Mon, 9 Nov 2026 16:40:44 +0000 Subject: [PATCH 04/11] Switch to diff-cover-action I hope this does a line-by-line diff. --- .github/workflows/test.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b91b6360..589d4018 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -115,9 +115,8 @@ jobs: uses: actions/download-artifact@v8 with: name: coverage-3.12 - - uses: 5monkeys/cobertura-action@master + - name: Report Diff Coverage + uses: Affanmir/diff-cover-action@v2 with: - path: coverage.xml - minimum_coverage: 100 - fail_below_threshold: true - only_changed_files: true + coverage-files: coverage.xml + fail-under: '100' From c8ddd18fabc595e1e0610da59274ad948bcb1213 Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Mon, 9 Nov 2026 16:44:25 +0000 Subject: [PATCH 05/11] More pointless python changes --- src/labthings_fastapi/thing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/labthings_fastapi/thing.py b/src/labthings_fastapi/thing.py index c487c91b..cbd7a2ff 100644 --- a/src/labthings_fastapi/thing.py +++ b/src/labthings_fastapi/thing.py @@ -126,6 +126,8 @@ def __init__(self, thing_server_interface: ThingServerInterface) -> None: # the file with default values). self._disable_saving_settings: bool = True self._pointless: bool = True + if False: + self._pointless = False def __init_subclass__(cls, **kwargs: Any) -> None: r"""Validate the class settings. From 22ef6f8fb25e3e807f454358de67725156b9c6ca Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Mon, 9 Nov 2026 16:49:42 +0000 Subject: [PATCH 06/11] Check out the repo I need to check out the repo in order to generate the git diff, even though it's not needed to run the tests. --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 589d4018..c456a9be 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -111,6 +111,7 @@ jobs: pull-requests: write checks: write steps: + - uses: actions/checkout@v4 # Needed to generate the PR diff - name: Download code coverage report uses: actions/download-artifact@v8 with: @@ -118,5 +119,6 @@ jobs: - name: Report Diff Coverage uses: Affanmir/diff-cover-action@v2 with: + compare-branch: origin/main coverage-files: coverage.xml fail-under: '100' From 72cd7993a5325b318a8b835deeb076f88a613913 Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Mon, 9 Nov 2026 16:56:04 +0000 Subject: [PATCH 07/11] More pointless python changes --- src/labthings_fastapi/thing.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/labthings_fastapi/thing.py b/src/labthings_fastapi/thing.py index cbd7a2ff..1d89a2f2 100644 --- a/src/labthings_fastapi/thing.py +++ b/src/labthings_fastapi/thing.py @@ -129,6 +129,10 @@ def __init__(self, thing_server_interface: ThingServerInterface) -> None: if False: self._pointless = False + def extra_method(self) -> None: + """Run a pointless extra method.""" + pass + def __init_subclass__(cls, **kwargs: Any) -> None: r"""Validate the class settings. From 35151db6224effaad46fdafbba76caeb3298a451 Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Mon, 9 Nov 2026 16:59:55 +0000 Subject: [PATCH 08/11] Revert changes to python code. --- src/labthings_fastapi/thing.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/labthings_fastapi/thing.py b/src/labthings_fastapi/thing.py index 1d89a2f2..f2de6ddd 100644 --- a/src/labthings_fastapi/thing.py +++ b/src/labthings_fastapi/thing.py @@ -125,13 +125,6 @@ def __init__(self, thing_server_interface: ThingServerInterface) -> None: # they have been loaded from the settings file (which would overwrite # the file with default values). self._disable_saving_settings: bool = True - self._pointless: bool = True - if False: - self._pointless = False - - def extra_method(self) -> None: - """Run a pointless extra method.""" - pass def __init_subclass__(cls, **kwargs: Any) -> None: r"""Validate the class settings. From 3ab0eb278e520cdbd44e18a13afac0d4feb38a58 Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Mon, 9 Nov 2026 22:36:56 +0000 Subject: [PATCH 09/11] Remove github code quality and restore badge The coverage badge is reinstated. I removed the code quality upload, as it's reserved for premium tiers. --- .github/workflows/test.yml | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c456a9be..c38d4949 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,6 +3,11 @@ name: Test on: - pull_request +permissions: + contents: read + pull-requests: write + checks: write + jobs: pip_audit: runs-on: ubuntu-latest @@ -106,19 +111,37 @@ jobs: coverage: needs: [test] runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - checks: write steps: - uses: actions/checkout@v4 # Needed to generate the PR diff + - name: Download code coverage report uses: actions/download-artifact@v8 with: name: coverage-3.12 + - name: Report Diff Coverage uses: Affanmir/diff-cover-action@v2 with: compare-branch: origin/main coverage-files: coverage.xml fail-under: '100' + + - name: Parse coverage percentage + id: get-coverage + run: | + COVERAGE=$(jq -r '.totals.percent_covered_display' coverage.json) + echo "coverage=$COVERAGE" >> $GITHUB_OUTPUT + + - name: Update Coverage Badge + # Only run this on the main branch so PRs don't overwrite the main badge + if: github.ref == 'refs/heads/main' + uses: Schneegans/dynamic-badges-action@v1.7.0 + with: + auth: ${{ secrets.GIST_SECRET }} + gistID: 936bcec8e1815a49d1e7f947924ffa3f + filename: labthings-fastapi-coverage.json + label: coverage + message: ${{ steps.get-coverage.outputs.coverage }}% + valColorRange: ${{ steps.get-coverage.outputs.coverage }} + maxColorRange: 90 + minColorRange: 50 From c088aca9f87d938bfb843f54cdfc0d628e7de9f3 Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Wed, 18 Nov 2026 11:45:03 +0000 Subject: [PATCH 10/11] Update docs and PR template with new test coverage strategy. --- .github/pull_request_template.md | 2 +- docs/source/testing.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index c4b23f0d..2d67952a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -33,7 +33,7 @@ assignees: '' * [ ] New or changed features have been added to (or updated in) the conceptual documentation. * [ ] Any new or updated dependencies have been added to the project configuration (e.g., `pyproject.toml`). * [ ] New functionality is fully tested. -* [ ] Any decrease in test coverage has been justified. +* [ ] Any new lines of code not covered by tests have been justified. * [ ] Either the `test-against-ofm-v3` job passes, or `test-against-ofm-feature-branch` passes. * [ ] New features have been used in a branch of the OpenFlexure Microscope, which is tested in `test-against-ofm-feature-branch`. diff --git a/docs/source/testing.md b/docs/source/testing.md index 54e664bc..618183bc 100644 --- a/docs/source/testing.md +++ b/docs/source/testing.md @@ -11,7 +11,7 @@ Here is what the CI pipeline tests: * **Matrix Testing:** We run the core test suite (`pytest`) and static type checks (`mypy`) across Python versions 3.11, 3.12, and 3.13. * **Code Quality & Security:** The pipeline runs `ruff` (formatting and linting), `flake8` (docstrings), `codespell` (spelling), and a `pip-audit` to check for dependency vulnerabilities. -* **Coverage:** We track code coverage. A report is generated on your PR to show if your changes increased or decreased overall test coverage. +* **Coverage:** We track code coverage. A report is generated on your PR to flag any uncovered lines in the diff. The CI job will fail if there are changes being added which aren't fully covered by tests, which requires either a justification or more testing before it's merged. * **Dependency Checks:** We run the tests twice: once with pinned dependencies (`dev-requirements.txt`) for reproducibility, and once with unpinned dependencies (`.[dev]`) to catch upstream breakages early. * **Downstream Integration:** The pipeline installs your version of `labthings-fastapi` alongside the `v3` branch of the OpenFlexure Microscope Server. It then runs the entire OFM test suite (unit, integration, and lifecycle) to guarantee backwards compatibility. *(Note: You can target a specific OFM branch by adding `OFM-Feature-Branch: branch-name` to your PR description).* @@ -56,7 +56,7 @@ mypy ### 4. Unit Tests & Coverage -We use `pytest` for our core test suite. Execute the unit tests and generate a coverage report using: +We use `pytest` for our core test suite. Coverage reports may be generated by adding the relevant command-line arguments, or by using your IDE's coverage feature. ```bash pytest From cbff0ff0e9a1e3ed2041030f7ad509345e37f07c Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Wed, 18 Nov 2026 13:41:16 +0000 Subject: [PATCH 11/11] Restore diff coverage and fix badge I've restored the base coverage job, and added steps to the coverage job that compare total coverage of this branch to `main`. The job will now fail if coverage has dropped by more than 0.2%. I no longer feel the need to set the threshold at 0. I've also set the test job to run on the `main` branch, so that we can get the badge updating again. --- .github/workflows/test.yml | 72 +++++++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c38d4949..286d34fa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,12 +1,12 @@ name: Test on: - - pull_request + pull_request: + # All pushes to a PR should run this workflow + push: + branches: + - main -permissions: - contents: read - pull-requests: write - checks: write jobs: pip_audit: @@ -108,8 +108,47 @@ jobs: if: success() || failure() run: bash ./docs/source/quickstart/test_quickstart_example.sh + base-coverage: + continue-on-error: true + runs-on: ubuntu-latest + outputs: + coverage: ${{ steps.get-coverage.outputs.coverage }} + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.base_ref }} + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.12 + + - name: Install Dependencies + run: pip install -e . -r dev-requirements.txt + + - name: Test with pytest + run: | + pytest --cov="src" --cov-report="xml" --cov-report="lcov" --cov-report="json" + mv coverage.lcov base-coverage.lcov + + - name: Parse coverage percentage + id: get-coverage + run: | + COVERAGE=$(jq -r '.totals.percent_covered_display' coverage.json) + echo "coverage=$COVERAGE" >> $GITHUB_OUTPUT + + - name: Upload code coverage for base branch + uses: actions/upload-artifact@v7 + with: + name: base-coverage.lcov + path: ./base-coverage.lcov + coverage: - needs: [test] + needs: [test, base-coverage] + permissions: + contents: read + pull-requests: write + checks: write runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 # Needed to generate the PR diff @@ -119,7 +158,7 @@ jobs: with: name: coverage-3.12 - - name: Report Diff Coverage + - name: Report diff coverage uses: Affanmir/diff-cover-action@v2 with: compare-branch: origin/main @@ -132,7 +171,7 @@ jobs: COVERAGE=$(jq -r '.totals.percent_covered_display' coverage.json) echo "coverage=$COVERAGE" >> $GITHUB_OUTPUT - - name: Update Coverage Badge + - name: Update coverage badge # Only run this on the main branch so PRs don't overwrite the main badge if: github.ref == 'refs/heads/main' uses: Schneegans/dynamic-badges-action@v1.7.0 @@ -145,3 +184,20 @@ jobs: valColorRange: ${{ steps.get-coverage.outputs.coverage }} maxColorRange: 90 minColorRange: 50 + + - name: Compute the change in coverage + id: coverage-delta + env: + BASE_COV: ${{ needs.base-coverage.outputs.coverage }} + BRANCH_COV: ${{ steps.get-coverage.outputs.coverage}} + run: | + change=$(echo "$BRANCH_COV - $BASE_COV" | bc) + echo "Coverage changed by $change %" + echo "delta=$change" >> $GITHUB_OUTPUT + + - name: Check coverage tolerance + if: ${{ steps.coverage-delta.outputs.delta < -0.2 }} + uses: actions/github-script@v9 + with: + script: | + core.setFailed('Coverage has decreased too much.')