diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index c4b23f0d..2d67952a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -33,7 +33,7 @@ assignees: '' * [ ] New or changed features have been added to (or updated in) the conceptual documentation. * [ ] Any new or updated dependencies have been added to the project configuration (e.g., `pyproject.toml`). * [ ] New functionality is fully tested. -* [ ] Any decrease in test coverage has been justified. +* [ ] Any new lines of code not covered by tests have been justified. * [ ] Either the `test-against-ofm-v3` job passes, or `test-against-ofm-feature-branch` passes. * [ ] New features have been used in a branch of the OpenFlexure Microscope, which is tested in `test-against-ofm-feature-branch`. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 48553e77..286d34fa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,36 +1,14 @@ name: Test on: - - pull_request + pull_request: + # All pushes to a PR should run this workflow + push: + branches: + - main -jobs: - base_coverage: - continue-on-error: true - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ github.base_ref }} - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: 3.12 - - - name: Install Dependencies - run: pip install -e . -r dev-requirements.txt - - - name: Test with pytest - run: | - pytest - mv coverage.lcov base-coverage.lcov - - - name: Upload code coverage for base branch - uses: actions/upload-artifact@v7 - with: - name: base-coverage.lcov - path: ./base-coverage.lcov - +jobs: pip_audit: runs-on: ubuntu-latest steps: @@ -77,7 +55,7 @@ jobs: run: flake8 src - name: Test with pytest - run: pytest + run: pytest --cov="src" --cov-report="xml" --cov-report="lcov" --cov-report="json" - name: Upload code coverage uses: actions/upload-artifact@v7 @@ -86,6 +64,7 @@ jobs: path: | ./coverage.lcov ./coverage.json + ./coverage.xml - name: Analyse with MyPy run: mypy @@ -129,22 +108,70 @@ jobs: if: success() || failure() run: bash ./docs/source/quickstart/test_quickstart_example.sh + base-coverage: + continue-on-error: true + runs-on: ubuntu-latest + outputs: + coverage: ${{ steps.get-coverage.outputs.coverage }} + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.base_ref }} + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.12 + + - name: Install Dependencies + run: pip install -e . -r dev-requirements.txt + + - name: Test with pytest + run: | + pytest --cov="src" --cov-report="xml" --cov-report="lcov" --cov-report="json" + mv coverage.lcov base-coverage.lcov + + - name: Parse coverage percentage + id: get-coverage + run: | + COVERAGE=$(jq -r '.totals.percent_covered_display' coverage.json) + echo "coverage=$COVERAGE" >> $GITHUB_OUTPUT + + - name: Upload code coverage for base branch + uses: actions/upload-artifact@v7 + with: + name: base-coverage.lcov + path: ./base-coverage.lcov + coverage: + needs: [test, base-coverage] + permissions: + contents: read + pull-requests: write + checks: write runs-on: ubuntu-latest - needs: [base_coverage, test] steps: + - uses: actions/checkout@v4 # Needed to generate the PR diff + - name: Download code coverage report uses: actions/download-artifact@v8 with: name: coverage-3.12 + - name: Report diff coverage + uses: Affanmir/diff-cover-action@v2 + with: + compare-branch: origin/main + coverage-files: coverage.xml + fail-under: '100' + - name: Parse coverage percentage id: get-coverage run: | COVERAGE=$(jq -r '.totals.percent_covered_display' coverage.json) echo "coverage=$COVERAGE" >> $GITHUB_OUTPUT - - name: Update Coverage Badge + - name: Update coverage badge # Only run this on the main branch so PRs don't overwrite the main badge if: github.ref == 'refs/heads/main' uses: Schneegans/dynamic-badges-action@v1.7.0 @@ -158,34 +185,19 @@ jobs: maxColorRange: 90 minColorRange: 50 - - name: Download code coverage report for base branch - id: download-base-coverage - continue-on-error: true - uses: actions/download-artifact@v8 - with: - name: base-coverage.lcov - - - name: Generate Code Coverage report - # Note, due to continue on error (to make job pass) we need to check the - # Status of the step directly not just use success() or failure() - if: steps.download-base-coverage.outcome == 'success' - id: code-coverage - uses: barecheck/code-coverage-action@v1 - with: - barecheck-github-app-token: ${{ secrets.BARECHECK_GITHUB_APP_TOKEN }} - lcov-file: "./coverage.lcov" - base-lcov-file: "./base-coverage.lcov" - minimum-ratio: 0 - send-summary-comment: true - show-annotations: "warning" - - - name: Generate Code Coverage report if base job fails - if: steps.download-base-coverage.outcome == 'failure' - id: code-coverage-without-base - uses: barecheck/code-coverage-action@v1 + - name: Compute the change in coverage + id: coverage-delta + env: + BASE_COV: ${{ needs.base-coverage.outputs.coverage }} + BRANCH_COV: ${{ steps.get-coverage.outputs.coverage}} + run: | + change=$(echo "$BRANCH_COV - $BASE_COV" | bc) + echo "Coverage changed by $change %" + echo "delta=$change" >> $GITHUB_OUTPUT + + - name: Check coverage tolerance + if: ${{ steps.coverage-delta.outputs.delta < -0.2 }} + uses: actions/github-script@v9 with: - barecheck-github-app-token: ${{ secrets.BARECHECK_GITHUB_APP_TOKEN }} - lcov-file: "./coverage.lcov" - minimum-ratio: 0 - send-summary-comment: true - show-annotations: "warning" + script: | + core.setFailed('Coverage has decreased too much.') diff --git a/docs/source/testing.md b/docs/source/testing.md index 54e664bc..618183bc 100644 --- a/docs/source/testing.md +++ b/docs/source/testing.md @@ -11,7 +11,7 @@ Here is what the CI pipeline tests: * **Matrix Testing:** We run the core test suite (`pytest`) and static type checks (`mypy`) across Python versions 3.11, 3.12, and 3.13. * **Code Quality & Security:** The pipeline runs `ruff` (formatting and linting), `flake8` (docstrings), `codespell` (spelling), and a `pip-audit` to check for dependency vulnerabilities. -* **Coverage:** We track code coverage. A report is generated on your PR to show if your changes increased or decreased overall test coverage. +* **Coverage:** We track code coverage. A report is generated on your PR to flag any uncovered lines in the diff. The CI job will fail if there are changes being added which aren't fully covered by tests, which requires either a justification or more testing before it's merged. * **Dependency Checks:** We run the tests twice: once with pinned dependencies (`dev-requirements.txt`) for reproducibility, and once with unpinned dependencies (`.[dev]`) to catch upstream breakages early. * **Downstream Integration:** The pipeline installs your version of `labthings-fastapi` alongside the `v3` branch of the OpenFlexure Microscope Server. It then runs the entire OFM test suite (unit, integration, and lifecycle) to guarantee backwards compatibility. *(Note: You can target a specific OFM branch by adding `OFM-Feature-Branch: branch-name` to your PR description).* @@ -56,7 +56,7 @@ mypy ### 4. Unit Tests & Coverage -We use `pytest` for our core test suite. Execute the unit tests and generate a coverage report using: +We use `pytest` for our core test suite. Coverage reports may be generated by adding the relevant command-line arguments, or by using your IDE's coverage feature. ```bash pytest diff --git a/pyproject.toml b/pyproject.toml index 906911a1..6bb58b16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,11 +68,6 @@ artifacts = ["src/*.json", "src/*.jinja"] artifacts = ["src/*.json", "src/*.jinja"] [tool.pytest.ini_options] -addopts = [ - "--cov=src", - "--cov-report=lcov", - "--cov-report=json", -] markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", ]