diff --git a/.github/workflows/_build-docker-action.yml b/.github/workflows/_build-docker-action.yml new file mode 100644 index 00000000..dee0e88e --- /dev/null +++ b/.github/workflows/_build-docker-action.yml @@ -0,0 +1,50 @@ +name: 'Build Docker Image' +on: + workflow_call: +env: + REGISTRY: docker.io + IMAGE_NAME: aberger4/jabs +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: aberger4 + password: ${{ secrets.DOCKER_SECRET }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable={{is_default_branch}} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/_format-lint-action.yml b/.github/workflows/_format-lint-action.yml new file mode 100644 index 00000000..0453eb2f --- /dev/null +++ b/.github/workflows/_format-lint-action.yml @@ -0,0 +1,33 @@ +name: 'Lint Code Definition' +on: + workflow_call: + inputs: + python-version: + description: 'Python version to set up' + required: false + default: '3.13' + type: string +jobs: + format-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + version: "latest" + + - name: Install dependencies with uv + run: uv sync --only-group lint + + - name: Run Ruff Linter + run: uv run --only-group lint ruff check src/ + + - name: Run Ruff Formatter + run: uv run --only-group lint ruff format --check src/ diff --git a/.github/workflows/_run-tests-action.yml b/.github/workflows/_run-tests-action.yml new file mode 100644 index 00000000..47a36948 --- /dev/null +++ b/.github/workflows/_run-tests-action.yml @@ -0,0 +1,35 @@ +name: 'Python Tests Definition' +on: + workflow_call: + inputs: + python-version: + description: Python version to set up' + required: false + default: '3.13' + type: string + runner-os: + description: 'Runner OS' + required: false + default: 'ubuntu-latest' + type: string +jobs: + run-tests: + runs-on: ${{ inputs.runner-os }} + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + version: "latest" + + - name: Install dependencies with uv + run: uv sync + + - name: Test with pytest + run: uv run pytest --ignore=tests/test_search_bar_widget.py tests diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml new file mode 100644 index 00000000..749fcdd4 --- /dev/null +++ b/.github/workflows/pull-request.yml @@ -0,0 +1,15 @@ +name: Pull Request Checks + +on: + pull_request: + branches: [ main, master ] + +jobs: + format-lint: + name: "Format and Lint" + uses: ./.github/workflows/_format-lint-action.yml + + test: + name: "Run Tests" + needs: format-lint + uses: ./.github/workflows/_run-tests-action.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..f7820a92 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,134 @@ +name: Build and Release + +on: + push: + branches: [ main, master ] + paths: + - 'pyproject.toml' + +permissions: + contents: write + id-token: write + packages: write + +jobs: + format-lint: + name: "Format and Lint" + uses: ./.github/workflows/_format-lint-action.yml + + test: + name: "Run Tests" + needs: format-lint + uses: ./.github/workflows/_run-tests-action.yml + + check-version-changed: + name: "Check if version changed" + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + version-changed: ${{ steps.check.outputs.changed }} + is-prerelease: ${{ steps.version.outputs.is-prerelease }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Get current version + id: version + run: | + VERSION=$(grep '^version = ' pyproject.toml | sed 's/version = "\(.*\)"/\1/') + echo "version=$VERSION" >> $GITHUB_OUTPUT + + # Check if version contains letters (indicating pre-release) + if echo "$VERSION" | grep -q '[a-zA-Z]'; then + echo "is-prerelease=true" >> $GITHUB_OUTPUT + echo "Detected pre-release version: $VERSION" + else + echo "is-prerelease=false" >> $GITHUB_OUTPUT + echo "Detected stable release version: $VERSION" + fi + + - name: Check if version changed + id: check + run: | + if git diff HEAD~1 HEAD --name-only | grep -q pyproject.toml; then + if git diff HEAD~1 HEAD pyproject.toml | grep -q '^+version = '; then + echo "changed=true" >> $GITHUB_OUTPUT + echo "Version changed in pyproject.toml" + else + echo "changed=false" >> $GITHUB_OUTPUT + echo "pyproject.toml changed but version did not change" + fi + else + echo "changed=false" >> $GITHUB_OUTPUT + echo "pyproject.toml was not changed" + fi + + build-package: + name: "Build Python Package" + runs-on: ubuntu-latest + needs: [build, check-version-changed] + if: needs.check-version-changed.outputs.version-changed == 'true' + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Set up Python + run: uv python install 3.10 + + - name: Build package + run: uv build + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish-pypi: + name: "Publish to PyPI" + runs-on: ubuntu-latest + needs: [build-package, check-version-changed] + if: needs.check-version-changed.outputs.version-changed == 'true' + environment: release + steps: + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Publish to PyPI + run: uv publish --token ${{ secrets.PYPI_API_TOKEN }} + working-directory: . + + create-release: + name: "Create GitHub Release" + runs-on: ubuntu-latest + needs: [publish-pypi, check-version-changed] + if: needs.check-version-changed.outputs.version-changed == 'true' + steps: + - uses: actions/checkout@v4 + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + tag_name: v${{ needs.check-version-changed.outputs.version }} + name: Release v${{ needs.check-version-changed.outputs.version }} + draft: false + prerelease: ${{ needs.check-version-changed.outputs.is-prerelease == 'true' }} + generate_release_notes: true + files: dist/* diff --git a/README.md b/README.md index f6dd6094..8bfd81f0 100644 --- a/README.md +++ b/README.md @@ -226,4 +226,71 @@ training data from a project folder, transfer it to our HPC cluster, and then tr Pickle files are tiny and efficient, but are not transferable across computers. We use these for large-scale predictions in pipelines (for example, using exported training data to train a classifier saved as a .pickle file, -which can then be used to classify many videos as part of a pipeline). \ No newline at end of file +which can then be used to classify many videos as part of a pipeline). + +## CI/CD and Release Management + +JABS uses GitHub Actions for continuous integration and automated releases to PyPI. +The CI/CD pipeline is defined in `.github/workflows/` and automatically manages package building, testing, and publishing. + +### Pull Request Checks + +Pull requests to the `main` branch trigger automated checks to ensure code quality and functionality: + +1. **Code Formatting and Linting**: Ensures code adheres to style guidelines +2. **Test Execution**: Runs the full test suite to verify functionality + +### Automated Release Process + +The release process is triggered automatically when the version number in `pyproject.toml` is changed on the `main` branch: + +1. **Version Detection**: The workflow monitors changes to `pyproject.toml` and extracts the version number +2. **Pre-release Detection**: Versions containing letters (e.g., `1.0.0a1`, `2.1.0rc1`) are automatically marked as pre-releases +3. **Build Pipeline**: If version changed, the system runs: + - Code formatting and linting checks + - Test execution + - Package building with `uv build` +4. **PyPI Publishing**: Successfully built packages are automatically published to PyPI +5. **GitHub Release**: A corresponding GitHub release is created with build artifacts + +### Release Workflow Files + +- **`.github/workflows/release.yml`**: Main release workflow that orchestrates the entire process +- **`.github/workflows/_format-lint-action.yml`**: Reusable workflow for code quality checks +- **`.github/workflows/_run-tests-action.yml`**: Reusable workflow for test execution +- **`.github/workflows/pull-request.yml`**: CI checks for pull requests + +### Creating a New Release + +To create a new release: + +1. Update the version number in `pyproject.toml`: + ```toml + version = "X.Y.Z" # for stable releases + version = "X.Y.Za1" # for alpha pre-releases + version = "X.Y.Zrc1" # for release candidates + ``` + +2. Re-lock the uv lock file: + ```bash + uv lock + ``` + +3. Commit and push the change: + ```bash + git add pyproject.toml uv.lock + git commit -m "Bump version to X.Y.Z" + ``` + +4. Merge your changes into the `main` branch via a pull request. + +3. The CI/CD pipeline will automatically: + - Detect the version change + - Run all quality checks and tests + - Build and publish the package to PyPI + - Create a GitHub release with generated release notes + +### Environment Requirements + +The release workflow requires: +- **PyPI API Token**: Stored as `PYPI_API_TOKEN` in GitHub repository secrets