eval-nightly.yml

# Eval harness nightly — disabled-by-default.
#
# This workflow runs the golden QA dataset + worked-pattern cases against a
# real Azure OpenAI deployment. It is `workflow_dispatch`-only by default
# to prevent accidental API spend. To enable nightly runs:
#
#   1. Set the Azure OpenAI secrets in repo settings:
#        AZURE_OPENAI_ENDPOINT       e.g. https://my.openai.azure.com
#        AZURE_OPENAI_API_KEY        the Azure resource key
#        AZURE_OPENAI_DEPLOYMENT     deployment name, e.g. gpt-4o-mini
#        AZURE_OPENAI_API_VERSION    optional, defaults to 2024-10-21
#
#   2. Replace the `on:` block below with:
#
#        on:
#          schedule:
#            - cron: "0 6 * * *"   # daily 06:00 UTC
#          workflow_dispatch:
#
#   3. Confirm `eval-nightly.yml` is in EXEMPT_WORKFLOWS in
#      `.github/scripts/check_required_contexts.py` (it is, by default
#      — scheduled runs never gate PRs).
#
# When the Azure secrets are absent, eval/test_golden_patterns.py is
# skipped via pytestmark — the toy eval/test_golden_qa.py case still
# runs as a smoke check on the runner mechanics.
#
# See docs/EVAL_HARNESS.md for the full setup story.

name: Eval nightly

on:
  workflow_dispatch:
    inputs:
      python_version:
        description: "Python version to use"
        required: false
        default: "3.14"

permissions:
  contents: read

jobs:
  eval:
    name: Run golden QA dataset
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
      - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57  # v8.0.0
      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
        with:
          python-version: ${{ inputs.python_version || '3.14' }}
      - run: uv sync --frozen --extra dev --extra eval
      - name: Run pytest eval/
        env:
          AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
          AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }}
          AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}
        run: uv run pytest eval/ -v