regression-test.yml

name: srcML regression test

on:
  workflow_call:
    inputs:
      srcml-installer-artifact-name:
        description: "Actions artifact containing the srcml installer package(s) (e.g. srcml_*.deb)"
        required: true
        type: string
      srcml-installer-artifact-run-id:
        description: "Run ID to download the artifact from. Leave empty for the current run."
        required: false
        type: string
        default: ""
      language:
        description: "Language key — must match a config/<language>.toml file (e.g. c, cpp, csharp, java, python)"
        required: true
        type: string
      project:
        description: "Project name — must match a [[projects]].name entry in config/<language>.toml"
        required: true
        type: string
      baseline-release-tag:
        description: "Override the per-project current_baseline_release from config"
        required: false
        type: string
        default: ""
      fail-on-regression:
        description: "Fail the workflow if any regressions are detected"
        required: false
        type: boolean
        default: true
      artifact-name:
        description: "Name for the uploaded results artifact"
        required: false
        type: string
        default: "srcml-regression-results"
    outputs:
      regressions-found:
        description: "'true' if regressions detected, else 'false'"
        value: ${{ jobs.test.outputs.regressed }}
      summary:
        description: "One-line machine-readable summary"
        value: ${{ jobs.test.outputs.summary }}

  workflow_dispatch:
    inputs:
      srcml-installer-artifact-name:
        description: "Actions artifact containing the srcml installer package(s) (e.g. srcml_*.deb)"
        required: true
        default: "srcml-installer"
      srcml-installer-artifact-run-id:
        description: "Run ID of the workflow that uploaded the artifact"
        required: false
        default: ""
      language:
        description: "Language key (c, cpp, csharp, java, python)"
        required: true
        default: "c"
      project:
        description: "Project name from config/<language>.toml"
        required: true
        default: "linux"
      baseline-release-tag:
        description: "Override the per-project current_baseline_release"
        required: false
        default: ""
      fail-on-regression:
        description: "Fail on regression"
        required: false
        type: boolean
        default: true

permissions:
  contents: read
  actions: read

concurrency:
  group: regression-${{ github.workflow }}-${{ github.ref }}-${{ inputs.language }}-${{ inputs.project }}
  cancel-in-progress: true

env:
  # hardcoded so cross-repo `gh release download` works regardless of caller.
  # update if this repo is transferred to a different org.
  BASELINE_REPO: ${{ github.repository_owner }}/srcMLLargeSystems

jobs:
  test:
    runs-on: ubuntu-24.04-arm
    timeout-minutes: 180
    outputs:
      regressed: ${{ steps.decide.outputs.regressed }}
      summary: ${{ steps.decide.outputs.summary }}
    steps:
      - name: Checkout this repo (for config + scripts)
        uses: actions/checkout@v4
        with:
          repository: ${{ env.BASELINE_REPO }}
          path: srcml-large-system-tests

      - name: Install host deps
        run: |
          sudo apt-get update
          sudo apt-get install -y zstd jq python3

      - name: Install mikefarah yq
        run: |
          ARCH="$(dpkg --print-architecture)"
          sudo curl -fsSL -o /usr/local/bin/yq \
            "https://github.com/mikefarah/yq/releases/latest/download/yq_linux_${ARCH}"
          sudo chmod +x /usr/local/bin/yq
          yq --version

      - name: Resolve baseline tag from config
        id: cfg
        env:
          LANGUAGE: ${{ inputs.language }}
          PROJECT: ${{ inputs.project }}
          TAG_OVERRIDE: ${{ inputs.baseline-release-tag }}
        run: |
          set -euo pipefail
          cd srcml-large-system-tests
          CFG="config/${LANGUAGE}.toml"
          if [ ! -f "$CFG" ]; then
            echo "::error::config file not found: $CFG"
            exit 1
          fi
          if [ -n "$TAG_OVERRIDE" ]; then
            TAG="$TAG_OVERRIDE"
          else
            TAG="$(PROJECT="$PROJECT" yq -p toml -r '.projects[] | select(.name == strenv(PROJECT)) | .current_baseline_release' "$CFG")"
          fi
          if [ -z "$TAG" ] || [ "$TAG" = "null" ]; then
            echo "::error::no current_baseline_release configured for ${LANGUAGE}/${PROJECT}. Run baseline-${LANGUAGE}.yml first (or pass baseline-release-tag)."
            exit 1
          fi
          echo "release_tag=$TAG"          >> "$GITHUB_OUTPUT"
          echo "zst_name=${TAG}.xml.zst"   >> "$GITHUB_OUTPUT"
          echo "manifest_name=${TAG}.manifest.json" >> "$GITHUB_OUTPUT"

      - name: Download srcml installer artifact
        uses: actions/download-artifact@v4
        with:
          name: ${{ inputs.srcml-installer-artifact-name }}
          path: srcml-installer
          run-id: ${{ inputs.srcml-installer-artifact-run-id != '' && inputs.srcml-installer-artifact-run-id || github.run_id }}
          github-token: ${{ github.token }}

      - name: Install srcml from .deb
        run: |
          set -euxo pipefail
          shopt -s nullglob
          debs=( ./srcml-installer/srcml_*.deb ./srcml-installer/srcml-dev_*.deb )
          if [ "${#debs[@]}" -eq 0 ]; then
            echo "::error::artifact does not contain any srcml*.deb files"
            ls -la srcml-installer/ || true
            exit 1
          fi
          echo "Installing: ${debs[*]}"
          sudo apt-get update
          sudo apt-get install -y "${debs[@]}"
          srcml --version
          command -v srcml

      - name: Download baseline release asset
        env:
          GH_TOKEN: ${{ github.token }}
          TAG: ${{ steps.cfg.outputs.release_tag }}
          ZST_NAME: ${{ steps.cfg.outputs.zst_name }}
          MANIFEST_NAME: ${{ steps.cfg.outputs.manifest_name }}
        run: |
          set -euxo pipefail
          gh release download "$TAG" \
            --repo "$BASELINE_REPO" \
            --pattern "$ZST_NAME" \
            --pattern "$MANIFEST_NAME"

      - name: Verify sha256
        env:
          ZST_NAME: ${{ steps.cfg.outputs.zst_name }}
          MANIFEST_NAME: ${{ steps.cfg.outputs.manifest_name }}
        run: |
          set -euo pipefail
          EXPECTED="$(jq -r '.sha256' "$MANIFEST_NAME")"
          ACTUAL="$(sha256sum "$ZST_NAME" | awk '{print $1}')"
          echo "expected: $EXPECTED"
          echo "actual:   $ACTUAL"
          if [ "$EXPECTED" != "$ACTUAL" ]; then
            echo "::error::sha256 mismatch for baseline asset"
            exit 1
          fi

      - name: Decompress baseline
        env:
          ZST_NAME: ${{ steps.cfg.outputs.zst_name }}
        run: |
          zstd -d "$ZST_NAME" -o baseline.xml
          ls -lh baseline.xml

      - name: Run parser-test
        env:
          SRCML_BIN: srcml
          BASELINE: baseline.xml
          OUT_DIR: results
        run: ./srcml-large-system-tests/scripts/run_parser_test.sh

      - name: Summarize
        env:
          LANGUAGE: ${{ inputs.language }}
          MANIFEST_NAME: ${{ steps.cfg.outputs.manifest_name }}
        run: |
          python3 srcml-large-system-tests/scripts/summarize_parser_test.py \
            results/parser-test.stdout \
            --out results/grouped-failures.md \
            --summary results/summary.txt \
            --archive baseline.xml \
            --manifest "$MANIFEST_NAME" \
            --migrations "srcml-large-system-tests/config/migrations.json" \
            --migrations "srcml-large-system-tests/config/${LANGUAGE}.migrations.json"
          cp "$MANIFEST_NAME" results/

      - name: Decide pass/fail
        id: decide
        run: |
          set -euo pipefail
          EXIT_CODE="$(cat results/exit-code.txt)"
          SUMMARY_LINE="$(cat results/summary.txt | head -n1)"
          FAILED="$(grep -oP 'failed=\K[0-9]+' results/summary.txt || echo 0)"
          REGRESSED="false"
          if [ "$EXIT_CODE" != "0" ] || [ "$FAILED" -gt 0 ]; then
            REGRESSED="true"
          fi
          echo "regressed=$REGRESSED"           >> "$GITHUB_OUTPUT"
          echo "summary=$SUMMARY_LINE"          >> "$GITHUB_OUTPUT"
          {
            echo "## srcML regression test — ${{ inputs.language }}/${{ inputs.project }}"
            echo ""
            echo "- **Baseline:** \`${{ steps.cfg.outputs.release_tag }}\`"
            echo "- **Result:** $([ "$REGRESSED" = "true" ] && echo "regressions found" || echo "clean")"
            echo "- **Summary:** \`$SUMMARY_LINE\`"
            echo ""
            echo "### Details"
            echo ""
            head -n 200 results/grouped-failures.md
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Upload results artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ inputs.artifact-name }}
          path: results/
          retention-days: 30

      - name: Fail on regression
        if: ${{ inputs.fail-on-regression && steps.decide.outputs.regressed == 'true' }}
        run: |
          echo "::error::srcml --parser-test reported regressions against baseline ${{ steps.cfg.outputs.release_tag }} (${{ inputs.language }}/${{ inputs.project }})"
          exit 1