Nightly #42
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly | |
| # build.yml runs on push and pull_request, so it only tells you about changes | |
| # made HERE. This repository's other moving part is outside it: the parser | |
| # artifact published to https://www.sqlparser.com/maven/. A release there can | |
| # change demo output or drop an API without a single commit landing in this | |
| # repo, and nothing would notice until someone ran a demo by hand. | |
| # | |
| # So this workflow runs on a timer and covers both: | |
| # pinned -- ${gsp.core.version} from pom.xml, the version the README | |
| # documents, on the two JDKs the demos claim to support. | |
| # latest -- whatever is newest on sqlparser.com right now. This is the job | |
| # that earns the nightly. When it goes red and `pinned` stays | |
| # green, a new parser release broke the demos. | |
| # | |
| # Both jobs do the same four things: prove no parser jar is committed, compile | |
| # everything, run the whole test suite, and then actually run the demos -- | |
| # every one of them for launch, and a table of them with real arguments checked | |
| # against expected output. | |
| on: | |
| schedule: | |
| # 03:17 UTC. Not on the hour: GitHub drops scheduled runs when too many | |
| # repositories ask for the same popular minute. | |
| - cron: "17 3 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| parser_version: | |
| description: "Parser version for the 'latest' job (blank = resolve newest from sqlparser.com)" | |
| required: false | |
| type: string | |
| permissions: | |
| contents: read | |
| jobs: | |
| pinned: | |
| name: "pinned parser (JDK ${{ matrix.java }})" | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # The parser jar is Java 8 bytecode and the POM pins source/target 1.8. | |
| java: ["8", "21"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ matrix.java }} | |
| cache: maven | |
| # A vendored parser is what once let these demos compile against a build | |
| # years older than their own source, so its absence is an invariant worth | |
| # asserting rather than a convention worth documenting. | |
| - name: No parser jar is committed | |
| run: | | |
| set -euo pipefail | |
| if git ls-files | grep -E '(^|/)(gsqlparser|gudusoft\.gsqlparser)[^/]*\.jar$'; then | |
| echo "::error::a parser jar is committed; it must be resolved from Gudu's Maven repository" | |
| exit 1 | |
| fi | |
| echo "ok: no parser jar in the index" | |
| - name: Parser version is consistent across all POMs | |
| run: .github/scripts/set-parser-version.sh --check | |
| - name: Report the parser being used | |
| run: | | |
| set -euo pipefail | |
| v=$(mvn -B -q help:evaluate -Dexpression=gsp.core.version -DforceStdout) | |
| echo "pinned parser version: $v" | |
| echo "### Pinned parser \`$v\` (JDK ${{ matrix.java }})" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Compile everything | |
| run: mvn -B package -DskipTests | |
| - name: Test | |
| run: mvn -B test | |
| - name: Check the results | |
| run: .github/scripts/check-test-results.sh | |
| # The standalone lineage tool, run rather than merely built. It was its own | |
| # POM until 2026-07-28, and only ever compiled here, which is how it | |
| # shipped a missing JAXB dependency (issue #47) and an unrunnable | |
| # documented command (issue #46) through green builds. | |
| - name: Run the standalone dlineage jar | |
| run: .github/scripts/smoke-dlineage-jar.sh | |
| - name: Every demo starts | |
| run: .github/scripts/run-all-demos.sh --timeout 90 | |
| - name: Demos produce their expected output | |
| run: .github/scripts/run-demo-cases.sh --timeout 180 | |
| latest: | |
| name: "latest published parser" | |
| runs-on: ubuntu-latest | |
| # Needed only by the last two steps, which open the bump PR and unblock its | |
| # build. Everything above them is read-only. | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| actions: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: "21" | |
| cache: maven | |
| # <release> is the newest non-snapshot; fall back to <latest>, then to the | |
| # last <version> listed. Failing to resolve is an error rather than a | |
| # silent fallback to the pinned version: a job that quietly re-tests what | |
| # `pinned` already covers looks green while testing nothing new. | |
| - name: Resolve the newest published parser | |
| id: ver | |
| run: | | |
| set -euo pipefail | |
| if [ -n "${{ inputs.parser_version }}" ]; then | |
| v="${{ inputs.parser_version }}" | |
| echo "using the version supplied to workflow_dispatch: $v" | |
| else | |
| url=https://www.sqlparser.com/maven/com/gudusoft/gsqlparser/maven-metadata.xml | |
| curl -sSf --retry 3 --retry-delay 5 -o meta.xml "$url" | |
| v=$(sed -n 's:.*<release>\(.*\)</release>.*:\1:p' meta.xml | head -1) | |
| [ -n "$v" ] || v=$(sed -n 's:.*<latest>\(.*\)</latest>.*:\1:p' meta.xml | head -1) | |
| [ -n "$v" ] || v=$(sed -n 's:.*<version>\(.*\)</version>.*:\1:p' meta.xml | tail -1) | |
| if [ -z "$v" ]; then | |
| echo "::error::could not read a version out of $url" | |
| cat meta.xml | |
| exit 1 | |
| fi | |
| fi | |
| pinned=$(mvn -B -q help:evaluate -Dexpression=gsp.core.version -DforceStdout) | |
| echo "version=$v" >> "$GITHUB_OUTPUT" | |
| echo "newest published: $v / pinned in pom.xml: $pinned" | |
| { | |
| echo "### Latest published parser \`$v\`" | |
| if [ "$v" = "$pinned" ]; then | |
| echo "Same as the pinned version, so this job duplicates \`pinned\`." | |
| else | |
| echo "Pinned version is \`$pinned\`. If this job is red and \`pinned\` is green," | |
| echo "release \`$v\` broke the demos. If it is green, \`gsp.core.version\`" | |
| echo "can be bumped to \`$v\`." | |
| fi | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Compile everything | |
| run: mvn -B package -DskipTests -Dgsp.core.version=${{ steps.ver.outputs.version }} | |
| - name: Test | |
| run: mvn -B test -Dgsp.core.version=${{ steps.ver.outputs.version }} | |
| - name: Check the results | |
| run: .github/scripts/check-test-results.sh | |
| # Same smoke run as the pinned job, against the jar "Compile everything" | |
| # just built with -Dgsp.core.version=<newest release>. This is the check | |
| # that would notice a new parser breaking data lineage at runtime while | |
| # still compiling, which is precisely the failure mode a build-only step | |
| # cannot see. | |
| - name: Run the standalone dlineage jar against the newest release | |
| run: .github/scripts/smoke-dlineage-jar.sh | |
| - name: Every demo starts | |
| env: | |
| MVN_ARGS: -Dgsp.core.version=${{ steps.ver.outputs.version }} | |
| run: .github/scripts/run-all-demos.sh --timeout 90 | |
| - name: Demos produce their expected output | |
| env: | |
| MVN_ARGS: -Dgsp.core.version=${{ steps.ver.outputs.version }} | |
| run: .github/scripts/run-demo-cases.sh --timeout 180 | |
| # Everything above passed against this version, so the bump is already | |
| # proven rather than proposed on faith: full suite, every demo started, | |
| # and the demo cases checked against expected output. Opening a PR instead | |
| # of committing keeps a human in the loop and gives the push-triggered | |
| # build.yml a chance to disagree. | |
| # | |
| # Deliberately the LAST step in the job: any earlier failure means the new | |
| # release broke something, and then no PR is what we want. | |
| - name: Propose the bump when the newest release is green | |
| id: bump | |
| if: github.ref == 'refs/heads/master' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| NEW: ${{ steps.ver.outputs.version }} | |
| run: | | |
| set -euo pipefail | |
| pinned=$(mvn -B -q help:evaluate -Dexpression=gsp.core.version -DforceStdout) | |
| if [ "$NEW" = "$pinned" ]; then | |
| echo "already on $NEW, nothing to propose" | |
| exit 0 | |
| fi | |
| branch="bump-parser-$NEW" | |
| if gh pr list --head "$branch" --state open --json number \ | |
| --jq 'length' | grep -qv '^0$'; then | |
| echo "a PR for $branch is already open" | |
| exit 0 | |
| fi | |
| if git ls-remote --exit-code --heads origin "$branch" >/dev/null 2>&1; then | |
| echo "branch $branch already exists on origin; leaving it alone" | |
| exit 0 | |
| fi | |
| .github/scripts/set-parser-version.sh "$NEW" | |
| .github/scripts/set-parser-version.sh --check | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git checkout -b "$branch" | |
| git commit -aqm "Bump the parser from $pinned to $NEW | |
| The nightly ran the newest published release through the same checks | |
| the pinned version gets, and it passed all of them: | |
| - full test suite, no failures | |
| - every demo starts | |
| - the demo cases in .github/scripts/demo-cases.tsv produce their | |
| expected output | |
| Opened automatically by .github/workflows/nightly.yml. Merge it if you | |
| want the demos on $NEW; close it to stay on $pinned, and the nightly | |
| will keep testing $NEW without asking again until a newer release | |
| appears." | |
| git push -q origin "$branch" | |
| gh pr create --base master --head "$branch" \ | |
| --title "Bump the parser from $pinned to $NEW" \ | |
| --body "\`$NEW\` is the newest release on sqlparser.com. Tonight's nightly ran it through the same checks the pinned \`$pinned\` gets, and it passed all of them: the full test suite with no failures, every demo starting, and every case in \`.github/scripts/demo-cases.tsv\` producing its expected output. | |
| This touches the four places the version is written -- \`pom.xml\` and the three \`licensed-only/*/pom.xml\` -- via \`.github/scripts/set-parser-version.sh\`, so they cannot drift apart. | |
| Merge to move the demos to \`$NEW\`. Close to stay on \`$pinned\`; the nightly will go on testing \`$NEW\` and will not reopen this until a newer release appears. | |
| Opened by [\`nightly.yml\`](https://github.com/${{ github.repository }}/actions/workflows/nightly.yml)." | |
| # Read by the step below, which has to find the run this push | |
| # triggered. Set only on the path that actually opened a PR. | |
| { | |
| echo "branch=$branch" | |
| echo "sha=$(git rev-parse HEAD)" | |
| } >> "$GITHUB_OUTPUT" | |
| # The step above says build.yml "gets a chance to disagree". On its own it | |
| # does not. GitHub gates workflow runs from first-time contributors -- | |
| # this repository's policy is `first_time_contributors`, read back from | |
| # /actions/permissions/fork-pr-contributor-approval -- and | |
| # github-actions[bot] counts as one, so the pull_request run for the | |
| # branch pushed above is created in `action_required` at 0s and never | |
| # starts. | |
| # | |
| # The failure mode is quiet in the worst way: `gh pr checks` on the PR | |
| # reports "no checks reported", which reads as "this workflow does not | |
| # apply here" rather than "this workflow is blocked". PR #51 | |
| # (4.1.11 -> 4.2.6) sat like that on 2026-08-24 and was only caught | |
| # because somebody happened to look. Merging in that state loses every | |
| # part of build.yml this job does not already duplicate: the JDK 8 build, | |
| # the pre-commit hook test, and the whole windows-bat job. | |
| # | |
| # So approve it here. If the token is not allowed to, leave the notice on | |
| # the PR itself, where whoever is about to merge will actually see it -- | |
| # a warning annotation on an otherwise green nightly is the same kind of | |
| # thing nobody reads. | |
| - name: Let the PR's build actually run | |
| if: steps.bump.outputs.branch != '' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| BRANCH: ${{ steps.bump.outputs.branch }} | |
| SHA: ${{ steps.bump.outputs.sha }} | |
| run: | | |
| set -euo pipefail | |
| runs_url="https://github.com/${{ github.repository }}/actions" | |
| # The run does not exist the instant `gh pr create` returns. | |
| id="" | |
| for _ in $(seq 1 12); do | |
| id=$(gh api \ | |
| "/repos/${{ github.repository }}/actions/runs?event=pull_request&branch=$BRANCH" \ | |
| --jq ".workflow_runs[] | select(.head_sha == \"$SHA\") | .id" \ | |
| 2>/dev/null | head -1 || true) | |
| if [ -n "$id" ]; then | |
| break | |
| fi | |
| sleep 10 | |
| done | |
| if [ -z "$id" ]; then | |
| echo "::warning::no pull_request run appeared for $BRANCH within two minutes; check $runs_url" | |
| exit 0 | |
| fi | |
| status=$(gh api "/repos/${{ github.repository }}/actions/runs/$id" --jq .status) | |
| echo "run $id is '$status'" | |
| if [ "$status" != "action_required" ]; then | |
| echo "not gated, nothing to approve" | |
| exit 0 | |
| fi | |
| if gh api --method POST \ | |
| "/repos/${{ github.repository }}/actions/runs/$id/approve" >/dev/null; then | |
| echo "approved run $id; the PR's build will start" | |
| exit 0 | |
| fi | |
| echo "::warning::could not approve run $id; the bump PR has no checks until someone does" | |
| gh pr comment "$BRANCH" --body \ | |
| "This PR's \`build.yml\` run is **blocked**, not missing: GitHub parked it in \`action_required\` because \`github-actions[bot]\` counts as a first-time contributor, and \`nightly.yml\` could not approve it with its own token. | |
| Approve it at $runs_url/runs/$id before merging. Until then \`gh pr checks\` will say *no checks reported*, which is not the same as green: the JDK 8 build, the pre-commit hook test and the \`windows-bat\` job have not run against this version." || \ | |
| echo "::warning::could not comment on the PR either; approve $runs_url/runs/$id by hand" | |
| # The .bat family is the original Windows, no-Maven workflow. It went stale | |
| # for years because nothing ran it; build.yml runs it per push and this runs | |
| # it nightly, against a parser fetched fresh rather than a cached one. | |
| windows-bat: | |
| name: "Windows .bat scripts" | |
| runs-on: windows-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: "8" | |
| cache: maven | |
| - name: setenv.bat bootstraps the parser from Maven | |
| shell: cmd | |
| run: | | |
| if exist external_lib\gsqlparser-*.jar ( | |
| echo ::error::a parser jar is committed to this repository; it must be fetched, not vendored | |
| exit /b 1 | |
| ) | |
| call setenv\fetch-parser.bat | |
| dir external_lib | |
| for %%f in (external_lib\gsqlparser-*.jar) do exit /b 0 | |
| echo ::error::fetch-parser.bat produced no parser jar | |
| exit /b 1 | |
| - name: Compile every demo via compile_*.bat | |
| shell: cmd | |
| run: | | |
| setlocal enabledelayedexpansion | |
| set FAILED=0 | |
| set COUNT=0 | |
| for /r "%GITHUB_WORKSPACE%\src\main\java" %%s in (compile_*.bat) do ( | |
| set /a COUNT+=1 | |
| pushd "%%~dps" | |
| call "%%~nxs" < NUL > "%GITHUB_WORKSPACE%\c.txt" 2>&1 | |
| findstr /i /c:"error" /c:"file not found" /c:"no source files" "%GITHUB_WORKSPACE%\c.txt" >NUL && ( | |
| echo ::error::%%~nxs failed | |
| type "%GITHUB_WORKSPACE%\c.txt" | |
| set /a FAILED+=1 | |
| ) || echo ok %%~nxs | |
| popd | |
| ) | |
| echo. | |
| echo compiled !COUNT! demos, !FAILED! failed | |
| rem An explicit exit is required: a trailing `if` whose condition is | |
| rem false leaves whatever ERRORLEVEL the loop last set. | |
| if !FAILED! GTR 0 exit /b 1 | |
| exit /b 0 | |
| - name: Launch every demo via run_*.bat | |
| shell: cmd | |
| run: | | |
| setlocal enabledelayedexpansion | |
| set FAILED=0 | |
| set COUNT=0 | |
| for /r "%GITHUB_WORKSPACE%\src\main\java" %%s in (run_*.bat) do ( | |
| set /a COUNT+=1 | |
| pushd "%%~dps" | |
| call "%%~nxs" < NUL > "%GITHUB_WORKSPACE%\r.txt" 2>&1 | |
| findstr /c:"ClassNotFoundException" /c:"NoClassDefFoundError" /c:"Main method not found" "%GITHUB_WORKSPACE%\r.txt" >NUL && ( | |
| echo ::error::%%~nxs could not launch its class | |
| type "%GITHUB_WORKSPACE%\r.txt" | |
| set /a FAILED+=1 | |
| ) || echo ok %%~nxs | |
| popd | |
| ) | |
| echo. | |
| echo launched !COUNT! demos, !FAILED! failed | |
| if !FAILED! GTR 0 exit /b 1 | |
| exit /b 0 | |
| - name: Run four demos with real arguments | |
| shell: cmd | |
| run: | | |
| echo SELECT a.id FROM ta a; > "%GITHUB_WORKSPACE%\q.sql" | |
| call :demo checksyntax checksyntax checksyntax "/f %GITHUB_WORKSPACE%\q.sql /t oracle" "syntax errors: 0" || exit /b 1 | |
| call :demo formatsql formatsql formatsql "%GITHUB_WORKSPACE%\q.sql" "SELECT" || exit /b 1 | |
| call :demo listGSPInfo listGSPInfo listGSPInfo "" "Supported DBs" || exit /b 1 | |
| call :demo modifysql modifysql replaceTablename "" "output sql" || exit /b 1 | |
| echo All .bat demos passed. | |
| exit /b 0 | |
| :demo | |
| setlocal | |
| set DEMO=%~1 | |
| set CSCRIPT=%~2 | |
| set RSCRIPT=%~3 | |
| set ARGS=%~4 | |
| set EXPECT=%~5 | |
| echo. | |
| echo ==== %DEMO% : compile_%CSCRIPT%.bat / run_%RSCRIPT%.bat ==== | |
| cd /d "%GITHUB_WORKSPACE%\src\main\java\gudusoft\gsqlparser\demos\%DEMO%" | |
| call compile_%CSCRIPT%.bat < NUL > "%GITHUB_WORKSPACE%\c.txt" 2>&1 | |
| type "%GITHUB_WORKSPACE%\c.txt" | |
| findstr /i /c:"error" "%GITHUB_WORKSPACE%\c.txt" >NUL && ( | |
| echo ::error::compile_%CSCRIPT%.bat reported an error | |
| endlocal & exit /b 1 | |
| ) | |
| call run_%RSCRIPT%.bat %ARGS% < NUL > "%GITHUB_WORKSPACE%\r.txt" 2>&1 | |
| type "%GITHUB_WORKSPACE%\r.txt" | |
| findstr /c:"%EXPECT%" "%GITHUB_WORKSPACE%\r.txt" >NUL || ( | |
| echo ::error::run_%RSCRIPT%.bat did not print "%EXPECT%" | |
| endlocal & exit /b 1 | |
| ) | |
| endlocal & exit /b 0 |