From 463ac2d7dec879e9576263d0bdfca3cbb48200df Mon Sep 17 00:00:00 2001 From: Guillaume Nodet Date: Thu, 26 Mar 2026 17:08:09 +0100 Subject: [PATCH] chore(ci): rationalize CI into single workflow with unified comment Merge three separate CI actions (incremental-build, detect-dependencies, component-test) into one unified incremental-build script and simplify the workflow architecture. Deleted: .github/actions/component-test/ .github/actions/detect-dependencies/ Added: .github/workflows/pr-test-commenter.yml .github/CI-ARCHITECTURE.md Co-Authored-By: Claude Opus 4.6 --- .github/CI-ARCHITECTURE.md | 143 ++++ .github/actions/component-test/action.yaml | 121 ---- .../actions/component-test/component-test.sh | 71 -- .../actions/detect-dependencies/action.yaml | 39 -- .../detect-dependencies/detect-test.sh | 96 --- .github/actions/incremental-build/action.yaml | 26 +- .../incremental-build/incremental-build.sh | 649 +++++++++++++----- .github/workflows/main-build.yml | 1 - .github/workflows/pr-build-main.yml | 81 +-- .github/workflows/pr-commenter.yml | 2 +- .../workflows/pr-manual-component-test.yml | 89 ++- .github/workflows/pr-test-commenter.yml | 121 ++++ 12 files changed, 842 insertions(+), 597 deletions(-) create mode 100644 .github/CI-ARCHITECTURE.md delete mode 100644 .github/actions/component-test/action.yaml delete mode 100755 .github/actions/component-test/component-test.sh delete mode 100644 .github/actions/detect-dependencies/action.yaml delete mode 100755 .github/actions/detect-dependencies/detect-test.sh create mode 100644 .github/workflows/pr-test-commenter.yml diff --git a/.github/CI-ARCHITECTURE.md b/.github/CI-ARCHITECTURE.md new file mode 100644 index 0000000000000..b5f0dd8e6e3d5 --- /dev/null +++ b/.github/CI-ARCHITECTURE.md @@ -0,0 +1,143 @@ +# CI Architecture + +Overview of the GitHub Actions CI/CD ecosystem for Apache Camel. + +## Workflow Overview + +``` +PR opened/updated + │ + ├──► pr-id.yml ──► pr-commenter.yml (welcome message) + │ + └──► pr-build-main.yml (Build and test) + │ + ├── regen.sh (full build, no tests) + ├── incremental-build (test affected modules) + │ ├── File-path analysis + │ ├── POM dependency analysis + │ └── Extra modules (/component-test) + │ + └──► pr-test-commenter.yml (post unified comment) + +PR comment: /component-test kafka http + │ + └──► pr-manual-component-test.yml + │ + └── dispatches "Build and test" with extra_modules +``` + +## Workflows + +### `pr-build-main.yml` — Build and test +- **Trigger**: `pull_request` (main branch), `workflow_dispatch` +- **Matrix**: JDK 17, 21, 25 (25 is experimental) +- **Steps**: + 1. Full build via `regen.sh` (`mvn install -DskipTests -Pregen`) + 2. Check for uncommitted generated files + 3. Run incremental tests (only affected modules) + 4. Upload test comment as artifact +- **Inputs** (workflow_dispatch): `pr_number`, `pr_ref`, `extra_modules` + +### `pr-test-commenter.yml` — Post CI test comment +- **Trigger**: `workflow_run` on "Build and test" completion +- **Purpose**: Posts the unified test summary comment on the PR +- **Why separate**: Uses `workflow_run` to run in base repo context, allowing + comment posting on fork PRs (where `GITHUB_TOKEN` is read-only) + +### `pr-manual-component-test.yml` — /component-test handler +- **Trigger**: `issue_comment` with `/component-test` prefix +- **Who**: MEMBER, OWNER, or CONTRIBUTOR only +- **What**: Resolves component names to module paths, dispatches the main + "Build and test" workflow with `extra_modules` + +### `pr-id.yml` + `pr-commenter.yml` — Welcome message +- **Trigger**: `pull_request` (all branches) +- **Purpose**: Posts the one-time welcome message on new PRs +- **Why two workflows**: `pr-id.yml` runs in PR context (uploads PR number), + `pr-commenter.yml` runs via `workflow_run` with write permissions + +### `main-build.yml` — Main branch build +- **Trigger**: `push` to main, camel-4.14.x, camel-4.18.x +- **Steps**: Same as PR build but without comment posting + +### Other workflows +- `pr-labeler.yml` — Auto-labels PRs based on changed files +- `pr-doc-validation.yml` — Validates documentation changes +- `pr-cleanup-branches.yml` — Cleans up merged PR branches +- `alternative-os-build-main.yml` — Tests on non-Linux OSes +- `check-container-versions.yml` — Checks test container version updates +- `generate-sbom-main.yml` — Generates SBOM for releases +- `security-scan.yml` — Security vulnerability scanning + +## Actions + +### `incremental-build` +The core test runner. Determines which modules to test using: +1. **File-path analysis**: Maps changed files to Maven modules +2. **POM dependency analysis**: For changed `pom.xml` files, detects property + changes and finds modules that reference the affected properties in their + `pom.xml` files +3. **Extra modules**: Additional modules passed via `/component-test` + +Results are merged, deduplicated, and tested. The script also: +- Detects tests disabled in CI (`@DisabledIfSystemProperty(named = "ci.env.name")`) +- Applies an exclusion list for generated/meta modules +- Generates a unified PR comment with all test information + +### `install-mvnd` +Installs the Maven Daemon (mvnd) for faster builds. + +### `install-packages` +Installs system packages required for the build. + +## PR Labels + +| Label | Effect | +|-------|--------| +| `skip-tests` | Skip all tests | +| `test-dependents` | Force testing dependent modules even if threshold exceeded | + +## CI Environment + +The CI sets `-Dci.env.name=github.com` via `MVND_OPTS` (in `install-mvnd`). +Tests can use `@DisabledIfSystemProperty(named = "ci.env.name")` to skip +flaky tests in CI. The test comment warns about these skipped tests. + +## Known Limitations of POM Dependency Detection + +The property-grep approach has structural limitations that can cause missed modules: + +1. **Managed dependencies without explicit ``** — Most Camel modules + inherit dependency versions via `` in the parent POM + and do not declare `${property}` themselves. When a managed + dependency version property changes, only modules that explicitly reference + the property are detected — modules relying on inheritance are missed. + +2. **Maven plugin version changes are completely invisible** — Plugin version + properties (e.g. ``) are both defined and + consumed in `parent/pom.xml` via ``. Since the module + search excludes `parent/pom.xml`, no modules are found and **no tests run + at all** for plugin updates. Modules inherit plugins from the parent without + any `${property}` reference in their own `pom.xml`. + +3. **BOM imports** — When a BOM version property changes (e.g. + ``), modules using artifacts from that BOM are not + detected because they reference the BOM's artifacts, not the BOM property. + +4. **Transitive dependency changes** — Modules affected only via transitive + dependencies are not detected. + +5. **Non-property version changes** — Direct edits to `` values (not + using `${property}` substitution) or structural changes to + `` sections are not caught. + +These limitations mean the incremental build may under-test when parent POM +properties change. A future improvement could use +[Maveniverse Toolbox](https://github.com/maveniverse/toolbox) `tree-find` or +[Scalpel](https://github.com/maveniverse/scalpel) to resolve the full +dependency graph and detect all affected modules. + +## Comment Markers + +PR comments use HTML markers for upsert (create-or-update) behavior: +- `` — Unified test summary comment diff --git a/.github/actions/component-test/action.yaml b/.github/actions/component-test/action.yaml deleted file mode 100644 index 72eace547d684..0000000000000 --- a/.github/actions/component-test/action.yaml +++ /dev/null @@ -1,121 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: "Component Test Runner" -description: "Runs tests of corresponding to the given comment" -inputs: - run-id: - description: 'Id of the job' - required: true - pr-id: - description: 'Id of the pull request to update' - required: true - comment-id: - description: 'Id of the comment (unused, kept for backward compatibility with older PR branches)' - required: false - default: '' - comment-body: - description: 'Body of the comment to process' - required: true - artifact-upload-suffix: - description: 'Suffix for artifacts stored' - required: false - default: '' -runs: - using: "composite" - steps: - - id: install-mvnd - uses: ./.github/actions/install-mvnd - - name: maven build - shell: bash - run: ${{ github.action_path }}/component-test.sh - env: - MAVEN_BINARY: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd - COMMENT_BODY: ${{ inputs.comment-body }} - FAST_BUILD: "true" - LOG_FILE: build.log - - name: archive logs - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - if: always() - with: - name: build-${{ inputs.artifact-upload-suffix }}.log - path: build.log - - name: maven test - shell: bash - run: ${{ github.action_path }}/component-test.sh - env: - MAVEN_BINARY: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd - COMMENT_BODY: ${{ inputs.comment-body }} - FAST_BUILD: "false" - LOG_FILE: tests.log - - name: archive logs - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - if: always() - with: - name: tests-${{ inputs.artifact-upload-suffix }}.log - path: tests.log - - name: Check for disabled tests - id: disabled-tests - if: always() - shell: bash - env: - COMMENT_BODY: ${{ inputs.comment-body }} - run: | - # Resolve component paths from the comment (same logic as component-test.sh) - componentList="${COMMENT_BODY:16}" - skipped_modules="" - for component in ${componentList}; do - if [[ ${component} = camel-* ]]; then - componentPath="components/${component}" - else - componentPath="components/camel-${component}" - fi - if [[ -d "${componentPath}" ]]; then - # Search for @DisabledIfSystemProperty(named = "ci.env.name") in test sources - matches=$(grep -rl 'DisabledIfSystemProperty' "${componentPath}" --include="*.java" 2>/dev/null \ - | xargs grep -l 'ci.env.name' 2>/dev/null || true) - if [[ -n "$matches" ]]; then - count=$(echo "$matches" | wc -l | tr -d ' ') - skipped_modules="${skipped_modules}\n- \`${componentPath}\`: ${count} test(s) disabled on GitHub Actions" - fi - fi - done - if [[ -n "$skipped_modules" ]]; then - { - echo 'warning<> $GITHUB_OUTPUT - fi - - name: Success comment - if: success() - uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 - with: - issue-number: ${{ inputs.pr-id }} - body: | - :white_check_mark: `${{ inputs.comment-body }}` tests passed successfully. - ${{ steps.disabled-tests.outputs.warning }} - - name: Failure comment - if: failure() - uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 - with: - issue-number: ${{ inputs.pr-id }} - body: | - :x: `${{ inputs.comment-body }}` tests failed. Please [check the logs](https://github.com/${{ github.repository }}/actions/runs/${{ inputs.run-id }}). - ${{ steps.disabled-tests.outputs.warning }} diff --git a/.github/actions/component-test/component-test.sh b/.github/actions/component-test/component-test.sh deleted file mode 100755 index 736ebdd4836b8..0000000000000 --- a/.github/actions/component-test/component-test.sh +++ /dev/null @@ -1,71 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -echo "Using MVND_OPTS=$MVND_OPTS" - -function main() { - local mavenBinary=$MAVEN_BINARY - local commentBody=$COMMENT_BODY - local fastBuild=$FAST_BUILD - local log=$LOG_FILE - - if [[ ${commentBody} = /component-test* ]] ; then - local componentList="${commentBody:16}" - echo "The list of components to test is ${componentList}" - else - echo "No components have been detected, the expected format is '/component-test (camel-)component-name1 (camel-)component-name2...'" - exit 1 - fi - local pl="" - for component in ${componentList} - do - if [[ ${component} = camel-* ]] ; then - componentPath="components/${component}" - else - componentPath="components/camel-${component}" - fi - if [[ -d "${componentPath}" ]] ; then - pl="$pl$(find "${componentPath}" -name pom.xml -not -path "*/src/it/*" -not -path "*/target/*" -exec dirname {} \; | sort | tr -s "\n" ",")" - fi - done - len=${#pl} - if [[ "$len" -gt "0" ]] ; then - pl="${pl::len-1}" - else - echo "The components to test don't exist" - exit 1 - fi - - if [[ ${fastBuild} = "true" ]] ; then - echo "Launching a fast build against the projects ${pl} and their dependencies" - $mavenBinary -l $log $MVND_OPTS -Dquickly install -pl "$pl" -am - else - echo "Launching tests of the projects ${pl}" - $mavenBinary -l $log $MVND_OPTS install -pl "$pl" - ret=$? - - if [[ ${ret} -ne 0 ]] ; then - echo "Processing surefire and failsafe reports to create the summary" - echo -e "| Failed Test | Duration | Failure Type |\n| --- | --- | --- |" > "$GITHUB_STEP_SUMMARY" - find . -path '*target/*-reports*' -iname '*.txt' -exec .github/actions/incremental-build/parse_errors.sh {} \; - fi - - exit $ret - fi -} - -main "$@" diff --git a/.github/actions/detect-dependencies/action.yaml b/.github/actions/detect-dependencies/action.yaml deleted file mode 100644 index 3db5cf5bcc707..0000000000000 --- a/.github/actions/detect-dependencies/action.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: "Parent pom dependencies detector" -description: "Test only projects which have a dependency changed in the parent pom (typically dependabot)" -inputs: - github-token: - description: 'The github token to access to the API' - required: false - base-ref: - description: 'The base branch to compare against (defaults to github.base_ref)' - required: false - default: '' -runs: - using: "composite" - steps: - - id: install-mvnd - uses: apache/camel/.github/actions/install-mvnd@main - with: - dry-run: ${{ inputs.skip-mvnd-install }} - - name: maven test - env: - GITHUB_TOKEN: ${{ inputs.github-token }} - shell: bash - run: ${{ github.action_path }}/detect-test.sh ${{ inputs.base-ref || github.base_ref }} ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd diff --git a/.github/actions/detect-dependencies/detect-test.sh b/.github/actions/detect-dependencies/detect-test.sh deleted file mode 100755 index dcdb3a9e218fc..0000000000000 --- a/.github/actions/detect-dependencies/detect-test.sh +++ /dev/null @@ -1,96 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -euo pipefail - -detect_changed_properties() { - local base_branch="$1" - - git diff "${base_branch}" -- parent/pom.xml | \ - grep -E '^[+-]\s*<[^>]+>[^<]*]+>' | \ - grep -vE '^\+\+\+|---' | \ - grep -E 'version|dependency|artifact' | \ - sed -E 's/^[+-]\s*<([^>]+)>.*/\1/' | \ - sort -u || true -} - -find_affected_modules() { - local property_name="$1" - local mavenBinary=${2} - local affected=() - - while IFS= read -r pom; do - # skip any target that may have been built previously - if [[ "$pom" != */target/* ]]; then - # only consider certain modules, nothing else - if [[ "$pom" == */catalog/* ]] || \ - [[ "$pom" == */components/* ]] || \ - [[ "$pom" == */core/* ]] || \ - ([[ "$pom" == */dsl/* ]] && [[ "$pom" != */dsl/camel-jbang* ]]); then - if grep -q "\${${property_name}}" "$pom"; then - affected+=("$pom") - fi - fi - fi - done < <(find . -name "pom.xml") - - affected_transformed="" - - for pom in "${affected[@]}"; do - if [[ -f "$pom" ]]; then - artifactId=$($mavenBinary -f "$pom" help:evaluate -Dexpression=project.artifactId -q --raw-streams -DforceStdout) - if [ ! -z "$artifactId" ]; then - affected_transformed+=":$artifactId," - fi - fi - done - - echo "$affected_transformed" -} - -main() { - echo "Using MVND_OPTS=$MVND_OPTS" - local base_branch=${1} - local mavenBinary=${2} - local exclusionList="!:camel-allcomponents,!:dummy-component,!:camel-catalog,!:camel-catalog-console,!:camel-catalog-lucene,!:camel-catalog-maven,!:camel-catalog-suggest,!:camel-route-parser,!:camel-csimple-maven-plugin,!:camel-report-maven-plugin,!:camel-endpointdsl,!:camel-componentdsl,!:camel-endpointdsl-support,!:camel-yaml-dsl,!:camel-kamelet-main,!:camel-yaml-dsl-deserializers,!:camel-yaml-dsl-maven-plugin,!:camel-jbang-core,!:camel-jbang-main,!:camel-jbang-plugin-generate,!:camel-jbang-plugin-edit,!:camel-jbang-plugin-kubernetes,!:camel-jbang-plugin-test,!:camel-launcher,!:camel-jbang-it,!:camel-itest,!:docs,!:apache-camel" - - git fetch origin $base_branch:$base_branch - - changed_props=$(detect_changed_properties "$base_branch") - - if [ -z "$changed_props" ]; then - echo "✅ No property changes detected." - exit 0 - fi - - modules_affected="" - - while read -r prop; do - modules=$(find_affected_modules "$prop" $mavenBinary) - modules_affected+="$modules" - done <<< "$changed_props" - - if [ -z "$modules_affected" ]; then - echo "✅ No components affected by property changes detected." - exit 0 - fi - - echo "🧪 Testing the following modules $modules_affected and its dependents" - $mavenBinary $MVND_OPTS test -pl "$modules_affected$exclusionList" -amd -} - -main "$@" diff --git a/.github/actions/incremental-build/action.yaml b/.github/actions/incremental-build/action.yaml index 0166f9d3aad17..728031894b669 100644 --- a/.github/actions/incremental-build/action.yaml +++ b/.github/actions/incremental-build/action.yaml @@ -15,15 +15,13 @@ # limitations under the License. # -name: "Incremental Build Runner" -description: "Build only affected projects" +name: "Incremental Test Runner" +description: "Test only affected projects, using file-path analysis and POM dependency detection" inputs: - mode: - description: 'The mode to launch, it can be build or test' - required: true pr-id: - description: 'Id of the pull request' - required: true + description: 'Id of the pull request (optional for push builds)' + required: false + default: '' github-token: description: 'The github token to access to the API' required: false @@ -39,6 +37,10 @@ inputs: description: 'Suffix for artifacts stored' required: false default: '' + extra-modules: + description: 'Additional modules to test (comma-separated paths, e.g. from /component-test)' + required: false + default: '' runs: using: "composite" steps: @@ -46,17 +48,17 @@ runs: uses: apache/camel/.github/actions/install-mvnd@main with: dry-run: ${{ inputs.skip-mvnd-install }} - - name: maven build + - name: maven test env: GITHUB_TOKEN: ${{ inputs.github-token }} - MODE: ${{ inputs.mode }} PR_ID: ${{ inputs.pr-id }} GITHUB_REPO: ${{ inputs.github-repo }} + EXTRA_MODULES: ${{ inputs.extra-modules }} shell: bash - run: ${{ github.action_path }}/incremental-build.sh ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd $MODE $PR_ID $GITHUB_REPO + run: ${{ github.action_path }}/incremental-build.sh ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd "$PR_ID" "$GITHUB_REPO" "$EXTRA_MODULES" - name: archive logs uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: always() with: - name: incremental-${{ inputs.mode }}-${{ inputs.artifact-upload-suffix }}.log - path: incremental-${{ inputs.mode }}.log + name: incremental-test-${{ inputs.artifact-upload-suffix }}.log + path: incremental-test.log diff --git a/.github/actions/incremental-build/incremental-build.sh b/.github/actions/incremental-build/incremental-build.sh index 37eb34395c909..82ce7d4992d3d 100755 --- a/.github/actions/incremental-build/incremental-build.sh +++ b/.github/actions/incremental-build/incremental-build.sh @@ -15,18 +15,34 @@ # limitations under the License. # +# Incremental test runner for Apache Camel PRs. +# +# Determines which modules to test by: +# 1. File-path analysis: maps changed files to their Maven modules +# 2. POM dependency analysis: for changed pom.xml files, detects property +# changes and finds modules that reference the affected properties +# +# Both sets of affected modules are merged and deduplicated before testing. + +set -euo pipefail + echo "Using MVND_OPTS=$MVND_OPTS" -maxNumberOfBuildableProjects=100 -maxNumberOfTestableProjects=50 +maxNumberOfTestableProjects=1000 + +# Modules excluded from targeted testing (generated code, meta-modules, etc.) +EXCLUSION_LIST="!:camel-allcomponents,!:dummy-component,!:camel-catalog,!:camel-catalog-console,!:camel-catalog-lucene,!:camel-catalog-maven,!:camel-catalog-suggest,!:camel-route-parser,!:camel-csimple-maven-plugin,!:camel-report-maven-plugin,!:camel-endpointdsl,!:camel-componentdsl,!:camel-endpointdsl-support,!:camel-yaml-dsl,!:camel-kamelet-main,!:camel-yaml-dsl-deserializers,!:camel-yaml-dsl-maven-plugin,!:camel-jbang-core,!:camel-jbang-main,!:camel-jbang-plugin-generate,!:camel-jbang-plugin-edit,!:camel-jbang-plugin-kubernetes,!:camel-jbang-plugin-test,!:camel-launcher,!:camel-jbang-it,!:camel-itest,!:docs,!:apache-camel,!:coverage" + +# ── Utility functions ────────────────────────────────────────────────── -function findProjectRoot () { +# Walk up from a file path to find the nearest directory containing a pom.xml +findProjectRoot() { local path=${1} while [[ "$path" != "." ]]; do - if [[ ! -e "$path/pom.xml" ]] ; then - path=$(dirname $path) - elif [[ $(dirname $path) == */src/it ]] ; then - path=$(dirname $(dirname $path)) + if [[ ! -e "$path/pom.xml" ]]; then + path=$(dirname "$path") + elif [[ $(dirname "$path") == */src/it ]]; then + path=$(dirname "$(dirname "$path")") else break fi @@ -34,210 +50,509 @@ function findProjectRoot () { echo "$path" } -function hasLabel() { - local issueNumber=${1} - local label="incremental-${2}" - local repository=${3} - curl -s \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}"\ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/${repository}/issues/${issueNumber}/labels" | jq -r '.[].name' | grep -c "$label" +# Check whether a PR label exists +hasLabel() { + local issueNumber=${1} + local label="incremental-${2}" + local repository=${3} + curl -s \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${repository}/issues/${issueNumber}/labels" | jq -r '.[].name' | { grep -c "$label" || true; } } -function main() { - local mavenBinary=${1} - local mode=${2} - local log="incremental-${mode}.log" - local prId=${3} - local ret=0 - local repository=${4} - local testedDependents="" +# Fetch the PR diff from the GitHub API. Returns the full unified diff. +fetchDiff() { + local prId="$1" + local repository="$2" - echo "Searching for affected projects" - local projects local diff_output - diff_output=$(curl -s -w "\n%{http_code}" -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "Accept: application/vnd.github.v3.diff" "https://api.github.com/repos/${repository}/pulls/${prId}") + diff_output=$(curl -s -w "\n%{http_code}" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "Accept: application/vnd.github.v3.diff" \ + "https://api.github.com/repos/${repository}/pulls/${prId}") + local http_code http_code=$(echo "$diff_output" | tail -n 1) local diff_body diff_body=$(echo "$diff_output" | sed '$d') - if [[ "$http_code" -lt 200 || "$http_code" -ge 300 || -z "$diff_body" ]] ; then - echo "WARNING: Failed to fetch PR diff (HTTP $http_code). Falling back to full build." - diff_body="" + + if [[ "$http_code" -lt 200 || "$http_code" -ge 300 || -z "$diff_body" ]]; then + echo "WARNING: Failed to fetch PR diff (HTTP $http_code). Falling back to full build." >&2 + return + fi + echo "$diff_body" +} + +# ── POM dependency analysis (previously detect-dependencies) ─────────── + +# Extract the diff section for a specific pom.xml file from the full diff +extractPomDiff() { + local diff_body="$1" + local pom_path="$2" + + echo "$diff_body" | awk -v target="a/${pom_path}" ' + /^diff --git/ && found { exit } + /^diff --git/ && index($0, target) { found=1 } + found { print } + ' +} + +# Detect which properties changed in a pom.xml diff. +# Returns one property name per line. +# Filters out structural XML elements (groupId, artifactId, version, etc.) +# to only return actual property names (e.g. openai-java-version). +detectChangedProperties() { + local diff_content="$1" + + # Known structural POM elements that are NOT property names + local structural_elements="groupId|artifactId|version|scope|type|classifier|optional|systemPath|exclusions|exclusion|dependency|dependencies|dependencyManagement|parent|modules|module|packaging|name|description|url|relativePath" + + echo "$diff_content" | \ + grep -E '^[+-][[:space:]]*<[^>]+>[^<]*]+>' | \ + grep -vE '^\+\+\+|^---' | \ + sed -E 's/^[+-][[:space:]]*<([^>]+)>.*/\1/' | \ + grep -vE "^(${structural_elements})$" | \ + sort -u || true +} + +# Find modules that reference a property in their pom.xml. +# Searches pom.xml files under catalog/, components/, core/, dsl/ for +# ${property_name} references and extracts the module's artifactId. +# Adds discovered artifactIds to the dep_module_ids variable +# (which must be declared in the caller). +findAffectedModules() { + local property="$1" + + local matches + matches=$(grep -rl "\${${property}}" --include="pom.xml" . 2>/dev/null | \ + grep -v "^\./parent/pom.xml" | \ + grep -v "/target/" || true) + + if [ -z "$matches" ]; then + return + fi + + while read -r pom_file; do + [ -z "$pom_file" ] && continue + + # Only consider catalog, components, core, dsl paths (same as original detect-test.sh) + if [[ "$pom_file" == */catalog/* ]] || \ + [[ "$pom_file" == */components/* ]] || \ + [[ "$pom_file" == */core/* ]] || \ + ([[ "$pom_file" == */dsl/* ]] && [[ "$pom_file" != */dsl/camel-jbang* ]]); then + local mod_artifact + mod_artifact=$(sed -n '//,/<\/parent>/!{ s/.*\([^<]*\)<\/artifactId>.*/\1/p }' "$pom_file" | head -1) + if [ -n "$mod_artifact" ] && ! echo ",$dep_module_ids," | grep -q ",:${mod_artifact},"; then + echo " Property '\${${property}}' referenced by: $mod_artifact" + dep_module_ids="${dep_module_ids:+${dep_module_ids},}:${mod_artifact}" + fi + fi + done <<< "$matches" +} + +# Analyze pom.xml changes to find affected modules via property grep. +# Adds discovered module artifactIds to the dep_module_ids variable +# (which must be declared in the caller). +analyzePomDependencies() { + local diff_body="$1" + local pom_path="$2" # e.g. "parent/pom.xml" or "components/camel-foo/pom.xml" + + local pom_diff + pom_diff=$(extractPomDiff "$diff_body" "$pom_path") + if [ -z "$pom_diff" ]; then + return + fi + + local changed_props + changed_props=$(detectChangedProperties "$pom_diff") + if [ -z "$changed_props" ]; then + return + fi + + echo " Property changes detected in ${pom_path}:" + echo "$changed_props" | while read -r p; do echo " - $p"; done + + while read -r prop; do + [ -z "$prop" ] && continue + findAffectedModules "$prop" + done <<< "$changed_props" +} + +# ── Disabled-test detection ───────────────────────────────────────────── + +# Scan tested modules for @DisabledIfSystemProperty(named = "ci.env.name") +# and return a markdown warning listing affected files. +detectDisabledTests() { + local final_pl="$1" + local skipped="" + + for mod_path in $(echo "$final_pl" | tr ',' '\n'); do + # Skip artifactId-style references (e.g. :camel-openai) — only scan paths + if [[ "$mod_path" == :* ]]; then + continue + fi + if [ -d "$mod_path" ]; then + local matches + matches=$(grep -rl 'DisabledIfSystemProperty' "$mod_path" --include="*.java" 2>/dev/null \ + | xargs grep -l 'ci.env.name' 2>/dev/null || true) + if [ -n "$matches" ]; then + local count + count=$(echo "$matches" | wc -l | tr -d ' ') + skipped="${skipped}\n- \`${mod_path}\`: ${count} test(s) disabled on GitHub Actions" + fi + fi + done + + if [ -n "$skipped" ]; then + echo -e "$skipped" + fi +} + +# ── Comment generation ───────────────────────────────────────────────── + +writeComment() { + local comment_file="$1" + local pl="$2" + local dep_ids="$3" + local changed_props_summary="$4" + local testedDependents="$5" + local extra_modules="$6" + + echo "" > "$comment_file" + + # Section 1: file-path-based modules + if [ -n "$pl" ]; then + echo ":test_tube: **CI tested the following changed modules:**" >> "$comment_file" + echo "" >> "$comment_file" + for w in $(echo "$pl" | tr ',' '\n'); do + echo "- \`$w\`" >> "$comment_file" + done + + if [[ "${testedDependents}" = "false" ]]; then + echo "" >> "$comment_file" + echo "> :information_source: Dependent modules were not tested because the total number of affected modules exceeded the threshold (${maxNumberOfTestableProjects}). Use the \`test-dependents\` label to force testing all dependents." >> "$comment_file" + fi fi + + # Section 2: pom dependency-detected modules + if [ -n "$dep_ids" ]; then + echo "" >> "$comment_file" + if [ -n "$changed_props_summary" ]; then + echo ":white_check_mark: **POM dependency changes: targeted tests included**" >> "$comment_file" + echo "" >> "$comment_file" + echo "Changed properties: ${changed_props_summary}" >> "$comment_file" + echo "" >> "$comment_file" + local dep_count + dep_count=$(echo "$dep_ids" | tr ',' '\n' | wc -l | tr -d ' ') + echo "
Modules affected by dependency changes (${dep_count})" >> "$comment_file" + echo "" >> "$comment_file" + echo "$dep_ids" | tr ',' '\n' | while read -r m; do + echo "- \`$m\`" >> "$comment_file" + done + echo "" >> "$comment_file" + echo "
" >> "$comment_file" + fi + fi + + # Section 3: extra modules (from /component-test) + if [ -n "$extra_modules" ]; then + echo "" >> "$comment_file" + echo ":heavy_plus_sign: **Additional modules tested** (via \`/component-test\`):" >> "$comment_file" + echo "" >> "$comment_file" + for w in $(echo "$extra_modules" | tr ',' '\n'); do + echo "- \`$w\`" >> "$comment_file" + done + fi + + if [ -z "$pl" ] && [ -z "$dep_ids" ] && [ -z "$extra_modules" ]; then + echo ":information_source: CI did not run targeted module tests." >> "$comment_file" + fi +} + +# ── Main ─────────────────────────────────────────────────────────────── + +main() { + local mavenBinary=${1} + local prId=${2} + local repository=${3} + local extraModules=${4:-} + local log="incremental-test.log" + local ret=0 + local testedDependents="" + + # Check for skip-tests label (only for PR builds) + if [ -n "$prId" ]; then + local mustSkipTests + mustSkipTests=$(hasLabel "${prId}" "skip-tests" "${repository}") + if [[ ${mustSkipTests} = "1" ]]; then + echo "The skip-tests label has been detected, no tests will be launched" + echo "" > "incremental-test-comment.md" + echo ":information_source: CI did not run targeted module tests (skip-tests label detected)." >> "incremental-test-comment.md" + exit 0 + fi + fi + + # Fetch the diff (PR diff via API, or git diff for push builds) + local diff_body + if [ -n "$prId" ]; then + echo "Fetching PR #${prId} diff..." + diff_body=$(fetchDiff "$prId" "$repository") + else + echo "No PR ID, using git diff HEAD~1..." + diff_body=$(git diff HEAD~1 2>/dev/null || true) + fi + + if [ -z "$diff_body" ]; then + echo "Could not fetch diff, skipping tests" + exit 0 + fi + + # ── Step 1: File-path analysis ── + echo "Searching for affected projects by file path..." + local projects projects=$(echo "$diff_body" | sed -n -e '/^diff --git a/p' | awk '{print $3}' | cut -b 3- | sed 's|\(.*\)/.*|\1|' | uniq | sort) + local pl="" local lastProjectRoot="" - local buildAll=false local totalAffected=0 - for project in ${projects} - do - if [[ ${project} == */archetype-resources ]] ; then + local pom_files="" + + for project in ${projects}; do + if [[ ${project} == */archetype-resources ]]; then continue - elif [[ ${project} != .* ]] ; then + elif [[ ${project} != .* ]]; then local projectRoot - projectRoot=$(findProjectRoot ${project}) - if [[ ${projectRoot} = "." ]] ; then - echo "The root project is affected, so a complete build is triggered" - buildAll=true - totalAffected=1 - break - elif [[ ${projectRoot} != "${lastProjectRoot}" ]] ; then - (( totalAffected ++ )) + projectRoot=$(findProjectRoot "${project}") + if [[ ${projectRoot} = "." ]]; then + # Root project change — don't add to pl, will rely on dependency analysis + # for pom.xml changes, or skip if it's just config files + continue + elif [[ ${projectRoot} != "${lastProjectRoot}" ]]; then + totalAffected=$((totalAffected + 1)) pl="$pl,${projectRoot}" lastProjectRoot=${projectRoot} fi fi done + pl="${pl:1}" # strip leading comma - if [[ ${totalAffected} = 0 ]] ; then - echo "There is nothing to build" - exit 0 - elif [[ ${totalAffected} -gt ${maxNumberOfBuildableProjects} ]] ; then - echo "There are too many affected projects, so a complete build is triggered" - buildAll=true - fi - pl="${pl:1}" - - if [[ ${mode} = "build" ]] ; then - local mustBuildAll - mustBuildAll=$(hasLabel ${prId} "build-all" ${repository}) - if [[ ${mustBuildAll} = "1" ]] ; then - echo "The build-all label has been detected thus all projects must be built" - buildAll=true - fi - if [[ ${buildAll} = "true" ]] ; then - echo "Building all projects" - $mavenBinary -l $log $MVND_OPTS -DskipTests install - ret=$? - else - local buildDependents - buildDependents=$(hasLabel ${prId} "build-dependents" ${repository}) - local totalTestableProjects - if [[ ${buildDependents} = "1" ]] ; then - echo "The build-dependents label has been detected thus the projects that depend on the affected projects will be built" - totalTestableProjects=0 - else - for w in $pl; do - echo "$w" - done - totalTestableProjects=$(./mvnw -B -q -amd exec:exec -Dexec.executable="pwd" -pl "$pl" | wc -l) - fi - if [[ ${totalTestableProjects} -gt ${maxNumberOfTestableProjects} ]] ; then - echo "Launching fast build command against the projects ${pl}, their dependencies and the projects that depend on them" - for w in $pl; do - echo "$w" - done - $mavenBinary -l $log $MVND_OPTS -DskipTests install -pl "$pl" -amd -am - ret=$? - else - echo "Launching fast build command against the projects ${pl} and their dependencies" - for w in $pl; do - echo "$w" - done - $mavenBinary -l $log $MVND_OPTS -DskipTests install -pl "$pl" -am - ret=$? + # Collect all changed pom.xml files for dependency analysis + pom_files=$(echo "$diff_body" | sed -n -e '/^diff --git a/p' | awk '{print $3}' | cut -b 3- | grep '/pom\.xml$' | sort -u || true) + + # ── Step 2: POM dependency analysis ── + # Variables shared with analyzePomDependencies/findAffectedModules + local dep_module_ids="" + local all_changed_props="" + + if [ -n "$pom_files" ]; then + echo "" + echo "Analyzing POM dependency changes..." + while read -r pom_file; do + [ -z "$pom_file" ] && continue + + # Capture changed props for this pom before calling analyze + local pom_diff + pom_diff=$(extractPomDiff "$diff_body" "$pom_file") + if [ -n "$pom_diff" ]; then + local props + props=$(detectChangedProperties "$pom_diff") + if [ -n "$props" ]; then + all_changed_props="${all_changed_props:+${all_changed_props}, }$(echo "$props" | tr '\n' ',' | sed 's/,$//')" + fi fi + + analyzePomDependencies "$diff_body" "$pom_file" + done <<< "$pom_files" + fi + + # ── Step 3: Merge and deduplicate ── + # Separate file-path modules into testable (has src/test) and pom-only. + # Pom-only modules (e.g. "parent") are kept in the build list but must NOT + # be expanded with -amd, since that would pull in every dependent module. + local testable_pl="" + local pom_only_pl="" + for w in $(echo "$pl" | tr ',' '\n'); do + if [ -d "$w/src/test" ]; then + testable_pl="${testable_pl:+${testable_pl},}${w}" + else + pom_only_pl="${pom_only_pl:+${pom_only_pl},}${w}" + echo " Pom-only module (no src/test, won't expand dependents): $w" fi - [[ -z $(git status --porcelain | grep -v antora.yml) ]] || { echo 'There are uncommitted changes'; git status; echo; echo; git diff; exit 1; } - else - local mustSkipTests - mustSkipTests=$(hasLabel ${prId} "skip-tests" ${repository}) - if [[ ${mustSkipTests} = "1" ]] ; then - echo "The skip-tests label has been detected thus no test will be launched" - buildAll=true - elif [[ ${buildAll} = "true" ]] ; then - echo "Cannot launch the tests of all projects, so no test will be launched" + done + + # Build final_pl: testable file-path modules + dependency-detected + pom-only + extra + local final_pl="" + if [ -n "$testable_pl" ]; then + final_pl="$testable_pl" + fi + if [ -n "$dep_module_ids" ]; then + final_pl="${final_pl:+${final_pl},}${dep_module_ids}" + fi + if [ -n "$pom_only_pl" ]; then + final_pl="${final_pl:+${final_pl},}${pom_only_pl}" + fi + + # Merge extra modules (e.g. from /component-test) + if [ -n "$extraModules" ]; then + echo "" + echo "Extra modules requested: $extraModules" + final_pl="${final_pl:+${final_pl},}${extraModules}" + fi + + if [ -z "$final_pl" ]; then + echo "" + echo "No modules to test" + writeComment "incremental-test-comment.md" "" "" "" "" "" + exit 0 + fi + + echo "" + echo "Modules to test:" + for w in $(echo "$final_pl" | tr ',' '\n'); do + echo " - $w" + done + echo "" + + # ── Step 4: Run tests ── + # Decide whether to use -amd (also-make-dependents): + # - Use -amd when there are testable file-path modules (to test their dependents) + # - Subject to threshold check to avoid testing too many modules + # - Pom-only modules are excluded from -pl to prevent -amd from pulling in everything + # (Maven builds them implicitly as dependencies of child modules) + local use_amd=false + local testDependents="0" + + if [ -n "$testable_pl" ]; then + # File-path modules with tests — use -amd to catch dependents + if [ -n "$prId" ]; then + testDependents=$(hasLabel "${prId}" "test-dependents" "${repository}") + fi + + if [[ ${testDependents} = "1" ]]; then + echo "The test-dependents label has been detected, testing dependents too" + use_amd=true + testedDependents=true else - local testDependents - testDependents=$(hasLabel ${prId} "test-dependents" ${repository}) local totalTestableProjects - if [[ ${testDependents} = "1" ]] ; then - echo "The test-dependents label has been detected thus the projects that depend on affected projects will be tested" - testedDependents=true - totalTestableProjects=0 - else - totalTestableProjects=$(./mvnw -B -q -amd exec:exec -Dexec.executable="pwd" -pl "$pl" | wc -l) - fi - if [[ ${totalTestableProjects} -gt ${maxNumberOfTestableProjects} ]] ; then - echo "There are too many projects to test (${totalTestableProjects} > ${maxNumberOfTestableProjects}) so only the affected projects are tested:" + totalTestableProjects=$(./mvnw -B -q -amd exec:exec -Dexec.executable="pwd" -pl "$testable_pl" 2>/dev/null | wc -l || echo "0") + + if [[ ${totalTestableProjects} -gt ${maxNumberOfTestableProjects} ]]; then + echo "Too many dependent modules (${totalTestableProjects} > ${maxNumberOfTestableProjects}), testing only the affected modules" testedDependents=false - for w in $pl; do - echo "$w" - done - # This need to install, other commands like test are not enough, otherwise test-infra will fail due to jandex maven plugin - $mavenBinary -l $log $MVND_OPTS install -pl "$pl" - ret=$? else - echo "Testing the affected projects and the projects that depend on them (${totalTestableProjects} modules):" + echo "Testing affected modules and their dependents (${totalTestableProjects} modules)" + use_amd=true testedDependents=true - for w in $pl; do - echo "$w" - done - # This need to install, other commands like test are not enough, otherwise test-infra will fail due to jandex maven plugin - $mavenBinary -l $log $MVND_OPTS install -pl "$pl" -amd - ret=$? fi fi + elif [ -n "$dep_module_ids" ]; then + # Only dependency-detected modules (no file-path code changes) + echo "POM dependency analysis found affected modules — testing specific modules" + testedDependents=true + else + # Only pom-only modules, no testable code and no dependency results + echo "Only pom-only modules changed with no detected dependency impact" + testedDependents=true fi - # Write the list of tested modules to the step summary and PR comment file - local comment_file="incremental-${mode}-comment.md" - if [[ -n "$pl" && ${buildAll} != "true" ]] ; then - echo "### Changed modules" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "" > "$comment_file" - echo ":test_tube: **CI tested the following changed modules:**" >> "$comment_file" + # Build the -pl argument: + # - Exclude pom-only modules from -pl when using -amd (they'd pull in everything) + # - Append exclusion list when dependency-detected modules are present + local build_pl="$final_pl" + if [[ "$use_amd" = true ]] && [ -n "$pom_only_pl" ]; then + # Remove pom-only modules — Maven builds them implicitly as dependencies + build_pl="" + if [ -n "$testable_pl" ]; then + build_pl="$testable_pl" + fi + if [ -n "$dep_module_ids" ]; then + build_pl="${build_pl:+${build_pl},}${dep_module_ids}" + fi + if [ -n "$extraModules" ]; then + build_pl="${build_pl:+${build_pl},}${extraModules}" + fi + fi + if [ -n "$dep_module_ids" ]; then + build_pl="${build_pl},${EXCLUSION_LIST}" + fi + + # This needs to install, not just test, otherwise test-infra will fail due to jandex maven plugin + if [[ "$use_amd" = true ]]; then + $mavenBinary -l "$log" $MVND_OPTS install -pl "$build_pl" -amd || ret=$? + else + $mavenBinary -l "$log" $MVND_OPTS install -pl "$build_pl" || ret=$? + fi + + # ── Step 5: Write comment and summary ── + local comment_file="incremental-test-comment.md" + writeComment "$comment_file" "$pl" "$dep_module_ids" "$all_changed_props" "$testedDependents" "$extraModules" + + # Check for tests disabled in CI via @DisabledIfSystemProperty(named = "ci.env.name") + local disabled_tests + disabled_tests=$(detectDisabledTests "$final_pl") + if [ -n "$disabled_tests" ]; then echo "" >> "$comment_file" - for w in $(echo "$pl" | tr ',' '\n'); do - echo "- \`$w\`" >> "$GITHUB_STEP_SUMMARY" - echo "- \`$w\`" >> "$comment_file" - done - echo "" >> "$GITHUB_STEP_SUMMARY" - # Add note about dependent modules testing scope - if [[ ${mode} = "test" && "${testedDependents:-}" = "false" ]] ; then + echo ":warning: **Some tests are disabled on GitHub Actions** (\`@DisabledIfSystemProperty(named = \"ci.env.name\")\`) and require manual verification:" >> "$comment_file" + echo "$disabled_tests" >> "$comment_file" + fi + + # Append reactor module list from build log + if [[ -f "$log" ]]; then + local reactor_modules + reactor_modules=$(grep '^\[INFO\] Camel ::' "$log" | sed 's/\[INFO\] //' | sed 's/ \..*$//' | sort -u || true) + if [[ -n "$reactor_modules" ]]; then + local count + count=$(echo "$reactor_modules" | wc -l | tr -d ' ') + local reactor_label + if [[ "${testedDependents}" = "false" ]]; then + reactor_label="Build reactor — dependencies compiled but only changed modules were tested" + else + reactor_label="All tested modules" + fi + echo "" >> "$comment_file" - echo "> :information_source: Dependent modules were not tested because the total number of affected modules exceeded the threshold (${maxNumberOfTestableProjects}). Use the \`test-dependents\` label to force testing all dependents." >> "$comment_file" + echo "
${reactor_label} ($count modules)" >> "$comment_file" echo "" >> "$comment_file" - fi - # Extract full reactor module list from the build log - if [[ -f "$log" ]] ; then - local reactor_modules - reactor_modules=$(grep '^\[INFO\] Camel ::' "$log" | sed 's/\[INFO\] //' | sed 's/ \..*$//' | sort -u) - if [[ -n "$reactor_modules" ]] ; then - local count - count=$(echo "$reactor_modules" | wc -l | tr -d ' ') - local reactor_label - if [[ ${mode} = "test" && "${testedDependents:-}" = "false" ]] ; then - reactor_label="Build reactor — dependencies compiled but only changed modules were tested" - else - reactor_label="All tested modules" - fi + + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + echo "" >> "$GITHUB_STEP_SUMMARY" echo "
${reactor_label} ($count)" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$comment_file" - echo "
${reactor_label} ($count modules)" >> "$comment_file" - echo "" >> "$comment_file" - echo "$reactor_modules" | while read -r m; do - echo "- $m" >> "$GITHUB_STEP_SUMMARY" - echo "- $m" >> "$comment_file" - done + fi + + echo "$reactor_modules" | while read -r m; do + [ -n "${GITHUB_STEP_SUMMARY:-}" ] && echo "- $m" >> "$GITHUB_STEP_SUMMARY" + echo "- $m" >> "$comment_file" + done + + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then echo "" >> "$GITHUB_STEP_SUMMARY" echo "
" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$comment_file" - echo "
" >> "$comment_file" fi + echo "" >> "$comment_file" + echo "
" >> "$comment_file" fi - elif [[ ${buildAll} = "true" ]] ; then - echo "" > "$comment_file" - echo ":information_source: CI did not run targeted module tests (all projects built or tests skipped)." >> "$comment_file" fi - if [[ ${ret} -ne 0 ]] ; then + # Write step summary header + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### Tested modules" + echo "" + for w in $(echo "$final_pl" | tr ',' '\n'); do + echo "- \`$w\`" + done + echo "" + } >> "$GITHUB_STEP_SUMMARY" + fi + + if [[ ${ret} -ne 0 ]]; then echo "Processing surefire and failsafe reports to create the summary" - echo -e "| Failed Test | Duration | Failure Type |\n| --- | --- | --- |" >> "$GITHUB_STEP_SUMMARY" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + echo -e "| Failed Test | Duration | Failure Type |\n| --- | --- | --- |" >> "$GITHUB_STEP_SUMMARY" + fi find . -path '*target/*-reports*' -iname '*.txt' -exec .github/actions/incremental-build/parse_errors.sh {} \; fi diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml index 95419d8c7d2e2..dcfc80711e733 100644 --- a/.github/workflows/main-build.yml +++ b/.github/workflows/main-build.yml @@ -68,7 +68,6 @@ jobs: - name: mvn test uses: ./.github/actions/incremental-build with: - mode: test github-token: ${{ secrets.GITHUB_TOKEN }} skip-mvnd-install: 'true' artifact-upload-suffix: main-java-${{ matrix.java }} diff --git a/.github/workflows/pr-build-main.yml b/.github/workflows/pr-build-main.yml index 539a1250555c8..27420a92fdc5e 100644 --- a/.github/workflows/pr-build-main.yml +++ b/.github/workflows/pr-build-main.yml @@ -21,6 +21,8 @@ on: pull_request: branches: - main + # CI-only changes don't need a full build. Use workflow_dispatch to + # test CI changes: gh workflow run "Build and test" -f pr_number=XXXX -f pr_ref=branch-name paths-ignore: - .github/** - README.md @@ -38,9 +40,14 @@ on: description: 'Git ref of the pull request branch' required: true type: string + extra_modules: + description: 'Additional modules to test (comma-separated paths, e.g. from /component-test)' + required: false + type: string + default: '' concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || inputs.pr_number || github.ref }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || inputs.pr_number || github.ref }}${{ inputs.extra_modules && '-component-test' || '' }} cancel-in-progress: true permissions: @@ -51,7 +58,6 @@ jobs: if: github.repository == 'apache/camel' permissions: contents: read - pull-requests: write runs-on: ubuntu-latest continue-on-error: ${{ matrix.experimental }} strategy: @@ -92,58 +98,31 @@ jobs: - name: mvn test uses: ./.github/actions/incremental-build with: - mode: test pr-id: ${{ github.event.number || inputs.pr_number }} github-token: ${{ secrets.GITHUB_TOKEN }} skip-mvnd-install: 'true' artifact-upload-suffix: java-${{ matrix.java }} - - name: Post CI test summary comment + extra-modules: ${{ inputs.extra_modules || '' }} + - name: Save PR number and test comment for commenter workflow if: always() && !matrix.experimental - uses: actions/github-script@v8 - with: - script: | - const fs = require('fs'); - const commentFile = 'incremental-test-comment.md'; - if (!fs.existsSync(commentFile)) return; - const body = fs.readFileSync(commentFile, 'utf8').trim(); - if (!body) return; - - const prNumber = ${{ github.event.number || inputs.pr_number || 0 }}; - if (!prNumber) { - core.warning('Could not determine PR number, skipping test summary comment'); - return; - } - - const marker = ''; - - try { - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - }); - const existing = comments.find(c => c.body && c.body.includes(marker)); - - if (existing) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existing.id, - body: body, - }); - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: body, - }); - } - } catch (error) { - core.warning(`Failed to post CI test summary comment: ${error.message}`); - } - - name: mvn test parent pom dependencies changed - uses: ./.github/actions/detect-dependencies + shell: bash + env: + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + mkdir -p ci-comment-artifact + prNumber="${{ github.event.number || inputs.pr_number }}" + echo "$prNumber" > ci-comment-artifact/pr-number + if [ -f incremental-test-comment.md ]; then + cp incremental-test-comment.md ci-comment-artifact/ + # Append link to the workflow run for detailed results + echo "" >> ci-comment-artifact/incremental-test-comment.md + echo "---" >> ci-comment-artifact/incremental-test-comment.md + echo ":gear: [View full build and test results](${RUN_URL})" >> ci-comment-artifact/incremental-test-comment.md + fi + - name: Upload CI comment artifact + if: always() && !matrix.experimental + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - github-token: ${{ secrets.GITHUB_TOKEN }} - base-ref: ${{ github.base_ref || 'main' }} + name: ci-comment + path: ci-comment-artifact/ + overwrite: true diff --git a/.github/workflows/pr-commenter.yml b/.github/workflows/pr-commenter.yml index 71a0cb190af5f..6f1cd620e73fd 100644 --- a/.github/workflows/pr-commenter.yml +++ b/.github/workflows/pr-commenter.yml @@ -95,7 +95,7 @@ jobs: * First-time contributors **require MANUAL approval** for the GitHub Actions to run * You can use the command \`/component-test (camel-)component-name1 (camel-)component-name2..\` to request a test from the test bot although they are normally detected and executed by CI. - * You can label PRs using \`build-all\`, \`build-dependents\`, \`skip-tests\` and \`test-dependents\` to fine-tune the checks executed by this PR. + * You can label PRs using \`skip-tests\` and \`test-dependents\` to fine-tune the checks executed by this PR. * Build and test logs are available in the summary page. **Only** [Apache Camel committers](https://camel.apache.org/community/team/#committers) have access to the summary. :warning: Be careful when sharing logs. Review their contents before sharing them publicly.` diff --git a/.github/workflows/pr-manual-component-test.yml b/.github/workflows/pr-manual-component-test.yml index 84a916379629a..2cc0c5ceca74a 100644 --- a/.github/workflows/pr-manual-component-test.yml +++ b/.github/workflows/pr-manual-component-test.yml @@ -28,20 +28,18 @@ jobs: name: PR comment if: ${{ github.repository == 'apache/camel' && github.event.issue.pull_request && (github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER' || github.event.comment.author_association == 'CONTRIBUTOR') && startsWith(github.event.comment.body, '/component-test') }} permissions: - pull-requests: write # to comment on a pull request - actions: read # to download artifact + pull-requests: write + actions: write # to dispatch workflows runs-on: ubuntu-latest - strategy: - matrix: - java: [ '21' ] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - submodules: recursive + sparse-checkout: components + sparse-checkout-cone-mode: true - name: Check Permission uses: actions-cool/check-user-permission@7b90a27f92f3961b368376107661682c441f6103 - - name: Retrieve sha + - name: Retrieve PR sha and ref id: pr env: PR_NUMBER: ${{ github.event.issue.number }} @@ -51,9 +49,8 @@ jobs: run: | pr="$(gh api /repos/${GH_REPO}/pulls/${PR_NUMBER})" head_sha="$(echo "$pr" | jq -r .head.sha)" + head_ref="$(echo "$pr" | jq -r .head.ref)" # Check that the PR branch was not pushed to after the comment was created. - # Use the head commit date (not head.repo.pushed_at which is repo-level - # and changes whenever any branch is pushed, causing false negatives). commit="$(gh api /repos/${GH_REPO}/commits/${head_sha})" committed_at="$(echo "$commit" | jq -r .commit.committer.date)" if [[ $(date -d "$committed_at" +%s) -gt $(date -d "$COMMENT_AT" +%s) ]]; then @@ -61,39 +58,55 @@ jobs: exit 1 fi echo "pr_sha=$head_sha" >> $GITHUB_OUTPUT + echo "pr_ref=$head_ref" >> $GITHUB_OUTPUT - name: React to comment env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} run: | gh api /repos/${GH_REPO}/issues/comments/${{ github.event.comment.id }}/reactions -f content="+1" - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: ${{ steps.pr.outputs.pr_sha }} - submodules: recursive - - id: install-packages - uses: ./.github/actions/install-packages - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 - with: - distribution: 'temurin' - java-version: ${{ matrix.java }} - cache: 'maven' - - id: test + - name: Resolve component paths and dispatch env: - comment_body: ${{ github.event.comment.body }} - name: Component test execution - uses: ./.github/actions/component-test - with: - run-id: ${{ github.run_id }} - pr-id: ${{ github.event.issue.number }} - comment-id: ${{ github.event.comment.id }} - comment-body: ${{ env.comment_body }} - artifact-upload-suffix: java-${{ matrix.java }} - - name: Post failure comment - if: failure() && steps.test.outcome != 'failure' - uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 - with: - issue-number: ${{ github.event.issue.number }} - body: | - :x: The `/component-test` run failed. Please [check the logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENT_BODY: ${{ github.event.comment.body }} + PR_NUMBER: ${{ github.event.issue.number }} + PR_REF: ${{ steps.pr.outputs.pr_ref }} + run: | + componentList="${COMMENT_BODY:16}" + if [[ -z "$componentList" ]]; then + echo "No components specified. Expected format: /component-test component1 component2..." + exit 1 + fi + + # Resolve component names to module paths + pl="" + for component in ${componentList}; do + if [[ ${component} = camel-* ]]; then + componentPath="components/${component}" + else + componentPath="components/camel-${component}" + fi + if [[ -d "${componentPath}" ]]; then + # Find all sub-modules (pom.xml dirs) under the component + modules=$(find "${componentPath}" -name pom.xml -not -path "*/src/it/*" -not -path "*/target/*" -exec dirname {} \; | sort | tr -s "\n" ",") + pl="${pl}${modules}" + else + echo "WARNING: Component path '${componentPath}' not found, skipping" + fi + done + + # Strip trailing comma + pl="${pl%,}" + + if [[ -z "$pl" ]]; then + echo "No valid component paths found" + exit 1 + fi + + echo "Dispatching main workflow with extra modules: $pl" + gh workflow run "Build and test" \ + --repo "${GH_REPO}" \ + -f pr_number="${PR_NUMBER}" \ + -f pr_ref="${PR_REF}" \ + -f extra_modules="${pl}" diff --git a/.github/workflows/pr-test-commenter.yml b/.github/workflows/pr-test-commenter.yml new file mode 100644 index 0000000000000..3462f12b3ebe5 --- /dev/null +++ b/.github/workflows/pr-test-commenter.yml @@ -0,0 +1,121 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Posts the CI test summary comment on PRs. +# Uses workflow_run trigger so it runs in the base repo context with +# full permissions — this allows posting comments on fork PRs too. + +name: Post CI test comment + +on: + workflow_run: + workflows: ["Build and test"] + types: + - completed + +jobs: + comment: + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' || + github.event.workflow_run.event == 'workflow_dispatch' + permissions: + pull-requests: write + actions: read + steps: + - name: Download CI comment artifact + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + const match = artifacts.data.artifacts.find(a => a.name === 'ci-comment'); + if (!match) { + core.info('No ci-comment artifact found, skipping'); + return; + } + const download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: match.id, + archive_format: 'zip', + }); + const fs = require('fs'); + fs.writeFileSync('${{ github.workspace }}/ci-comment.zip', Buffer.from(download.data)); + - name: Extract artifact + run: | + if [ -f ci-comment.zip ]; then + unzip -o ci-comment.zip -d ci-comment-artifact + fi + - name: Post or update PR comment + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const prFile = 'ci-comment-artifact/pr-number'; + const commentFile = 'ci-comment-artifact/incremental-test-comment.md'; + + if (!fs.existsSync(prFile)) { + core.info('No PR number file found, skipping'); + return; + } + const prNumber = parseInt(fs.readFileSync(prFile, 'utf8').trim(), 10); + if (!prNumber) { + core.warning('Invalid PR number, skipping'); + return; + } + + if (!fs.existsSync(commentFile)) { + core.info('No comment file found, skipping'); + return; + } + const body = fs.readFileSync(commentFile, 'utf8').trim(); + if (!body) return; + + const marker = ''; + + try { + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + }); + const existing = comments.find(c => c.body && c.body.includes(marker)); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body: body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: body, + }); + } + } catch (error) { + core.warning(`Failed to post CI test summary comment: ${error.message}`); + }