From b7c0f0b121a5558f743b04bf08bfef61a31e78e5 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Sep 2026 12:49:25 -0500 Subject: [PATCH 1/2] Test shared Maven workflow --- .github/workflows/build.yaml | 51 +-- .github/workflows/pr.yaml | 797 ++--------------------------------- ci/build_java.sh | 8 +- ci/build_lucene.sh | 18 +- 4 files changed, 66 insertions(+), 808 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bdd46b298f..3ebae27587 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -155,17 +155,8 @@ jobs: node_type: "gpu-l4-latest-1" script: "ci/build_go.sh" sha: ${{ inputs.sha }} - java-build-matrix: - needs: cpp-build - permissions: - contents: read - uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main - with: - build_type: pull-request - matrix_name: conda-cpp-build - matrix_filter: map(select(.ARCH == "amd64")) java-build: - needs: java-build-matrix + needs: cpp-build permissions: actions: read contents: read @@ -173,25 +164,22 @@ jobs: packages: read pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. - # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }} + uses: rapidsai/shared-workflows/.github/workflows/maven-build.yaml@1fc50a1789a34a4e7b67ae23af71fc041067a1af with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} - arch: "amd64" date: ${{ inputs.date }} - container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" - script: "ci/build_java.sh" - artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}" - file_to_upload: "java/cuvs-java/target/" sha: ${{ inputs.sha }} + matrix-build-type: pull-request + matrix-filter: map(select(.ARCH == "amd64")) + script: ci/build_java.sh + file-to-upload: java/cuvs-java/target/ + artifact-name-prefix: cuvs_java + container-image-repository: rapidsai/ci-conda + container-image-version: '26.10' lucene-build: # Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against. - needs: [java-build-matrix, java-build] + needs: java-build permissions: actions: read contents: read @@ -199,22 +187,19 @@ jobs: packages: read pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. - # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }} + uses: rapidsai/shared-workflows/.github/workflows/maven-build.yaml@1fc50a1789a34a4e7b67ae23af71fc041067a1af with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} - arch: "amd64" date: ${{ inputs.date }} - container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" - script: "ci/build_lucene.sh cuvs-java-cuda${{ matrix.CUDA_VER }}" - artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}" - file_to_upload: "java/cuvs-lucene/target/" sha: ${{ inputs.sha }} + matrix-build-type: pull-request + matrix-filter: map(select(.ARCH == "amd64")) + script: ci/build_lucene.sh + file-to-upload: java/cuvs-lucene/target/ + artifact-name-prefix: cuvs_lucene + container-image-repository: rapidsai/ci-conda + container-image-version: '26.10' python-build: needs: [build-details, cpp-build] permissions: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d6f7cd7d7e..eb1d029fe3 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -1,467 +1,33 @@ name: pr + on: push: branches: - "pull-request/[0-9]+" + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true + permissions: {} + jobs: pr-builder: needs: - - build-details - - check-nightly-ci - - changed-files - - checks - - check-c-abi - - conda-cpp-build - - conda-cpp-tests - - conda-cpp-checks - - conda-python-build - - conda-python-tests - - rocky8-clib-standalone-build-matrix - - rocky8-clib-standalone-build - - rocky8-clib-tests-matrix - - rocky8-clib-tests - - conda-java-build-and-tests-matrix - - conda-java-build-and-tests - - conda-lucene-build-and-tests - - rust-build-matrix - - rust-build - - go-build-matrix - - go-build - - docs-build - - wheel-build-libcuvs - - wheel-build-cuvs - - wheel-tests-cuvs - - devcontainer - - telemetry-setup + - java-build + - lucene-build uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main permissions: contents: read if: always() with: needs: ${{ toJSON(needs) }} - telemetry-setup: - continue-on-error: true - runs-on: ubuntu-latest - permissions: - contents: read - env: - OTEL_SERVICE_NAME: 'pr-cuvs' - steps: - - name: Telemetry setup - if: ${{ vars.TELEMETRY_ENABLED == 'true' }} - uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main - build-details: - uses: rapidsai/shared-workflows/.github/workflows/compute-build-details.yaml@main - check-nightly-ci: - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - id-token: write - pull-requests: read - env: - GH_TOKEN: ${{ github.token }} - steps: - - name: Get PR Info - id: get-pr-info - uses: nv-gha-runners/get-pr-info@090577647b8ddc4e06e809e264f7881650ecdccf # main - - name: Check if nightly CI is passing - uses: rapidsai/shared-actions/check_nightly_success/dispatch@main - with: - repo: ${{ github.repository }} - target-branch: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.ref }} - max-days-without-success: 30 - changed-files: - needs: telemetry-setup - permissions: - actions: read - contents: read - packages: read - pull-requests: read - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@main - with: - files_yaml: | - build_docs: - - '**' - - '!**/*/REVIEW_GUIDELINES.md' - - '!.coderabbit.yaml' - - '!.devcontainer/**' - - '!.github/CODEOWNERS' - - '!.github/ISSUE_TEMPLATE/**' - - '!.github/PULL_REQUEST_TEMPLATE.md' - - '!.github/copy-pr-bot.yaml' - - '!.github/labeler.yml' - - '!.github/ops-bot.yaml' - - '!.github/release.yml' - - '!.github/workflows/build.yaml' - - '!.github/workflows/check-c-abi.yaml' - - '!.github/workflows/labeler.yml' - - '!.github/workflows/publish-rust.yaml' - - '!.github/workflows/store-c-abi-baseline.yaml' - - '!.github/workflows/test.yaml' - - '!.github/workflows/trigger-breaking-change-alert.yaml' - - '!.github/workflows/update-c-abi-baseline.yaml' - - '!.pre-commit-config.yaml' - - '!.shellcheckrc' - - '!.yamllint.yaml' - - '!Dockerfile' - - '!SECURITY.md' - - '!ci/build_wheel*.sh' - - '!ci/check_style.sh' - - '!ci/release/update-version.sh' - - '!ci/run*.sh' - - '!ci/test*.sh' - - '!ci/validate_wheel.sh' - - '!thirdparty/LICENSES/**' - test_cpp: - - '**' - - '!**/*/REVIEW_GUIDELINES.md' - - '!.coderabbit.yaml' - - '!.devcontainer/**' - - '!.github/CODEOWNERS' - - '!.github/ISSUE_TEMPLATE/**' - - '!.github/PULL_REQUEST_TEMPLATE.md' - - '!.github/copy-pr-bot.yaml' - - '!.github/labeler.yml' - - '!.github/ops-bot.yaml' - - '!.github/release.yml' - - '!.github/workflows/build.yaml' - - '!.github/workflows/check-c-abi.yaml' - - '!.github/workflows/labeler.yml' - - '!.github/workflows/publish-rust.yaml' - - '!.github/workflows/store-c-abi-baseline.yaml' - - '!.github/workflows/test.yaml' - - '!.github/workflows/trigger-breaking-change-alert.yaml' - - '!.github/workflows/update-c-abi-baseline.yaml' - - '!.pre-commit-config.yaml' - - '!.shellcheckrc' - - '!.yamllint.yaml' - - '!Dockerfile' - - '!README.md' - - '!SECURITY.md' - - '!ci/build_docs.sh' - - '!ci/build_go.sh' - - '!ci/build_java.sh' - - '!ci/build_lucene.sh' - - '!ci/build_python.sh' - - '!ci/build_rust.sh' - - '!ci/build_wheel*.sh' - - '!ci/check_style.sh' - - '!ci/release/update-version.sh' - - '!ci/run_cuvs_pytests.sh' - - '!ci/test_java.sh' - - '!ci/test_lucene.sh' - - '!ci/test_python.sh' - - '!ci/test_wheel_cuvs.sh' - - '!ci/validate_wheel.sh' - - '!docs/**' - - '!fern/**' - - '!go/**' - - '!img/**' - - '!java/**' - - '!notebooks/**' - - '!python/**' - - '!rust/**' - - '!thirdparty/LICENSES/**' - test_java: - - '**' - - '!**/*/REVIEW_GUIDELINES.md' - - '!.coderabbit.yaml' - - '!.devcontainer/**' - - '!.github/CODEOWNERS' - - '!.github/ISSUE_TEMPLATE/**' - - '!.github/PULL_REQUEST_TEMPLATE.md' - - '!.github/copy-pr-bot.yaml' - - '!.github/labeler.yml' - - '!.github/ops-bot.yaml' - - '!.github/release.yml' - - '!.github/workflows/build.yaml' - - '!.github/workflows/check-c-abi.yaml' - - '!.github/workflows/labeler.yml' - - '!.github/workflows/publish-rust.yaml' - - '!.github/workflows/store-c-abi-baseline.yaml' - - '!.github/workflows/test.yaml' - - '!.github/workflows/trigger-breaking-change-alert.yaml' - - '!.github/workflows/update-c-abi-baseline.yaml' - - '!.pre-commit-config.yaml' - - '!.shellcheckrc' - - '!.yamllint.yaml' - - '!CODEOWNERS' - - '!Dockerfile' - - '!README.md' - - '!SECURITY.md' - - '!ci/build_docs.sh' - - '!ci/build_go.sh' - - '!ci/build_python.sh' - - '!ci/build_rust.sh' - - '!ci/build_wheel*.sh' - - '!ci/checkstyle.sh' - - '!ci/run_cuvs_pytests.sh' - - '!ci/test_cpp.sh' - - '!ci/test_python.sh' - - '!ci/test_wheel_cuvs.sh' - - '!ci/validate_wheel.sh' - - '!conda/environments/**' - - '!conda/recipes/cuvs/**' - - '!conda/recipes/cuvs-bench/**' - - '!conda/recipes/cuvs-bench-cpu/**' - - '!docs/**' - - '!examples/**' - - '!fern/**' - - '!go/**' - - '!img/**' - - '!notebooks/**' - - '!pyproject.toml' - - '!python/**' - - '!rust/**' - - '!thirdparty/LICENSES/**' - test_python_conda: - - '**' - - '!**/*/REVIEW_GUIDELINES.md' - - '!.coderabbit.yaml' - - '!.devcontainer/**' - - '!.github/CODEOWNERS' - - '!.github/ISSUE_TEMPLATE/**' - - '!.github/PULL_REQUEST_TEMPLATE.md' - - '!.github/copy-pr-bot.yaml' - - '!.github/labeler.yml' - - '!.github/ops-bot.yaml' - - '!.github/release.yml' - - '!.github/workflows/build.yaml' - - '!.github/workflows/check-c-abi.yaml' - - '!.github/workflows/labeler.yml' - - '!.github/workflows/publish-rust.yaml' - - '!.github/workflows/store-c-abi-baseline.yaml' - - '!.github/workflows/test.yaml' - - '!.github/workflows/trigger-breaking-change-alert.yaml' - - '!.github/workflows/update-c-abi-baseline.yaml' - - '!.pre-commit-config.yaml' - - '!.shellcheckrc' - - '!.yamllint.yaml' - - '!ci/build_go.sh' - - '!ci/build_java.sh' - - '!ci/build_lucene.sh' - - '!ci/build_rust.sh' - - '!ci/check_style.sh' - - '!ci/release/update-version.sh' - - '!ci/run_ctests.sh' - - '!ci/test_cpp.sh' - - '!ci/test_java.sh' - - '!ci/test_lucene.sh' - - '!ci/test_standalone_c.sh' - - '!ci/test_wheel_cuvs.sh' - - '!ci/validate_wheel.sh' - - '!cpp/.clang-format' - - '!cpp/.clang-tidy' - - '!cpp/doxygen/**' - - '!CODEOWNERS' - - '!Dockerfile' - - '!README.md' - - '!SECURITY.md' - - '!docs/**' - - '!fern/**' - - '!go/**' - - '!img/**' - - '!java/**' - - '!notebooks/**' - - '!rust/**' - - '!thirdparty/LICENSES/**' - test_python_wheels: - - '**' - - '!**/*/REVIEW_GUIDELINES.md' - - '!.coderabbit.yaml' - - '!.devcontainer/**' - - '!.github/CODEOWNERS' - - '!.github/ISSUE_TEMPLATE/**' - - '!.github/PULL_REQUEST_TEMPLATE.md' - - '!.github/copy-pr-bot.yaml' - - '!.github/labeler.yml' - - '!.github/ops-bot.yaml' - - '!.github/release.yml' - - '!.github/workflows/build.yaml' - - '!.github/workflows/check-c-abi.yaml' - - '!.github/workflows/labeler.yml' - - '!.github/workflows/publish-rust.yaml' - - '!.github/workflows/store-c-abi-baseline.yaml' - - '!.github/workflows/test.yaml' - - '!.github/workflows/trigger-breaking-change-alert.yaml' - - '!.github/workflows/update-c-abi-baseline.yaml' - - '!.pre-commit-config.yaml' - - '!.shellcheckrc' - - '!.yamllint.yaml' - - '!ci/build_cpp.sh' - - '!ci/build_docs.sh' - - '!ci/build_go.sh' - - '!ci/build_java.sh' - - '!ci/build_lucene.sh' - - '!ci/build_python.sh' - - '!ci/build_rust.sh' - - '!ci/build_standalone_c.sh' - - '!ci/check_style.sh' - - '!ci/release/update-version.sh' - - '!ci/run_ctests.sh' - - '!ci/test_cpp.sh' - - '!ci/test_java.sh' - - '!ci/test_lucene.sh' - - '!ci/test_python.sh' - - '!ci/test_standalone_c.sh' - - '!ci/validate_wheel.sh' - - '!CODEOWNERS' - - '!Dockerfile' - - '!README.md' - - '!SECURITY.md' - - '!conda/**' - - '!cpp/.clang-format' - - '!cpp/.clang-tidy' - - '!cpp/doxygen/**' - - '!docs/**' - - '!fern/**' - - '!java/**' - - '!go/**' - - '!img/**' - - '!notebooks/**' - - '!rust/**' - - '!thirdparty/LICENSES/**' - test_rust: - - '**' - - '!**/*/REVIEW_GUIDELINES.md' - - '!.coderabbit.yaml' - - '!.devcontainer/**' - - '!.github/CODEOWNERS' - - '!.github/ISSUE_TEMPLATE/**' - - '!.github/PULL_REQUEST_TEMPLATE.md' - - '!.github/copy-pr-bot.yaml' - - '!.github/labeler.yml' - - '!.github/ops-bot.yaml' - - '!.github/release.yml' - - '!.github/workflows/build.yaml' - - '!.github/workflows/check-c-abi.yaml' - - '!.github/workflows/labeler.yml' - - '!.github/workflows/publish-rust.yaml' - - '!.github/workflows/store-c-abi-baseline.yaml' - - '!.github/workflows/test.yaml' - - '!.github/workflows/trigger-breaking-change-alert.yaml' - - '!.github/workflows/update-c-abi-baseline.yaml' - - '!.pre-commit-config.yaml' - - '!.shellcheckrc' - - '!.yamllint.yaml' - - '!CODEOWNERS' - - '!Dockerfile' - - '!README.md' - - '!SECURITY.md' - - '!ci/build_docs.sh' - - '!ci/build_go.sh' - - '!ci/build_java.sh' - - '!ci/build_lucene.sh' - - '!ci/build_python.sh' - - '!ci/build_wheel*.sh' - - '!ci/check_style.sh' - - '!ci/release/update-version.sh' - - '!ci/run_ctests.sh' - - '!ci/run_cuvs_pytests.sh' - - '!ci/test_cpp.sh' - - '!ci/test_java.sh' - - '!ci/test_lucene.sh' - - '!ci/test_python.sh' - - '!ci/test_wheel_cuvs.sh' - - '!ci/validate_wheel.sh' - - '!docs/**' - - '!fern/**' - - '!go/**' - - '!img/**' - - '!java/**' - - '!notebooks/**' - - '!python/**' - - '!thirdparty/LICENSES/**' - test_go: - - '**' - - '!**/*/REVIEW_GUIDELINES.md' - - '!.coderabbit.yaml' - - '!.devcontainer/**' - - '!.github/CODEOWNERS' - - '!.github/ISSUE_TEMPLATE/**' - - '!.github/PULL_REQUEST_TEMPLATE.md' - - '!.github/copy-pr-bot.yaml' - - '!.github/labeler.yml' - - '!.github/ops-bot.yaml' - - '!.github/release.yml' - - '!.github/workflows/build.yaml' - - '!.github/workflows/check-c-abi.yaml' - - '!.github/workflows/labeler.yml' - - '!.github/workflows/publish-rust.yaml' - - '!.github/workflows/store-c-abi-baseline.yaml' - - '!.github/workflows/test.yaml' - - '!.github/workflows/trigger-breaking-change-alert.yaml' - - '!.github/workflows/update-c-abi-baseline.yaml' - - '!.pre-commit-config.yaml' - - '!.shellcheckrc' - - '!.yamllint.yaml' - - '!CODEOWNERS' - - '!Dockerfile' - - '!README.md' - - '!SECURITY.md' - - '!ci/build_docs.sh' - - '!ci/build_java.sh' - - '!ci/build_lucene.sh' - - '!ci/build_python.sh' - - '!ci/build_rust.sh' - - '!ci/build_wheel*.sh' - - '!ci/check_style.sh' - - '!ci/release/update-version.sh' - - '!ci/run_ctests.sh' - - '!ci/run_cuvs_pytests.sh' - - '!ci/test_cpp.sh' - - '!ci/test_java.sh' - - '!ci/test_lucene.sh' - - '!ci/test_python.sh' - - '!ci/test_standalone_c.sh' - - '!ci/test_wheel_cuvs.sh' - - '!ci/validate_wheel.sh' - - '!docs/**' - - '!fern/**' - - '!img/**' - - '!java/**' - - '!notebooks/**' - - '!python/**' - - '!rust/**' - - '!thirdparty/LICENSES/**' - checks: - needs: telemetry-setup - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main - permissions: - contents: read - with: - enable_check_generated_files: false - ignored_pr_jobs: "telemetry-summarize" - check-c-abi: - needs: telemetry-setup - permissions: - actions: read - contents: read - issues: write - pull-requests: read - uses: ./.github/workflows/check-c-abi.yaml - conda-cpp-build: - needs: [build-details, checks] - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@main - with: - build_type: pull-request - build-datetime: ${{ needs.build-details.outputs.build-datetime }} - node_type: cpu16 - script: ci/build_cpp.sh - conda-cpp-tests: - needs: [conda-cpp-build, changed-files] + + # This draft deliberately resolves the published libcuvs package so the + # shared Maven workflow can be tested on CPU runners without rebuilding C++ + # or Python. GPU tests remain separate because the current test scripts also + # rebuild the JARs; see https://github.com/rapidsai/cuvs/issues/868. + java-build: permissions: actions: read contents: read @@ -469,26 +35,18 @@ jobs: packages: read pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + uses: rapidsai/shared-workflows/.github/workflows/maven-build.yaml@1fc50a1789a34a4e7b67ae23af71fc041067a1af with: build_type: pull-request - script: ci/test_cpp.sh - conda-cpp-checks: - needs: conda-cpp-build - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@main - with: - build_type: pull-request - symbol_exclusions: (void (thrust::|cub::)) - package_name: libcuvs - conda-python-build: - needs: [build-details, conda-cpp-build] + matrix-filter: map(select(.ARCH == "amd64")) + script: RAPIDS_USE_PUBLISHED_LIBCUVS=true ci/build_java.sh + file-to-upload: java/cuvs-java/target/ + artifact-name-prefix: cuvs_java + container-image-repository: rapidsai/ci-conda + container-image-version: '26.10' + + lucene-build: + needs: java-build permissions: actions: read contents: read @@ -496,309 +54,12 @@ jobs: packages: read pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/maven-build.yaml@1fc50a1789a34a4e7b67ae23af71fc041067a1af with: build_type: pull-request - build-datetime: ${{ needs.build-details.outputs.build-datetime }} - script: ci/build_python.sh - # Build a conda package for each CUDA x ARCH x minimum supported Python version - matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber))) - conda-python-tests: - needs: [conda-python-build, changed-files] - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_conda - with: - build_type: pull-request - script: ci/test_python.sh - rocky8-clib-standalone-build-matrix: - needs: checks - permissions: - contents: read - uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main - with: - build_type: pull-request - matrix_name: conda-cpp-build - rocky8-clib-standalone-build: - needs: [build-details, rocky8-clib-standalone-build-matrix] - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.rocky8-clib-standalone-build-matrix.outputs.matrix) }} - with: - build_type: pull-request - build-datetime: ${{ needs.build-details.outputs.build-datetime }} - arch: "${{matrix.arch}}" - date: ${{ inputs.date }}_c - container_image: "rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" - node_type: "cpu16" - requires_license_builder: true - script: "ci/build_standalone_c.sh --build-tests" - artifact-name: "libcuvs_c_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }}.tar.gz" - file_to_upload: "libcuvs_c.tar.gz" - sha: ${{ inputs.sha }} - rocky8-clib-tests-matrix: - needs: [rocky8-clib-standalone-build, changed-files] - permissions: - contents: read - uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - with: - build_type: pull-request - matrix_name: conda-cpp-build - rocky8-clib-tests: - needs: rocky8-clib-tests-matrix - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.rocky8-clib-tests-matrix.outputs.matrix) }} - with: - build_type: pull-request - node_type: "gpu-l4-latest-1" - arch: "${{matrix.arch}}" - date: ${{ inputs.date }}_c - container_image: "rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" - script: "ci/test_standalone_c.sh libcuvs_c_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }}.tar.gz" - sha: ${{ inputs.sha }} - conda-java-build-and-tests-matrix: - needs: [conda-cpp-build, changed-files] - permissions: - contents: read - uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java || fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - with: - build_type: pull-request - matrix_name: conda-cpp-build - matrix_filter: map(select(.ARCH == "amd64")) - conda-java-build-and-tests: - needs: conda-java-build-and-tests-matrix - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. - # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.conda-java-build-and-tests-matrix.outputs.matrix) }} - with: - build_type: pull-request - node_type: "gpu-l4-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" - script: "ci/test_java.sh" - artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}" - file_to_upload: "java/cuvs-java/target/" - conda-lucene-build-and-tests: - # Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against, and - # reuses its matrix so the two always agree on the set of CUDA versions. - needs: [conda-java-build-and-tests-matrix, conda-java-build-and-tests] - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. - # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.conda-java-build-and-tests-matrix.outputs.matrix) }} - with: - build_type: pull-request - node_type: "gpu-l4-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" - script: "ci/test_lucene.sh cuvs-java-cuda${{ matrix.CUDA_VER }}" - artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}" - file_to_upload: "java/cuvs-lucene/target/" - rust-build-matrix: - needs: [conda-cpp-build, changed-files] - permissions: - contents: read - uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_rust || fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - with: - build_type: pull-request - matrix_name: conda-cpp-build - matrix_filter: map(select(.ARCH == "amd64")) - rust-build: - needs: rust-build-matrix - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. - # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.rust-build-matrix.outputs.matrix) }} - with: - build_type: pull-request - node_type: "gpu-l4-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" - script: "ci/build_rust.sh" - go-build-matrix: - needs: [conda-cpp-build, changed-files] - permissions: - contents: read - uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_go || fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - with: - build_type: pull-request - matrix_name: conda-cpp-build - matrix_filter: map(select(.ARCH == "amd64")) - go-build: - needs: go-build-matrix - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. - # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.go-build-matrix.outputs.matrix) }} - with: - build_type: pull-request - node_type: "gpu-l4-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" - script: "ci/build_go.sh" - docs-build: - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - with: - build_type: pull-request - node_type: "gpu-l4-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:26.10-latest" - script: "ci/build_docs.sh" - wheel-build-libcuvs: - needs: [build-details, checks] - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main - with: - build_type: pull-request - build-datetime: ${{ needs.build-details.outputs.build-datetime }} - node_type: cpu16 - script: ci/build_wheel_libcuvs.sh - # build for every combination of arch and CUDA version, but only for the latest Python - matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) - package-name: libcuvs - package-type: cpp - wheel-build-cuvs: - needs: [build-details, wheel-build-libcuvs] - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main - with: - build_type: pull-request - build-datetime: ${{ needs.build-details.outputs.build-datetime }} - node_type: cpu8 - script: ci/build_wheel_cuvs.sh - package-name: cuvs - package-type: python - # Build a wheel for each CUDA x ARCH x minimum supported Python version - matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber))) - wheel-tests-cuvs: - needs: [wheel-build-cuvs, changed-files] - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels - with: - build_type: pull-request - script: ci/test_wheel_cuvs.sh - devcontainer: - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - secrets: inherit # zizmor: ignore[secrets-inherit] - needs: telemetry-setup - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@main - with: - arch: '["amd64", "arm64"]' - cuda: '["13.3"]' - node_type: "cpu8" - env: | - SCCACHE_DIST_MAX_RETRIES=inf - SCCACHE_SERVER_LOG=sccache=debug - SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false - build_command: | - sccache --zero-stats; - build-all -j0 --verbose 2>&1 | tee telemetry-artifacts/build.log; - sccache --show-adv-stats | tee telemetry-artifacts/sccache-stats.txt; - telemetry-summarize: - # This job must use a self-hosted runner to record telemetry traces. - runs-on: linux-amd64-cpu4 - needs: pr-builder - permissions: - actions: write - if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }} - continue-on-error: true - steps: - - name: Telemetry summarize - uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main - env: - GH_TOKEN: ${{ github.token }} + matrix-filter: map(select(.ARCH == "amd64")) + script: RAPIDS_USE_PUBLISHED_LIBCUVS=true ci/build_lucene.sh + file-to-upload: java/cuvs-lucene/target/ + artifact-name-prefix: cuvs_lucene + container-image-repository: rapidsai/ci-conda + container-image-version: '26.10' diff --git a/ci/build_java.sh b/ci/build_java.sh index 73b5529e07..01f24354a7 100755 --- a/ci/build_java.sh +++ b/ci/build_java.sh @@ -26,7 +26,11 @@ rapids-logger "Configuring conda strict channel priority" conda config --set channel_priority strict rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcuvs cuvs --cuda "$RAPIDS_CUDA_VERSION")") +CPP_CHANNEL_ARGS=() +if [[ "${RAPIDS_USE_PUBLISHED_LIBCUVS:-false}" != "true" ]]; then + CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcuvs cuvs --cuda "$RAPIDS_CUDA_VERSION")") + CPP_CHANNEL_ARGS=(--prepend-channel "${CPP_CHANNEL}") +fi rapids-logger "Generate Java testing dependencies" @@ -35,7 +39,7 @@ ENV_YAML_DIR="$(mktemp -d)" rapids-dependency-file-generator \ --output conda \ --file-key java \ - --prepend-channel "${CPP_CHANNEL}" \ + "${CPP_CHANNEL_ARGS[@]}" \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n java diff --git a/ci/build_lucene.sh b/ci/build_lucene.sh index 266e6ccd20..1d9ce8b81b 100755 --- a/ci/build_lucene.sh +++ b/ci/build_lucene.sh @@ -18,9 +18,13 @@ for arg in "$@"; do esac done -if [ -z "${CUVS_JAVA_ARTIFACT}" ]; then - echo "Error: name of the cuvs-java artifact is missing" >&2 - exit 1 +if [[ -z "${CUVS_JAVA_ARTIFACT}" ]]; then + case "$(arch)" in + x86_64) ARTIFACT_ARCH=amd64 ;; + aarch64) ARTIFACT_ARCH=arm64 ;; + *) echo "Unsupported architecture: $(arch)" >&2; exit 1 ;; + esac + CUVS_JAVA_ARTIFACT="cuvs_java_${ARTIFACT_ARCH}_cu${RAPIDS_CUDA_VERSION%%.*}" fi if [ -e "/opt/conda/etc/profile.d/conda.sh" ]; then @@ -31,7 +35,11 @@ rapids-logger "Configuring conda strict channel priority" conda config --set channel_priority strict rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcuvs cuvs --cuda "$RAPIDS_CUDA_VERSION")") +CPP_CHANNEL_ARGS=() +if [[ "${RAPIDS_USE_PUBLISHED_LIBCUVS:-false}" != "true" ]]; then + CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcuvs cuvs --cuda "$RAPIDS_CUDA_VERSION")") + CPP_CHANNEL_ARGS=(--prepend-channel "${CPP_CHANNEL}") +fi CUVS_JAVA_DIR=$(rapids-download-from-github "${CUVS_JAVA_ARTIFACT}") rapids-logger "Generate Java testing dependencies" @@ -41,7 +49,7 @@ ENV_YAML_DIR="$(mktemp -d)" rapids-dependency-file-generator \ --output conda \ --file-key java \ - --prepend-channel "${CPP_CHANNEL}" \ + "${CPP_CHANNEL_ARGS[@]}" \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n java From 3a76e60f4ea1302cd24810db3f0752f33b021dd6 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Sep 2026 13:12:30 -0500 Subject: [PATCH 2/2] Fix published libcuvs environment override --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index eb1d029fe3..7113651cb1 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -39,7 +39,7 @@ jobs: with: build_type: pull-request matrix-filter: map(select(.ARCH == "amd64")) - script: RAPIDS_USE_PUBLISHED_LIBCUVS=true ci/build_java.sh + script: env RAPIDS_USE_PUBLISHED_LIBCUVS=true ci/build_java.sh file-to-upload: java/cuvs-java/target/ artifact-name-prefix: cuvs_java container-image-repository: rapidsai/ci-conda @@ -58,7 +58,7 @@ jobs: with: build_type: pull-request matrix-filter: map(select(.ARCH == "amd64")) - script: RAPIDS_USE_PUBLISHED_LIBCUVS=true ci/build_lucene.sh + script: env RAPIDS_USE_PUBLISHED_LIBCUVS=true ci/build_lucene.sh file-to-upload: java/cuvs-lucene/target/ artifact-name-prefix: cuvs_lucene container-image-repository: rapidsai/ci-conda