Merge branch 'main' into tyler.finethy/repro-retry-failure

Author: tylfin

.github/workflows/build_deploy.yml

@@ -93,7 +93,8 @@ jobs:
       - name: Build sdist
         run: |
           pip install "setuptools_scm[toml]>=4" "cython" "cmake>=3.24.2,<3.28" "setuptools-rust"
-          python setup.py sdist
+          # Disable cython extensions to avoid compiling .pyx files
+          DD_CYTHONIZE=0 python setup.py sdist
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: source-dist
@@ -110,6 +111,17 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
+
+      - name: Enable sccache
+        uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad  # 0.0.9
+
+      - name: Add addtional GHA cache-related env vars
+        uses: actions/github-script@v7
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL'])
+            core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL'])
+
       - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
           name: source-dist
@@ -127,6 +139,10 @@ jobs:
       - name: Install source package
         env:
           CMAKE_BUILD_PARALLEL_LEVEL: 12
+          CARGO_BUILD_JOBS: 12
+          SCCACHE_GHA_ENABLED: true
+          SCCACHE_CACHE_SIZE: 1G
+          DD_USE_SCCACHE: 1
         run: pip install dist/*.tar.gz
 
       - name: Test the source package

.github/workflows/build_python_3.yml

@@ -83,52 +83,26 @@ jobs:
       fail-fast: false
       matrix:
         include: ${{ fromJson(needs.build-wheels-matrix.outputs.include) }}
-    env:
-      SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE: ${{ needs.compute_version.outputs.library_version }}
-      CIBW_SKIP: ${{ inputs.cibw_skip }}
-      CIBW_PRERELEASE_PYTHONS: ${{ inputs.cibw_prerelease_pythons }}
-      CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
-      CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
-      CIBW_MUSLLINUX_I686_IMAGE: ghcr.io/datadog/dd-trace-py/pypa_musllinux_1_2_i686:latest
-      CIBW_BEFORE_ALL_WINDOWS: ${{ matrix.os == 'windows-latest' && 'rustup target add i686-pc-windows-msvc' || (matrix.os == 'windows-11-arm' && 'rustup target add aarch64-pc-windows-msvc') }}
-      CIBW_BEFORE_ALL_MACOS: rustup target add aarch64-apple-darwin
-      CIBW_BEFORE_ALL_LINUX: |
-        if [[ "$(uname -m)-$(uname -i)-$(uname -o | tr '[:upper:]' '[:lower:]')-$(ldd --version 2>&1 | head -n 1 | awk '{print $1}')" != "i686-unknown-linux-musl" ]]; then
-          curl -sSf https://sh.rustup.rs | sh -s -- -y;
-        fi
-      CIBW_ENVIRONMENT_LINUX: PATH=$HOME/.cargo/bin:$PATH CMAKE_BUILD_PARALLEL_LEVEL=24 CMAKE_ARGS="-DNATIVE_TESTING=OFF" SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
-      # SYSTEM_VERSION_COMPAT is a workaround for versioning issue, a.k.a.
-      # `platform.mac_ver()` reports incorrect MacOS version at 11.0
-      # See: https://stackoverflow.com/a/65402241
-      CIBW_ENVIRONMENT_MACOS: CMAKE_BUILD_PARALLEL_LEVEL=24 SYSTEM_VERSION_COMPAT=0 CMAKE_ARGS="-DNATIVE_TESTING=OFF" SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
-      CIBW_ENVIRONMENT_WINDOWS: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
-      # cibuildwheel repair will copy anything's under /output directory from the
-      # build container to the host machine. This is a bit hacky way, but seems
-      # to be the only way getting debug symbols out from the container while
-      # we don't mess up with RECORD file.
-      CIBW_REPAIR_WHEEL_COMMAND_LINUX: |
-        mkdir -p /output/debugwheelhouse &&
-        python scripts/extract_debug_symbols.py {wheel} --output-dir /output/debugwheelhouse &&
-        python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
-        mkdir ./tempwheelhouse &&
-        unzip -l {wheel} | grep '\.so' &&
-        auditwheel repair -w ./tempwheelhouse {wheel} &&
-        mv ./tempwheelhouse/*.whl {dest_dir} &&
-        rm -rf ./tempwheelhouse
-      CIBW_REPAIR_WHEEL_COMMAND_MACOS: |
-        mkdir -p ./debugwheelhouse &&
-        python scripts/extract_debug_symbols.py {wheel} --output-dir ./debugwheelhouse &&
-        python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
-        MACOSX_DEPLOYMENT_TARGET=12.7 delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
-      CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: python scripts/zip_filter.py "{wheel}" "*.c" "*.cpp" "*.cc" "*.h" "*.hpp" "*.pyx" "*.md" && mv "{wheel}" "{dest_dir}"
-      CIBW_TEST_COMMAND: "python {project}/tests/smoke_test.py"
-
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
           fetch-depth: 0
 
+      - name: Enable sccache
+        if: runner.os != 'Windows'
+        uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad  # 0.0.9
+        with:
+          disable_annotations: true
+
+      - name: Add addtional GHA cache-related env vars
+        uses: actions/github-script@v7
+        if: runner.os != 'Windows'
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL'])
+            core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL'])
+
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         name: Install Python
         with:
@@ -144,7 +118,76 @@ jobs:
         uses: pypa/cibuildwheel@c923d83ad9c1bc00211c5041d0c3f73294ff88f6 # v3.1.4
         with:
           only: ${{ matrix.only }}
-
+        env:
+          SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE: ${{ needs.compute_version.outputs.library_version }}
+          CIBW_SKIP: ${{ inputs.cibw_skip }}
+          CIBW_PRERELEASE_PYTHONS: ${{ inputs.cibw_prerelease_pythons }}
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
+          CIBW_MUSLLINUX_I686_IMAGE: ghcr.io/datadog/dd-trace-py/pypa_musllinux_1_2_i686:latest
+          CIBW_BEFORE_ALL_WINDOWS: ${{ matrix.os == 'windows-latest' && 'rustup target add i686-pc-windows-msvc' || (matrix.os == 'windows-11-arm' && 'rustup target add aarch64-pc-windows-msvc') }}
+          CIBW_BEFORE_ALL_MACOS: rustup target add aarch64-apple-darwin
+          CIBW_BEFORE_ALL_LINUX: |
+            if [[ "$(uname -m)-$(uname -i)-$(uname -o | tr '[:upper:]' '[:lower:]')-$(ldd --version 2>&1 | head -n 1 | awk '{print $1}')" != "i686-unknown-linux-musl" ]]; then
+              curl -sSf https://sh.rustup.rs | sh -s -- -y;
+            fi
+          CIBW_ENVIRONMENT_LINUX: >
+            PATH=$HOME/.cargo/bin:$PATH
+            CARGO_BUILD_JOBS=24
+            CMAKE_BUILD_PARALLEL_LEVEL=24
+            CMAKE_ARGS="-DNATIVE_TESTING=OFF"
+            SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
+            SCCACHE_GHA_ENABLED=true
+            SCCACHE_DIR=/host/${{ env.SCCACHE_DIR }}
+            SCCACHE_PATH=/host/${{ env.SCCACHE_PATH }}
+            SCCACHE_CACHE_SIZE=1G
+            ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
+            ACTIONS_RUNTIME_URL=${{ env.ACTIONS_RUNTIME_URL }}
+            ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
+            ACTIONS_CACHE_URL=${{ env.ACTIONS_CACHE_URL }}
+            ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
+            DD_USE_SCCACHE=1
+          # SYSTEM_VERSION_COMPAT is a workaround for versioning issue, a.k.a.
+          # `platform.mac_ver()` reports incorrect MacOS version at 11.0
+          # See: https://stackoverflow.com/a/65402241
+          CIBW_ENVIRONMENT_MACOS: >
+            CMAKE_BUILD_PARALLEL_LEVEL=24
+            CARGO_BUILD_JOBS=24
+            SYSTEM_VERSION_COMPAT=0
+            CMAKE_ARGS="-DNATIVE_TESTING=OFF"
+            SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
+            SCCACHE_GHA_ENABLED=true
+            SCCACHE_CACHE_SIZE=1G
+            ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
+            ACTIONS_RUNTIME_URL=${{ env.ACTIONS_RUNTIME_URL }}
+            ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
+            ACTIONS_CACHE_URL=${{ env.ACTIONS_CACHE_URL }}
+            ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
+            DD_USE_SCCACHE=1
+          CIBW_ENVIRONMENT_WINDOWS: >
+            SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
+          # cibuildwheel repair will copy anything's under /output directory from the
+          # build container to the host machine. This is a bit hacky way, but seems
+          # to be the only way getting debug symbols out from the container while
+          # we don't mess up with RECORD file.
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: |
+            mkdir -p /output/debugwheelhouse &&
+            python scripts/extract_debug_symbols.py {wheel} --output-dir /output/debugwheelhouse &&
+            python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
+            mkdir ./tempwheelhouse &&
+            unzip -l {wheel} | grep '\.so' &&
+            auditwheel repair -w ./tempwheelhouse {wheel} &&
+            mv ./tempwheelhouse/*.whl {dest_dir} &&
+            rm -rf ./tempwheelhouse
+          CIBW_REPAIR_WHEEL_COMMAND_MACOS: |
+            mkdir -p ./debugwheelhouse &&
+            python scripts/extract_debug_symbols.py {wheel} --output-dir ./debugwheelhouse &&
+            python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
+            MACOSX_DEPLOYMENT_TARGET=12.7 delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: python scripts/zip_filter.py "{wheel}" "*.c" "*.cpp" "*.cc" "*.h" "*.hpp" "*.pyx" "*.md" && mv "{wheel}" "{dest_dir}"
+          CIBW_BEFORE_TEST_LINUX: /host/${{ env.SCCACHE_PATH }} --show-stats
+          CIBW_BEFORE_TEST_MACOS: ${{ env.SCCACHE_PATH }} --show-stats
+          CIBW_TEST_COMMAND: "python {project}/tests/smoke_test.py"
       - name: Validate wheel RECORD files
         shell: bash
         run: |

.gitlab/benchmarks/macrobenchmarks.yml

@@ -17,6 +17,10 @@ candidate:
   image: $PACKAGE_IMAGE
   stage: build
   tags: [ "arch:amd64" ]
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+$/
+      interruptible: false
+    - interruptible: true
   needs:
     - pipeline: $PARENT_PIPELINE_ID
       job: download_ddtrace_artifacts
@@ -33,6 +37,10 @@ candidate:
   stage: test
   needs: [ "candidate" ]
   tags: ["runner:apm-k8s-same-cpu"]
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+$/
+      interruptible: false
+    - interruptible: true
   timeout: 1h
   retry:
     max: 2

.gitlab/benchmarks/microbenchmarks.yml

@@ -16,7 +16,10 @@ variables:
   when: on_success
   tags: ["runner:apm-k8s-tweaked-metal"]
   image: $MICROBENCHMARKS_CI_IMAGE
-  interruptible: true
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+$/
+      interruptible: false
+    - interruptible: true
   timeout: 30m
   dependencies: [ "baseline:build", "candidate" ]
   script: |

.riot/requirements/13e0d21.txt

@@ -0,0 +1,23 @@
+#
+# This file is autogenerated by pip-compile with Python 3.13
+# by the following command:
+#
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/13e0d21.in
+#
+aiokafka==0.9.0
+async-timeout==5.0.1
+attrs==25.4.0
+coverage[toml]==7.11.0
+hypothesis==6.45.0
+iniconfig==2.3.0
+mock==5.2.0
+opentracing==2.4.0
+packaging==25.0
+pluggy==1.6.0
+pygments==2.19.2
+pytest==8.4.2
+pytest-asyncio==1.2.0
+pytest-cov==7.0.0
+pytest-mock==3.15.1
+pytest-randomly==4.0.1
+sortedcontainers==2.4.0

.riot/requirements/16ebde6.txt

@@ -0,0 +1,24 @@
+#
+# This file is autogenerated by pip-compile with Python 3.13
+# by the following command:
+#
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/1c72bfb.in
+#
+aiokafka==0.12.0
+async-timeout==5.0.1
+attrs==25.4.0
+coverage[toml]==7.11.0
+hypothesis==6.45.0
+iniconfig==2.3.0
+mock==5.2.0
+opentracing==2.4.0
+packaging==25.0
+pluggy==1.6.0
+pygments==2.19.2
+pytest==8.4.2
+pytest-asyncio==1.2.0
+pytest-cov==7.0.0
+pytest-mock==3.15.1
+pytest-randomly==4.0.1
+sortedcontainers==2.4.0
+typing-extensions==4.15.0

.riot/requirements/1c72bfb.txt

@@ -0,0 +1,23 @@
+#
+# This file is autogenerated by pip-compile with Python 3.14
+# by the following command:
+#
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/1ded764.in
+#
+aiokafka==0.9.0
+async-timeout==5.0.1
+attrs==25.4.0
+coverage[toml]==7.11.0
+hypothesis==6.45.0
+iniconfig==2.3.0
+mock==5.2.0
+opentracing==2.4.0
+packaging==25.0
+pluggy==1.6.0
+pygments==2.19.2
+pytest==8.4.2
+pytest-asyncio==1.2.0
+pytest-cov==7.0.0
+pytest-mock==3.15.1
+pytest-randomly==4.0.1
+sortedcontainers==2.4.0

.riot/requirements/1ded764.txt

@@ -0,0 +1,24 @@
+#
+# This file is autogenerated by pip-compile with Python 3.12
+# by the following command:
+#
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/329b0ed.in
+#
+aiokafka==0.9.0
+async-timeout==5.0.1
+attrs==25.4.0
+coverage[toml]==7.11.0
+hypothesis==6.45.0
+iniconfig==2.3.0
+mock==5.2.0
+opentracing==2.4.0
+packaging==25.0
+pluggy==1.6.0
+pygments==2.19.2
+pytest==8.4.2
+pytest-asyncio==1.2.0
+pytest-cov==7.0.0
+pytest-mock==3.15.1
+pytest-randomly==4.0.1
+sortedcontainers==2.4.0
+typing-extensions==4.15.0

.riot/requirements/329b0ed.txt

@@ -0,0 +1,24 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/4532043.in
+#
+aiokafka==0.12.0
+async-timeout==5.0.1
+attrs==25.4.0
+coverage[toml]==7.11.0
+hypothesis==6.45.0
+iniconfig==2.3.0
+mock==5.2.0
+opentracing==2.4.0
+packaging==25.0
+pluggy==1.6.0
+pygments==2.19.2
+pytest==8.4.2
+pytest-asyncio==1.2.0
+pytest-cov==7.0.0
+pytest-mock==3.15.1
+pytest-randomly==4.0.1
+sortedcontainers==2.4.0
+typing-extensions==4.15.0