From 59d5849586844bed76372a28eefaefd85d868e94 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Thu, 13 Nov 2025 09:21:18 -0500 Subject: [PATCH 1/4] ci: only build baseline if we cannot find a wheel anywhere for it --- .gitlab/benchmarks/steps/build-baseline.sh | 42 +++++++++++++++------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/.gitlab/benchmarks/steps/build-baseline.sh b/.gitlab/benchmarks/steps/build-baseline.sh index 8ed61992164..becfceb0966 100755 --- a/.gitlab/benchmarks/steps/build-baseline.sh +++ b/.gitlab/benchmarks/steps/build-baseline.sh @@ -2,20 +2,36 @@ set -e -o pipefail # If we have a tag (e.g. v2.21.1), then use the PyPI published wheel -# Otherwise, build the wheel from commit hash if [[ -n "${BASELINE_TAG}" ]]; then python3.9 -m pip download --no-deps "ddtrace==${BASELINE_TAG:1}" -else - ulimit -c unlimited - curl -sSf https://sh.rustup.rs | sh -s -- -y; - export PATH="$HOME/.cargo/bin:$PATH" - echo "Building wheel for ${BASELINE_BRANCH}:${BASELINE_COMMIT_SHA}" - git checkout "${BASELINE_COMMIT_SHA}" - mkdir ./tmp - PYO3_PYTHON=python3.9 CIBW_BUILD=1 python3.9 -m pip wheel --no-deps -w ./tmp/ ./ - for wheel in ./tmp/*.whl; - do - auditwheel repair "$wheel" --plat "manylinux2014_x86_64" -w ./ - done + exit 0 fi + + +# If we have a commit SHA, try to download the prebuild wheel from GitHub Actions +if [[ -n "${BASELINE_COMMIT_SHA}" ]]; +then + CI_COMMIT_SHA="${BASELINE_COMMIT_SHA}" .gitlab/download-wheels-from-gh-actions.sh || true + + # If we have a ./pywheels/ for cp39 manylinux x86_64 use that wheel + if [ -d "./pywheels" ] && ls ./pywheels/ddtrace-*cp39*manylinux* | grep -q "x86_64"; + then + echo "Using prebuilt wheel from GitHub Actions for baseline commit ${BASELINE_COMMIT_SHA}" + cp ./pywheels/ddtrace-*cp39*manylinux*x86_64.whl ./ + exit 0 + fi +fi + +# Otherwise, build from source +ulimit -c unlimited +curl -sSf https://sh.rustup.rs | sh -s -- -y; +export PATH="$HOME/.cargo/bin:$PATH" +echo "Building wheel for ${BASELINE_BRANCH}:${BASELINE_COMMIT_SHA}" +git checkout "${BASELINE_COMMIT_SHA}" +mkdir ./tmp +PYO3_PYTHON=python3.9 CIBW_BUILD=1 python3.9 -m pip wheel --no-deps -w ./tmp/ ./ +for wheel in ./tmp/*.whl; +do + auditwheel repair "$wheel" --plat "manylinux2014_x86_64" -w ./ +done From 29e35c4081693e8bbb2fcdd295352d793357f1e7 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Thu, 13 Nov 2025 09:36:28 -0500 Subject: [PATCH 2/4] make iterations faster --- .gitlab/package.yml | 2 +- .gitlab/scripts/get-pipelines-for-ref.sh | 31 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100755 .gitlab/scripts/get-pipelines-for-ref.sh diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 151664c18c1..cd677b32e44 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -44,7 +44,7 @@ download_ddtrace_artifacts: # Prevent git operation errors: # failed to determine base repo: failed to run git: fatal: detected dubious ownership in repository at ... git config --global --add safe.directory "${CI_PROJECT_DIR}" - .gitlab/download-wheels-from-gh-actions.sh + CI_COMMIT_SHA=59d5849586844bed76372a28eefaefd85d868e94 .gitlab/download-wheels-from-gh-actions.sh artifacts: paths: - "pywheels/*.whl" diff --git a/.gitlab/scripts/get-pipelines-for-ref.sh b/.gitlab/scripts/get-pipelines-for-ref.sh new file mode 100755 index 00000000000..f073715f1ac --- /dev/null +++ b/.gitlab/scripts/get-pipelines-for-ref.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Usage: +# ./get-pipelines-for-ref.sh +# +# Example: +# ./get-pipelines-for-ref.sh 3dbf82a +# ./get-pipelines-for-ref.sh v2.1.0 +# +# Environment: +# GITLAB_TOKEN (required) + +GITLAB_API_URL=${GITLAB_API_URL:"https://gitlab.ddbuild.io/api/v4"} +GITLAB_PROJECT_ID=${GITLAB_PROJECT_ID:"358"} + +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi + +if [[ -z "${GITLAB_TOKEN:-}" ]]; then + echo "GITLAB_TOKEN is not set" >&2 + exit 1 +fi + +REF="$1" + +curl --fail --silent --show-error \ + --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" \ + "${GITLAB_API_URL}/projects/${GITLAB_PROJECT_ID}/pipelines?ref=${REF}" From 64af6163f0a2b4bfa193add24c04a343323f40a5 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Thu, 13 Nov 2025 09:38:10 -0500 Subject: [PATCH 3/4] test --- .gitlab/benchmarks/steps/build-baseline.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.gitlab/benchmarks/steps/build-baseline.sh b/.gitlab/benchmarks/steps/build-baseline.sh index becfceb0966..1510437fb1e 100755 --- a/.gitlab/benchmarks/steps/build-baseline.sh +++ b/.gitlab/benchmarks/steps/build-baseline.sh @@ -12,15 +12,19 @@ fi # If we have a commit SHA, try to download the prebuild wheel from GitHub Actions if [[ -n "${BASELINE_COMMIT_SHA}" ]]; then - CI_COMMIT_SHA="${BASELINE_COMMIT_SHA}" .gitlab/download-wheels-from-gh-actions.sh || true + # Try to download the wheel from the public S3 bucket if we have one + .gitlab/scripts/get-pipelines-for-ref.sh "${BASELINE_COMMIT_SHA}" || true + + exit 1 + # CI_COMMIT_SHA="${BASELINE_COMMIT_SHA}" .gitlab/download-wheels-from-gh-actions.sh || true # If we have a ./pywheels/ for cp39 manylinux x86_64 use that wheel - if [ -d "./pywheels" ] && ls ./pywheels/ddtrace-*cp39*manylinux* | grep -q "x86_64"; - then - echo "Using prebuilt wheel from GitHub Actions for baseline commit ${BASELINE_COMMIT_SHA}" - cp ./pywheels/ddtrace-*cp39*manylinux*x86_64.whl ./ - exit 0 - fi + # if [ -d "./pywheels" ] && ls ./pywheels/ddtrace-*cp39*manylinux* | grep -q "x86_64"; + # then + # echo "Using prebuilt wheel from GitHub Actions for baseline commit ${BASELINE_COMMIT_SHA}" + # cp ./pywheels/ddtrace-*cp39*manylinux*x86_64.whl ./ + # exit 0 + # fi fi # Otherwise, build from source From 6097ce954d25b8fe1a3892de50ceb999a0d8ccfb Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Thu, 13 Nov 2025 09:55:20 -0500 Subject: [PATCH 4/4] don't try to use gha artifacts --- .gitlab/benchmarks/steps/build-baseline.sh | 19 ++++++++----------- .gitlab/package.yml | 2 +- .gitlab/scripts/get-pipelines-for-ref.sh | 9 ++++----- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/.gitlab/benchmarks/steps/build-baseline.sh b/.gitlab/benchmarks/steps/build-baseline.sh index 1510437fb1e..d77c7080faf 100755 --- a/.gitlab/benchmarks/steps/build-baseline.sh +++ b/.gitlab/benchmarks/steps/build-baseline.sh @@ -4,6 +4,7 @@ set -e -o pipefail # If we have a tag (e.g. v2.21.1), then use the PyPI published wheel if [[ -n "${BASELINE_TAG}" ]]; then + echo "Downloading wheel for tag ${BASELINE_TAG} from PyPI" python3.9 -m pip download --no-deps "ddtrace==${BASELINE_TAG:1}" exit 0 fi @@ -13,18 +14,14 @@ fi if [[ -n "${BASELINE_COMMIT_SHA}" ]]; then # Try to download the wheel from the public S3 bucket if we have one - .gitlab/scripts/get-pipelines-for-ref.sh "${BASELINE_COMMIT_SHA}" || true + PIPELINE_ID=$(.gitlab/scripts/get-pipelines-for-ref.sh "${BASELINE_COMMIT_SHA}" | jq -r 'if length > 0 then .[0].id else empty end') - exit 1 - # CI_COMMIT_SHA="${BASELINE_COMMIT_SHA}" .gitlab/download-wheels-from-gh-actions.sh || true - - # If we have a ./pywheels/ for cp39 manylinux x86_64 use that wheel - # if [ -d "./pywheels" ] && ls ./pywheels/ddtrace-*cp39*manylinux* | grep -q "x86_64"; - # then - # echo "Using prebuilt wheel from GitHub Actions for baseline commit ${BASELINE_COMMIT_SHA}" - # cp ./pywheels/ddtrace-*cp39*manylinux*x86_64.whl ./ - # exit 0 - # fi + if [[ -n "${PIPELINE_ID}" ]]; + then + echo "Downloading prebuilt wheel for ${BASELINE_BRANCH}:${BASELINE_COMMIT_SHA} from pipeline ${PIPELINE_ID}" + python3.9 -m pip download --no-deps --index-url "https://dd-trace-py-builds.s3.amazonaws.com/${PIPELINE_ID}/" "ddtrace" + exit 0 + fi fi # Otherwise, build from source diff --git a/.gitlab/package.yml b/.gitlab/package.yml index cd677b32e44..151664c18c1 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -44,7 +44,7 @@ download_ddtrace_artifacts: # Prevent git operation errors: # failed to determine base repo: failed to run git: fatal: detected dubious ownership in repository at ... git config --global --add safe.directory "${CI_PROJECT_DIR}" - CI_COMMIT_SHA=59d5849586844bed76372a28eefaefd85d868e94 .gitlab/download-wheels-from-gh-actions.sh + .gitlab/download-wheels-from-gh-actions.sh artifacts: paths: - "pywheels/*.whl" diff --git a/.gitlab/scripts/get-pipelines-for-ref.sh b/.gitlab/scripts/get-pipelines-for-ref.sh index f073715f1ac..3cf3706f0d3 100755 --- a/.gitlab/scripts/get-pipelines-for-ref.sh +++ b/.gitlab/scripts/get-pipelines-for-ref.sh @@ -6,13 +6,12 @@ set -euo pipefail # # Example: # ./get-pipelines-for-ref.sh 3dbf82a -# ./get-pipelines-for-ref.sh v2.1.0 # # Environment: # GITLAB_TOKEN (required) -GITLAB_API_URL=${GITLAB_API_URL:"https://gitlab.ddbuild.io/api/v4"} -GITLAB_PROJECT_ID=${GITLAB_PROJECT_ID:"358"} +GITLAB_API_URL=${GITLAB_API_URL:-"https://gitlab.ddbuild.io/api/v4"} +GITLAB_PROJECT_ID=${GITLAB_PROJECT_ID:-"358"} if [[ $# -ne 1 ]]; then echo "usage: $0 " >&2 @@ -26,6 +25,6 @@ fi REF="$1" -curl --fail --silent --show-error \ +curl --show-error --silent --fail \ --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" \ - "${GITLAB_API_URL}/projects/${GITLAB_PROJECT_ID}/pipelines?ref=${REF}" + "${GITLAB_API_URL}/projects/${GITLAB_PROJECT_ID}/pipelines?sha=${REF}"