Skip to content

[CIEXE-2016] Migrate GitLab CI jobs to pod-level resource variables #4010

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
realFlowControl wants to merge 1 commit into from
+132 −157
Closed

[CIEXE-2016] Migrate GitLab CI jobs to pod-level resource variables #4010

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 34 additions & 38 deletions .gitlab/generate-appsec.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,14 +73,10 @@
extends: .appsec_test
image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-${PHP_MAJOR_MINOR}_bookworm-8
variables:
KUBERNETES_CPU_REQUEST: 3
KUBERNETES_CPU_LIMIT: 3
KUBERNETES_MEMORY_REQUEST: 6Gi
KUBERNETES_MEMORY_LIMIT: 6Gi
KUBERNETES_HELPER_CPU_REQUEST: 1
KUBERNETES_HELPER_CPU_LIMIT: 1
KUBERNETES_HELPER_MEMORY_REQUEST: 3Gi
KUBERNETES_HELPER_MEMORY_LIMIT: 3Gi
KUBERNETES_POD_CPU_REQUEST: 3
KUBERNETES_POD_CPU_LIMIT: 3
KUBERNETES_POD_MEMORY_REQUEST: 6Gi
KUBERNETES_POD_MEMORY_LIMIT: 6Gi
parallel:
matrix:
- PHP_MAJOR_MINOR: *all_minor_major_targets
Expand All @@ -107,9 +103,9 @@
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble # TODO: use a proper docker image with java pre-installed?
tags: [ "docker-in-docker:amd64" ]
variables:
KUBERNETES_CPU_REQUEST: 8
KUBERNETES_MEMORY_REQUEST: 24Gi
KUBERNETES_MEMORY_LIMIT: 30Gi
KUBERNETES_POD_CPU_REQUEST: 8
KUBERNETES_POD_MEMORY_REQUEST: 24Gi
KUBERNETES_POD_MEMORY_LIMIT: 30Gi
DOCKER_LOOPBACK_SIZE: 30G
ARCH: amd64
HELPER_RUST_FLAG: ""
Expand Down Expand Up @@ -204,9 +200,9 @@
interruptible: false
- when: on_success
variables:
KUBERNETES_CPU_REQUEST: 4
KUBERNETES_MEMORY_REQUEST: 8Gi
KUBERNETES_MEMORY_LIMIT: 10Gi
KUBERNETES_POD_CPU_REQUEST: 4
KUBERNETES_POD_MEMORY_REQUEST: 8Gi
KUBERNETES_POD_MEMORY_LIMIT: 10Gi
ARCH: amd64
before_script:
<?php echo $ecrLoginSnippet, "\n"; ?>
Expand Down Expand Up @@ -239,9 +235,9 @@
interruptible: false
- when: on_success
variables:
KUBERNETES_CPU_REQUEST: 4
KUBERNETES_MEMORY_REQUEST: 8Gi
KUBERNETES_MEMORY_LIMIT: 10Gi
KUBERNETES_POD_CPU_REQUEST: 4
KUBERNETES_POD_MEMORY_REQUEST: 8Gi
KUBERNETES_POD_MEMORY_LIMIT: 10Gi
ARCH: amd64
before_script:
<?php echo $ecrLoginSnippet, "\n"; ?>
Expand Down Expand Up @@ -313,9 +309,9 @@
interruptible: false
- when: on_success
variables:
KUBERNETES_CPU_REQUEST: 8
KUBERNETES_MEMORY_REQUEST: 24Gi
KUBERNETES_MEMORY_LIMIT: 30Gi
KUBERNETES_POD_CPU_REQUEST: 8
KUBERNETES_POD_MEMORY_REQUEST: 24Gi
KUBERNETES_POD_MEMORY_LIMIT: 30Gi
ARCH: amd64
DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED: "0"
before_script:
Expand Down Expand Up @@ -395,9 +391,9 @@
extends: .appsec_test
image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-8.3_bookworm-8
variables:
KUBERNETES_CPU_REQUEST: 3
KUBERNETES_MEMORY_REQUEST: 3Gi
KUBERNETES_MEMORY_LIMIT: 4Gi
KUBERNETES_POD_CPU_REQUEST: 3
KUBERNETES_POD_MEMORY_REQUEST: 3Gi
KUBERNETES_POD_MEMORY_LIMIT: 4Gi
ARCH: amd64
script:
- |
Expand Down Expand Up @@ -482,9 +478,9 @@
extends: .docker_push_job
tags: [ "docker-in-docker:${ARCH}" ]
variables:
KUBERNETES_CPU_REQUEST: 8
KUBERNETES_MEMORY_REQUEST: 16Gi
KUBERNETES_MEMORY_LIMIT: 24Gi
KUBERNETES_POD_CPU_REQUEST: 8
KUBERNETES_POD_MEMORY_REQUEST: 16Gi
KUBERNETES_POD_MEMORY_LIMIT: 24Gi
parallel:
matrix:
# XXX: docker-in-docker:arm64 is not supported yet
Expand All @@ -500,9 +496,9 @@
"push appsec docker images multiarch":
extends: .docker_push_job
variables:
KUBERNETES_CPU_REQUEST: 2
KUBERNETES_MEMORY_REQUEST: 4Gi
KUBERNETES_MEMORY_LIMIT: 6Gi
KUBERNETES_POD_CPU_REQUEST: 2
KUBERNETES_POD_MEMORY_REQUEST: 4Gi
KUBERNETES_POD_MEMORY_LIMIT: 6Gi
ARCH: amd64
rules:
- when: on_success
Expand All @@ -517,9 +513,9 @@
extends: .appsec_test
image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-8.3_bookworm-8
variables:
KUBERNETES_CPU_REQUEST: 3
KUBERNETES_MEMORY_REQUEST: 9Gi
KUBERNETES_MEMORY_LIMIT: 10Gi
KUBERNETES_POD_CPU_REQUEST: 3
KUBERNETES_POD_MEMORY_REQUEST: 9Gi
KUBERNETES_POD_MEMORY_LIMIT: 10Gi
ARCH: amd64
script:
- sudo apt install -y clang-format-20
Expand All @@ -539,9 +535,9 @@
extends: .appsec_test
image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:bookworm-8
variables:
KUBERNETES_CPU_REQUEST: 3
KUBERNETES_MEMORY_REQUEST: 3Gi
KUBERNETES_MEMORY_LIMIT: 4Gi
KUBERNETES_POD_CPU_REQUEST: 3
KUBERNETES_POD_MEMORY_REQUEST: 3Gi
KUBERNETES_POD_MEMORY_LIMIT: 4Gi
parallel:
matrix:
- ARCH: *arch_targets
Expand All @@ -565,9 +561,9 @@
# extends: .appsec_test
# image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:bookworm-8
# variables:
# KUBERNETES_CPU_REQUEST: 3
# KUBERNETES_MEMORY_REQUEST: 5Gi
# KUBERNETES_MEMORY_LIMIT: 6Gi
# KUBERNETES_POD_CPU_REQUEST: 3
# KUBERNETES_POD_MEMORY_REQUEST: 5Gi
# KUBERNETES_POD_MEMORY_LIMIT: 6Gi
# parallel:
# matrix:
# - ARCH: *arch_targets
Expand Down
106 changes: 53 additions & 53 deletions .gitlab/generate-package.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,9 +217,9 @@
ABI_NO: "<?= $abi_no ?>"
PHP_VERSION: "<?= $major_minor ?>"
CARGO_BUILD_JOBS: 12
KUBERNETES_CPU_REQUEST: 12
KUBERNETES_MEMORY_REQUEST: 4Gi
KUBERNETES_MEMORY_LIMIT: 8Gi
KUBERNETES_POD_CPU_REQUEST: 12
KUBERNETES_POD_MEMORY_REQUEST: 4Gi
KUBERNETES_POD_MEMORY_LIMIT: 8Gi
script:
- .gitlab/build-profiler.sh "datadog-profiling/${TRIPLET}/lib/php/${ABI_NO}" "nts"
- .gitlab/build-profiler.sh "datadog-profiling/${TRIPLET}/lib/php/${ABI_NO}" "zts"
Expand Down Expand Up @@ -259,9 +259,9 @@
ABI_NO: "<?= $abi_no ?>"
PHP_VERSION: "<?= $major_minor ?>"
MAKE_JOBS: 12
KUBERNETES_CPU_REQUEST: 12
KUBERNETES_MEMORY_REQUEST: 4Gi
KUBERNETES_MEMORY_LIMIT: 8Gi
KUBERNETES_POD_CPU_REQUEST: 12
KUBERNETES_POD_MEMORY_REQUEST: 4Gi
KUBERNETES_POD_MEMORY_LIMIT: 8Gi
script:
# Fix for $BASH_ENV not having a newline at the end of the file
- echo "" >> "$BASH_ENV"
Expand Down Expand Up @@ -293,9 +293,9 @@
- ARCH: ["amd64", "arm64" ]
variables:
MAKE_JOBS: 12
KUBERNETES_CPU_REQUEST: 12
KUBERNETES_MEMORY_REQUEST: 4Gi
KUBERNETES_MEMORY_LIMIT: 8Gi
KUBERNETES_POD_CPU_REQUEST: 12
KUBERNETES_POD_MEMORY_REQUEST: 4Gi
KUBERNETES_POD_MEMORY_LIMIT: 8Gi
script: .gitlab/build-appsec-helper.sh
artifacts:
paths:
Expand All @@ -311,9 +311,9 @@
- ARCH: ["amd64", "arm64" ]
variables:
MAKE_JOBS: 12
KUBERNETES_CPU_REQUEST: 12
KUBERNETES_MEMORY_REQUEST: 8Gi
KUBERNETES_MEMORY_LIMIT: 12Gi
KUBERNETES_POD_CPU_REQUEST: 12
KUBERNETES_POD_MEMORY_REQUEST: 8Gi
KUBERNETES_POD_MEMORY_LIMIT: 12Gi
script: .gitlab/build-appsec-helper-rust.sh
artifacts:
paths:
Expand Down Expand Up @@ -350,9 +350,9 @@
ABI_NO: "<?= $abi_no ?>"
PHP_VERSION: "<?= $major_minor ?>"
MAKE_JOBS: 12
KUBERNETES_CPU_REQUEST: 12
KUBERNETES_MEMORY_REQUEST: 4Gi
KUBERNETES_MEMORY_LIMIT: 8Gi
KUBERNETES_POD_CPU_REQUEST: 12
KUBERNETES_POD_MEMORY_REQUEST: 4Gi
KUBERNETES_POD_MEMORY_LIMIT: 8Gi
script:
# Fix for $BASH_ENV not having a newline at the end of the file
- echo "" >> "$BASH_ENV"
Expand Down Expand Up @@ -418,9 +418,9 @@
ARCH: "<?= $platform['arch'] ?>"
HOST_OS: "<?= $platform['host_os'] ?>"
CARGO_BUILD_JOBS: 16
KUBERNETES_CPU_REQUEST: 16
KUBERNETES_MEMORY_REQUEST: 5Gi
KUBERNETES_MEMORY_LIMIT: 8Gi
KUBERNETES_POD_CPU_REQUEST: 16
KUBERNETES_POD_MEMORY_REQUEST: 5Gi
KUBERNETES_POD_MEMORY_LIMIT: 8Gi
script:
- echo "" >> "$BASH_ENV"
- ./.gitlab/build-sidecar.sh "<?= $suffix ?>"
Expand Down Expand Up @@ -465,9 +465,9 @@
TRIPLET: "<?= $platform['triplet'] ?>"
ARCH: "<?= $platform['arch'] ?>"
ABI_NO: "<?= $abi_no ?>"
KUBERNETES_CPU_REQUEST: 12
KUBERNETES_MEMORY_REQUEST: 8Gi
KUBERNETES_MEMORY_LIMIT: 16Gi
KUBERNETES_POD_CPU_REQUEST: 12
KUBERNETES_POD_MEMORY_REQUEST: 8Gi
KUBERNETES_POD_MEMORY_LIMIT: 16Gi
script:
# Fix for $BASH_ENV not having a newline at the end of the file
- echo "" >> "$BASH_ENV"
Expand Down Expand Up @@ -499,9 +499,9 @@
ABI_NO: "<?= $abi_no ?>"
PHP_VERSION: "<?= $major_minor ?>"
MAKE_JOBS: 12
KUBERNETES_CPU_REQUEST: 12
KUBERNETES_MEMORY_REQUEST: 4Gi
KUBERNETES_MEMORY_LIMIT: 8Gi
KUBERNETES_POD_CPU_REQUEST: 12
KUBERNETES_POD_MEMORY_REQUEST: 4Gi
KUBERNETES_POD_MEMORY_LIMIT: 8Gi
script: ./.gitlab/build-tracing-asan.sh
artifacts:
paths:
Expand Down Expand Up @@ -791,9 +791,9 @@
stage: verify
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble # TODO: use a proper docker image with make, php and git pre-installed
variables:
KUBERNETES_CPU_REQUEST: 7
KUBERNETES_MEMORY_REQUEST: 30Gi
KUBERNETES_MEMORY_LIMIT: 40Gi
KUBERNETES_POD_CPU_REQUEST: 7
KUBERNETES_POD_MEMORY_REQUEST: 30Gi
KUBERNETES_POD_MEMORY_LIMIT: 40Gi
RUST_BACKTRACE: 1
DOCKER_COMPOSE_DOWNLOAD_NAME: docker-compose-linux-x86_64
before_script:
Expand Down Expand Up @@ -873,9 +873,9 @@
- job: datadog-setup.php
artifacts: true
variables:
KUBERNETES_CPU_REQUEST: 2
KUBERNETES_MEMORY_REQUEST: 2Gi
KUBERNETES_MEMORY_LIMIT: 4Gi
KUBERNETES_POD_CPU_REQUEST: 2
KUBERNETES_POD_MEMORY_REQUEST: 2Gi
KUBERNETES_POD_MEMORY_LIMIT: 4Gi
RUST_BACKTRACE: 1
before_script:
<?php dockerhub_login() ?>
Expand All @@ -895,9 +895,9 @@
- job: datadog-setup.php
artifacts: true
variables:
KUBERNETES_CPU_REQUEST: 2
KUBERNETES_MEMORY_REQUEST: 2Gi
KUBERNETES_MEMORY_LIMIT: 4Gi
KUBERNETES_POD_CPU_REQUEST: 2
KUBERNETES_POD_MEMORY_REQUEST: 2Gi
KUBERNETES_POD_MEMORY_LIMIT: 4Gi
RUST_BACKTRACE: 1
before_script:
<?php unset_dd_runner_env_vars() ?>
Expand All @@ -914,9 +914,9 @@
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble
tags: [ "docker-in-docker:amd64" ]
variables:
KUBERNETES_CPU_REQUEST: 2
KUBERNETES_MEMORY_REQUEST: 2Gi
KUBERNETES_MEMORY_LIMIT: 4Gi
KUBERNETES_POD_CPU_REQUEST: 2
KUBERNETES_POD_MEMORY_REQUEST: 2Gi
KUBERNETES_POD_MEMORY_LIMIT: 4Gi
needs:
- job: "package extension: [amd64, x86_64-unknown-linux-gnu]"
artifacts: true
Expand Down Expand Up @@ -963,9 +963,9 @@
services:
- !reference [.services, request-replayer]
variables:
KUBERNETES_CPU_REQUEST: 2
KUBERNETES_MEMORY_REQUEST: 2Gi
KUBERNETES_MEMORY_LIMIT: 4Gi
KUBERNETES_POD_CPU_REQUEST: 2
KUBERNETES_POD_MEMORY_REQUEST: 2Gi
KUBERNETES_POD_MEMORY_LIMIT: 4Gi
DD_AGENT_HOST: request-replayer
DD_TRACE_AGENT_PORT: 80
DD_TRACE_AGENT_FLUSH_INTERVAL: 1000
Expand Down Expand Up @@ -1086,9 +1086,9 @@
image: registry.ddbuild.io/images/mirror/debian:bullseye-slim
tags: [ "arch:<?= $arch ?>" ]
variables:
KUBERNETES_CPU_REQUEST: 2
KUBERNETES_MEMORY_REQUEST: 2Gi
KUBERNETES_MEMORY_LIMIT: 4Gi
KUBERNETES_POD_CPU_REQUEST: 2
KUBERNETES_POD_MEMORY_REQUEST: 2Gi
KUBERNETES_POD_MEMORY_LIMIT: 4Gi
PHP_VERSION: "<?= $major_minor ?>"
needs:
- job: "package extension: [<?= $arch ?>, <?= $pkgprefix ?>-unknown-linux-gnu]"
Expand All @@ -1108,9 +1108,9 @@
image: registry.ddbuild.io/images/mirror/alpine:3.12
tags: [ "arch:amd64" ]
variables:
KUBERNETES_CPU_REQUEST: 2
KUBERNETES_MEMORY_REQUEST: 2Gi
KUBERNETES_MEMORY_LIMIT: 4Gi
KUBERNETES_POD_CPU_REQUEST: 2
KUBERNETES_POD_MEMORY_REQUEST: 2Gi
KUBERNETES_POD_MEMORY_LIMIT: 4Gi
needs:
- job: "package extension: [amd64, x86_64-alpine-linux-musl]"
artifacts: true
Expand Down Expand Up @@ -1160,9 +1160,9 @@
- !reference [.services, request-replayer]
- !reference [.services, httpbin-integration]
variables:
KUBERNETES_CPU_REQUEST: 4
KUBERNETES_MEMORY_REQUEST: 3Gi
KUBERNETES_MEMORY_LIMIT: 5Gi
KUBERNETES_POD_CPU_REQUEST: 4
KUBERNETES_POD_MEMORY_REQUEST: 3Gi
KUBERNETES_POD_MEMORY_LIMIT: 5Gi
parallel:
matrix:
- PHP_VERSION: <?= json_encode($all_minor_major_targets), "\n" ?>
Expand Down Expand Up @@ -1220,10 +1220,10 @@
tags: [ "docker-in-docker:amd64" ]
variables:
TEST_LIBRARY: php
KUBERNETES_CPU_REQUEST: 8
KUBERNETES_POD_CPU_REQUEST: 8
PYTEST_XDIST_AUTO_NUM_WORKERS: 8
KUBERNETES_MEMORY_REQUEST: 3Gi
KUBERNETES_MEMORY_LIMIT: 4Gi
KUBERNETES_POD_MEMORY_REQUEST: 3Gi
KUBERNETES_POD_MEMORY_LIMIT: 4Gi
RUST_BACKTRACE: 1
BUILD_SH_ARGS: php
PIP_CACHE_DIR: $CI_PROJECT_DIR/.cache/pip
Expand Down Expand Up @@ -1630,8 +1630,8 @@
image: registry.ddbuild.io/images/mirror/php:8.2-cli
tags: [ "arch:amd64" ]
variables: # enough memory for the individual artifacts
KUBERNETES_MEMORY_REQUEST: 4Gi
KUBERNETES_MEMORY_LIMIT: 5Gi
KUBERNETES_POD_MEMORY_REQUEST: 4Gi
KUBERNETES_POD_MEMORY_LIMIT: 5Gi
only:
refs:
- /^ddtrace-.*$/
Expand Down
Loading
Loading