From ee3fff43c556a9dbb4aed9a1029986e41357b52a Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 3 Nov 2026 19:58:13 +0000 Subject: [PATCH 01/29] ci: configure systems tests using cloud build for google-cloud-bigquery-storage --- ci/cloudbuild/system-test.yaml | 116 +++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 ci/cloudbuild/system-test.yaml diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml new file mode 100644 index 000000000000..8a03af203889 --- /dev/null +++ b/ci/cloudbuild/system-test.yaml @@ -0,0 +1,116 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - name: 'us-central1-docker.pkg.dev/client-library-releases/release-images-prod/python-multi@sha256:d529954c373542171f9fc411e0102cebe542b511e320a965edf8820285f4f122' + id: 'run-system-tests-shard' + env: + # Populate standard GCP project environment variables expected by system test suites. + - 'GOOGLE_CLOUD_PROJECT=${PROJECT_ID}' + - 'GCLOUD_PROJECT=${PROJECT_ID}' + - 'PROJECT_ID=${PROJECT_ID}' + # Pass sharding and diff parameters consumed by ci/get_package_shards.py. + - 'MAX_SHARDS=${_MAX_SHARDS}' + - 'SHARD_INDEX=${_SHARD_INDEX}' + - 'BUILD_TYPE=${_BUILD_TYPE}' + - 'TARGET_BRANCH=${_BASE_BRANCH}' + - 'TEST_ALL_PACKAGES=${_TEST_ALL_PACKAGES}' + - 'ENABLED_PACKAGES=${_ENABLED_PACKAGES}' + - 'PACKAGE_WEIGHTS=${_PACKAGE_WEIGHTS}' + # Specify the nox session consumed by ci/run_conditional_tests.sh. + - 'NOX_SESSION=${_NOX_SESSION}' + script: | + #!/usr/bin/env bash + set -eo pipefail + + # Fetch target branch history so git diff in ci/get_package_shards.py can detect modified packages. + git fetch origin "${TARGET_BRANCH}" --depth=50 || true + + # Import ci/get_package_shards.py without modifying the script on disk. + # Filter get_packages() in memory to _ENABLED_PACKAGES so unmigrated packages are ignored. + PACKAGE_LIST=$(python3 -c ' + import io, json, os, sys + from contextlib import redirect_stdout + + sys.path.insert(0, "ci") + import get_package_shards as gps + + enabled = { + p.strip().rstrip("/") + "/" + for p in os.environ.get("ENABLED_PACKAGES", "").replace(",", " ").split() + if p.strip() + } + if enabled: + orig_get_packages = gps.get_packages + gps.get_packages = lambda: [p for p in orig_get_packages() if p in enabled] + + buf = io.StringIO() + with redirect_stdout(buf): + if hasattr(gps, "main"): + gps.main() + else: + import runpy + runpy.run_path("ci/get_package_shards.py", run_name="__main__") + + lines = [l for l in buf.getvalue().splitlines() if l.strip()] + data = json.loads(lines[-1]) if lines else [] + shards = data.get("include", data) if isinstance(data, dict) else data + + idx = int(os.environ.get("SHARD_INDEX", "0")) + print(shards[idx].get("packages", "") if idx < len(shards) else "") + ') + + # Report immediate green (< 10s) if no enabled packages in this PR map to this shard index. + if [[ -z "${PACKAGE_LIST}" ]]; then + echo "No enabled packages assigned to shard ${SHARD_INDEX}. Exiting 0." + exit 0 + fi + + # Package noxfile.py files skip the system session if GOOGLE_APPLICATION_CREDENTIALS is unset. + # Set a placeholder value for noxfile.py, then unset it at pytest startup so google.auth.default() + # uses native Cloud Build VM metadata server credentials (ADC) automatically. + if [[ -z "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then + mkdir -p /tmp/cb_pytest_shim + cat << 'EOF' > /tmp/cb_pytest_shim/cb_adc_shim.py + import os + if os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") == "CLOUD_BUILD_ADC": + os.environ.pop("GOOGLE_APPLICATION_CREDENTIALS", None) + EOF + export GOOGLE_APPLICATION_CREDENTIALS="CLOUD_BUILD_ADC" + export PYTHONPATH="/tmp/cb_pytest_shim:${PYTHONPATH:-}" + export PYTEST_ADDOPTS="-p cb_adc_shim ${PYTEST_ADDOPTS:-}" + fi + + # Export PACKAGE_LIST and delegate execution to the existing monorepo test runner script. + export PACKAGE_LIST + echo "=== System Shard ${SHARD_INDEX}/${MAX_SHARDS} running ${NOX_SESSION} for: ${PACKAGE_LIST} ===" + ./ci/run_conditional_tests.sh + +# Hard 30-minute (1800s) CI timeout ceiling. +timeout: 1800s + +substitutions: + _MAX_SHARDS: '1' + _SHARD_INDEX: '0' + _BUILD_TYPE: 'presubmit' + _BASE_BRANCH: 'main' + _TEST_ALL_PACKAGES: 'false' + _ENABLED_PACKAGES: 'packages/google-cloud-bigquery-storage' + _NOX_SESSION: 'system-3.12' + _PACKAGE_WEIGHTS: >- + google-cloud-bigquery-storage:5 + +options: + machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY From 6417a59c006fa8ff52435eda14345eb00b049a57 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 5 Nov 2026 18:17:17 +0000 Subject: [PATCH 02/29] fix build --- ci/cloudbuild/system-test.yaml | 39 ++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 8a03af203889..1e9aef67168c 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -28,20 +28,25 @@ steps: - 'TEST_ALL_PACKAGES=${_TEST_ALL_PACKAGES}' - 'ENABLED_PACKAGES=${_ENABLED_PACKAGES}' - 'PACKAGE_WEIGHTS=${_PACKAGE_WEIGHTS}' - # Specify the nox session consumed by ci/run_conditional_tests.sh. + # Specify the nox session consumed by ci/run_conditional_tests.sh and ci/run_single_test.sh. - 'NOX_SESSION=${_NOX_SESSION}' + - 'TEST_TYPE=${_NOX_SESSION}' script: | #!/usr/bin/env bash set -eo pipefail - # Fetch target branch history so git diff in ci/get_package_shards.py can detect modified packages. - git fetch origin "${TARGET_BRANCH}" --depth=50 || true + # Derive PY_VERSION (e.g. "3.12" from "system-3.12") required by ci/run_single_test.sh. + export PY_VERSION="${NOX_SESSION#system-}" + + # Deepen both HEAD and target branch history so git diff in ci/get_package_shards.py + # can find the merge base in Cloud Build's shallow checkout. + git fetch --no-tags origin "+refs/heads/${TARGET_BRANCH}:refs/remotes/origin/${TARGET_BRANCH}" --depth=200 || true + git fetch --no-tags origin HEAD --depth=200 || true # Import ci/get_package_shards.py without modifying the script on disk. # Filter get_packages() in memory to _ENABLED_PACKAGES so unmigrated packages are ignored. PACKAGE_LIST=$(python3 -c ' - import io, json, os, sys - from contextlib import redirect_stdout + import os, sys sys.path.insert(0, "ci") import get_package_shards as gps @@ -53,19 +58,17 @@ steps: } if enabled: orig_get_packages = gps.get_packages - gps.get_packages = lambda: [p for p in orig_get_packages() if p in enabled] - - buf = io.StringIO() - with redirect_stdout(buf): - if hasattr(gps, "main"): - gps.main() - else: - import runpy - runpy.run_path("ci/get_package_shards.py", run_name="__main__") + def filtered_get_packages(*args, **kwargs): + pkgs = {} + for name, paths in orig_get_packages(*args, **kwargs).items(): + kept = [p for p in paths if p in enabled] + if kept: + pkgs[name] = kept + return pkgs + gps.get_packages = filtered_get_packages - lines = [l for l in buf.getvalue().splitlines() if l.strip()] - data = json.loads(lines[-1]) if lines else [] - shards = data.get("include", data) if isinstance(data, dict) else data + packages = gps.get_packages_to_test() + shards = gps.group_packages(packages) idx = int(os.environ.get("SHARD_INDEX", "0")) print(shards[idx].get("packages", "") if idx < len(shards) else "") @@ -88,7 +91,7 @@ steps: os.environ.pop("GOOGLE_APPLICATION_CREDENTIALS", None) EOF export GOOGLE_APPLICATION_CREDENTIALS="CLOUD_BUILD_ADC" - export PYTHONPATH="/tmp/cb_pytest_shim:${PYTHONPATH:-}" + export PYTHONPATH="/tmp/cb_pytest_shim${PYTHONPATH:+:${PYTHONPATH}}" export PYTEST_ADDOPTS="-p cb_adc_shim ${PYTEST_ADDOPTS:-}" fi From 09e6180fa10a1d839d0a4adf0ea2a53f0863913f Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 5 Nov 2026 18:51:10 +0000 Subject: [PATCH 03/29] fix build --- ci/cloudbuild/system-test.yaml | 70 ++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 1e9aef67168c..6c8bbec80a26 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -31,6 +31,9 @@ steps: # Specify the nox session consumed by ci/run_conditional_tests.sh and ci/run_single_test.sh. - 'NOX_SESSION=${_NOX_SESSION}' - 'TEST_TYPE=${_NOX_SESSION}' + # Configure parallel package execution and unbuffered Python output for real-time logging. + - 'PARALLEL_WORKERS=${_PARALLEL_WORKERS}' + - 'PYTHONUNBUFFERED=1' script: | #!/usr/bin/env bash set -eo pipefail @@ -38,10 +41,15 @@ steps: # Derive PY_VERSION (e.g. "3.12" from "system-3.12") required by ci/run_single_test.sh. export PY_VERSION="${NOX_SESSION#system-}" - # Deepen both HEAD and target branch history so git diff in ci/get_package_shards.py - # can find the merge base in Cloud Build's shallow checkout. - git fetch --no-tags origin "+refs/heads/${TARGET_BRANCH}:refs/remotes/origin/${TARGET_BRANCH}" --depth=200 || true - git fetch --no-tags origin HEAD --depth=200 || true + # Deepen both the checked-out PR commit and target branch history so git diff in + # ci/get_package_shards.py finds the merge base in Cloud Build's shallow checkout, + # and populate local refs/heads/${TARGET_BRANCH} so ci/run_conditional_tests.sh's + # local insteadOf fetch succeeds silently. + git fetch --no-tags --quiet origin \ + "$(git rev-parse HEAD)" \ + "+refs/heads/${TARGET_BRANCH}:refs/remotes/origin/${TARGET_BRANCH}" \ + "+refs/heads/${TARGET_BRANCH}:refs/heads/${TARGET_BRANCH}" \ + --depth=200 || true # Import ci/get_package_shards.py without modifying the script on disk. # Filter get_packages() in memory to _ENABLED_PACKAGES so unmigrated packages are ignored. @@ -80,20 +88,57 @@ steps: exit 0 fi - # Package noxfile.py files skip the system session if GOOGLE_APPLICATION_CREDENTIALS is unset. - # Set a placeholder value for noxfile.py, then unset it at pytest startup so google.auth.default() - # uses native Cloud Build VM metadata server credentials (ADC) automatically. - if [[ -z "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then - mkdir -p /tmp/cb_pytest_shim - cat << 'EOF' > /tmp/cb_pytest_shim/cb_adc_shim.py + # Create runtime shims in /tmp/cloudbuild_shims so tracked repo files remain unmodified: + # - cloudbuild_pytest_plugin.py: Unsets the placeholder GOOGLE_APPLICATION_CREDENTIALS + # value ("CLOUD_BUILD_ADC" satisfies noxfile.py's env check) so google.auth.default() + # uses Cloud Build VM ADC, and flushes per-test results so `py.test --quiet` logs live. + # - nox wrapper: Streams [package]-prefixed output to fd 3 in real time across parallel + # workers, and only writes to stdout on failure so run_conditional_tests.sh does not + # print passing logs twice. + mkdir -p /tmp/cloudbuild_shims + cat << 'EOF' > /tmp/cloudbuild_shims/cloudbuild_pytest_plugin.py import os + if os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") == "CLOUD_BUILD_ADC": os.environ.pop("GOOGLE_APPLICATION_CREDENTIALS", None) + + + def pytest_runtest_logreport(report): + if report.when == "call" or ( + report.when in ("setup", "teardown") and report.failed + ): + print( + f"{report.outcome.upper()} {report.nodeid} ({report.duration:.1f}s)", + flush=True, + ) + EOF + + cat << 'EOF' > /tmp/cloudbuild_shims/nox + #!/usr/bin/env bash + PKG=$(basename "$PWD") + REAL_NOX=$(which -a nox | grep -v '/tmp/cloudbuild_shims' | head -n 1) + TMP_LOG=$(mktemp) + echo "[${PKG}] Starting nox $*" >&3 + "$REAL_NOX" "$@" 2>&1 | awk -v p="[${PKG}] " -v f="${TMP_LOG}" '{ + print $0 > f; fflush(f); + print p $0 > "/dev/fd/3"; fflush("/dev/fd/3"); + }' + RET=${PIPESTATUS[0]} + if [[ ${RET} -ne 0 ]]; then + cat "${TMP_LOG}" + fi + rm -f "${TMP_LOG}" + exit ${RET} EOF + chmod +x /tmp/cloudbuild_shims/nox + + if [[ -z "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then export GOOGLE_APPLICATION_CREDENTIALS="CLOUD_BUILD_ADC" - export PYTHONPATH="/tmp/cb_pytest_shim${PYTHONPATH:+:${PYTHONPATH}}" - export PYTEST_ADDOPTS="-p cb_adc_shim ${PYTEST_ADDOPTS:-}" fi + export PATH="/tmp/cloudbuild_shims:${PATH}" + export PYTHONPATH="/tmp/cloudbuild_shims${PYTHONPATH:+:${PYTHONPATH}}" + export PYTEST_ADDOPTS="-p cloudbuild_pytest_plugin ${PYTEST_ADDOPTS:-}" + exec 3>&1 # Export PACKAGE_LIST and delegate execution to the existing monorepo test runner script. export PACKAGE_LIST @@ -106,6 +151,7 @@ timeout: 1800s substitutions: _MAX_SHARDS: '1' _SHARD_INDEX: '0' + _PARALLEL_WORKERS: '4' _BUILD_TYPE: 'presubmit' _BASE_BRANCH: 'main' _TEST_ALL_PACKAGES: 'false' From 307af456532af1bdc0ae19a8184c9cdba2e3f0de Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 5 Nov 2026 19:28:42 +0000 Subject: [PATCH 04/29] fix build --- ci/cloudbuild/system-test.yaml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 6c8bbec80a26..4b16eeb812e2 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -91,10 +91,9 @@ steps: # Create runtime shims in /tmp/cloudbuild_shims so tracked repo files remain unmodified: # - cloudbuild_pytest_plugin.py: Unsets the placeholder GOOGLE_APPLICATION_CREDENTIALS # value ("CLOUD_BUILD_ADC" satisfies noxfile.py's env check) so google.auth.default() - # uses Cloud Build VM ADC, and flushes per-test results so `py.test --quiet` logs live. + # uses Cloud Build VM ADC, and enables verbose mode so `py.test --quiet` logs each test line. # - nox wrapper: Streams [package]-prefixed output to fd 3 in real time across parallel - # workers, and only writes to stdout on failure so run_conditional_tests.sh does not - # print passing logs twice. + # workers, and on failure outputs only the FAILURES summary to stdout for run_conditional_tests.sh. mkdir -p /tmp/cloudbuild_shims cat << 'EOF' > /tmp/cloudbuild_shims/cloudbuild_pytest_plugin.py import os @@ -103,14 +102,9 @@ steps: os.environ.pop("GOOGLE_APPLICATION_CREDENTIALS", None) - def pytest_runtest_logreport(report): - if report.when == "call" or ( - report.when in ("setup", "teardown") and report.failed - ): - print( - f"{report.outcome.upper()} {report.nodeid} ({report.duration:.1f}s)", - flush=True, - ) + def pytest_configure(config): + config.option.quiet = 0 + config.option.verbose = max(config.option.verbose, 1) EOF cat << 'EOF' > /tmp/cloudbuild_shims/nox @@ -125,7 +119,7 @@ steps: }' RET=${PIPESTATUS[0]} if [[ ${RET} -ne 0 ]]; then - cat "${TMP_LOG}" + sed -n '/=== FAILURES ===/,$p' "${TMP_LOG}" fi rm -f "${TMP_LOG}" exit ${RET} From fb5106083572e3a24b7297dbe10e7773007ba385 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 5 Nov 2026 19:33:03 +0000 Subject: [PATCH 05/29] fix build --- ci/cloudbuild/system-test.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 4b16eeb812e2..716f98f695b6 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -59,6 +59,11 @@ steps: sys.path.insert(0, "ci") import get_package_shards as gps + # Only run system tests for packages directly modified in the PR (unless + # TEST_ALL_PACKAGES=true on postsubmit), ignoring .github/, ci/, or core package changes. + gps.CI_INFRASTRUCTURE_DIRS = set() + gps.CORE_PACKAGES = set() + enabled = { p.strip().rstrip("/") + "/" for p in os.environ.get("ENABLED_PACKAGES", "").replace(",", " ").split() From b0fa48412a150b7d943b19aa878f081668039611 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 5 Nov 2026 19:49:50 +0000 Subject: [PATCH 06/29] fix build --- ci/cloudbuild/system-test.yaml | 143 ++++++++++++++++++++++----------- 1 file changed, 98 insertions(+), 45 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 716f98f695b6..41baa97a54e8 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -31,8 +31,10 @@ steps: # Specify the nox session consumed by ci/run_conditional_tests.sh and ci/run_single_test.sh. - 'NOX_SESSION=${_NOX_SESSION}' - 'TEST_TYPE=${_NOX_SESSION}' - # Configure parallel package execution and unbuffered Python output for real-time logging. + # Configure parallel package execution, time budget/threshold, and unbuffered Python output. - 'PARALLEL_WORKERS=${_PARALLEL_WORKERS}' + - 'TIME_BUDGET_MINUTES=${_TIME_BUDGET_MINUTES}' + - 'HARD_TIMEOUT_MINUTES=${_HARD_TIMEOUT_MINUTES}' - 'PYTHONUNBUFFERED=1' script: | #!/usr/bin/env bash @@ -54,37 +56,46 @@ steps: # Import ci/get_package_shards.py without modifying the script on disk. # Filter get_packages() in memory to _ENABLED_PACKAGES so unmigrated packages are ignored. PACKAGE_LIST=$(python3 -c ' - import os, sys + import os + import sys sys.path.insert(0, "ci") - import get_package_shards as gps - - # Only run system tests for packages directly modified in the PR (unless - # TEST_ALL_PACKAGES=true on postsubmit), ignoring .github/, ci/, or core package changes. - gps.CI_INFRASTRUCTURE_DIRS = set() - gps.CORE_PACKAGES = set() - - enabled = { - p.strip().rstrip("/") + "/" - for p in os.environ.get("ENABLED_PACKAGES", "").replace(",", " ").split() - if p.strip() + import get_package_shards + + # Only run system tests for packages directly modified in the pull request + # (unless TEST_ALL_PACKAGES=true on postsubmit), ignoring changes to .github/, + # ci/, or core dependency packages. + get_package_shards.CI_INFRASTRUCTURE_DIRS = set() + get_package_shards.CORE_PACKAGES = set() + + # Normalize enabled package paths to include a trailing slash so they match + # the directory format ("packages//") returned by get_package_shards.get_packages(). + enabled_packages = { + package_path.strip().rstrip("/") + "/" + for package_path in os.environ.get("ENABLED_PACKAGES", "").replace(",", " ").split() + if package_path.strip() } - if enabled: - orig_get_packages = gps.get_packages + if enabled_packages: + original_get_packages = get_package_shards.get_packages + def filtered_get_packages(*args, **kwargs): - pkgs = {} - for name, paths in orig_get_packages(*args, **kwargs).items(): - kept = [p for p in paths if p in enabled] - if kept: - pkgs[name] = kept - return pkgs - gps.get_packages = filtered_get_packages - - packages = gps.get_packages_to_test() - shards = gps.group_packages(packages) - - idx = int(os.environ.get("SHARD_INDEX", "0")) - print(shards[idx].get("packages", "") if idx < len(shards) else "") + filtered_packages = {} + for package_name, package_paths in original_get_packages(*args, **kwargs).items(): + kept_paths = [ + path for path in package_paths if path in enabled_packages + ] + if kept_paths: + filtered_packages[package_name] = kept_paths + return filtered_packages + + get_package_shards.get_packages = filtered_get_packages + + # Compute the packages that need testing and distribute them across balanced shards. + packages_to_test = get_package_shards.get_packages_to_test() + shards = get_package_shards.group_packages(packages_to_test) + + shard_index = int(os.environ.get("SHARD_INDEX", "0")) + print(shards[shard_index].get("packages", "") if shard_index < len(shards) else "") ') # Report immediate green (< 10s) if no enabled packages in this PR map to this shard index. @@ -93,13 +104,18 @@ steps: exit 0 fi - # Create runtime shims in /tmp/cloudbuild_shims so tracked repo files remain unmodified: - # - cloudbuild_pytest_plugin.py: Unsets the placeholder GOOGLE_APPLICATION_CREDENTIALS - # value ("CLOUD_BUILD_ADC" satisfies noxfile.py's env check) so google.auth.default() - # uses Cloud Build VM ADC, and enables verbose mode so `py.test --quiet` logs each test line. - # - nox wrapper: Streams [package]-prefixed output to fd 3 in real time across parallel - # workers, and on failure outputs only the FAILURES summary to stdout for run_conditional_tests.sh. + # Create runtime shims in /tmp/cloudbuild_shims so tracked repository files remain unmodified. mkdir -p /tmp/cloudbuild_shims + + # 1. Pytest plugin (cloudbuild_pytest_plugin.py): + # - Package noxfile.py files skip the system session if GOOGLE_APPLICATION_CREDENTIALS + # is unset. We export a placeholder value ("CLOUD_BUILD_ADC") to satisfy noxfile.py, + # then remove it here at pytest startup so google.auth.default() automatically uses + # the Cloud Build worker VM's Application Default Credentials (ADC). + # - Package noxfile.py files invoke `py.test --quiet`, which only outputs dots (".") + # without newlines and causes Cloud Build logs to appear frozen for several minutes. + # Overriding `quiet = 0` and `verbose = 1` makes pytest print one newline-terminated + # status line as each individual test finishes. cat << 'EOF' > /tmp/cloudbuild_shims/cloudbuild_pytest_plugin.py import os @@ -112,31 +128,66 @@ steps: config.option.verbose = max(config.option.verbose, 1) EOF + # 2. Nox wrapper (/tmp/cloudbuild_shims/nox): + # - When PARALLEL_WORKERS > 1, `ci/run_conditional_tests.sh` redirects each worker's + # stdout/stderr (file descriptors 1 and 2) to a temporary log file, which otherwise + # produces no console output until the entire package finishes. + # - Before invoking `ci/run_conditional_tests.sh`, we duplicate the container's live + # stdout to file descriptor 3 (`exec 3>&1`). This wrapper streams every output line + # prefixed with `[]` directly to file descriptor 3 in real time while + # also saving the raw output to a temporary file (`TEMP_LOG_FILE`). + # - At completion, it calculates how much of the per-package time budget was used + # (e.g., 18 minutes out of a 20-minute budget = 90%) and logs the summary. + # - If the nox session fails, it outputs the `=== FAILURES ===` section to stdout so + # `ci/run_conditional_tests.sh` prints a clean failure summary at the end of the run. cat << 'EOF' > /tmp/cloudbuild_shims/nox #!/usr/bin/env bash - PKG=$(basename "$PWD") + PACKAGE_NAME=$(basename "$PWD") REAL_NOX=$(which -a nox | grep -v '/tmp/cloudbuild_shims' | head -n 1) - TMP_LOG=$(mktemp) - echo "[${PKG}] Starting nox $*" >&3 - "$REAL_NOX" "$@" 2>&1 | awk -v p="[${PKG}] " -v f="${TMP_LOG}" '{ - print $0 > f; fflush(f); - print p $0 > "/dev/fd/3"; fflush("/dev/fd/3"); + TEMP_LOG_FILE=$(mktemp) + START_TIME=$(date +%s) + echo "[${PACKAGE_NAME}] Starting nox $*" >&3 + "$REAL_NOX" "$@" 2>&1 | awk -v package_prefix="[${PACKAGE_NAME}] " -v log_file="${TEMP_LOG_FILE}" '{ + print $0 > log_file; fflush(log_file); + print package_prefix $0 > "/dev/fd/3"; fflush("/dev/fd/3"); }' - RET=${PIPESTATUS[0]} - if [[ ${RET} -ne 0 ]]; then - sed -n '/=== FAILURES ===/,$p' "${TMP_LOG}" + EXIT_CODE=${PIPESTATUS[0]} + + # Calculate elapsed time and percentage of the target time budget spent on this package. + ELAPSED_SECONDS=$(( $(date +%s) - START_TIME )) + BUDGET_MINUTES="${TIME_BUDGET_MINUTES:-20}" + HARD_LIMIT_MINUTES="${HARD_TIMEOUT_MINUTES:-30}" + BUDGET_SECONDS=$(( BUDGET_MINUTES * 60 )) + BUDGET_PERCENTAGE=$(( (ELAPSED_SECONDS * 100 + BUDGET_SECONDS / 2) / BUDGET_SECONDS )) + BUDGET_MESSAGE=$(printf "Time budget usage: %d%% (%dm %02ds / %dm budget, %dm hard threshold)" \ + "${BUDGET_PERCENTAGE}" "$(( ELAPSED_SECONDS / 60 ))" "$(( ELAPSED_SECONDS % 60 ))" \ + "${BUDGET_MINUTES}" "${HARD_LIMIT_MINUTES}") + echo "[${PACKAGE_NAME}] ${BUDGET_MESSAGE}" >&3 + echo "${BUDGET_MESSAGE}" + + # Emit failure details to stdout (captured by run_conditional_tests.sh) only when tests fail + # so passing runs do not print their logs a second time. + if [[ ${EXIT_CODE} -ne 0 ]]; then + sed -n '/=== FAILURES ===/,$p' "${TEMP_LOG_FILE}" fi - rm -f "${TMP_LOG}" - exit ${RET} + rm -f "${TEMP_LOG_FILE}" + exit ${EXIT_CODE} EOF chmod +x /tmp/cloudbuild_shims/nox + # Set placeholder credentials for noxfile.py if not already set. if [[ -z "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then export GOOGLE_APPLICATION_CREDENTIALS="CLOUD_BUILD_ADC" fi + + # Prepend /tmp/cloudbuild_shims to PATH and PYTHONPATH so our nox wrapper and pytest + # plugin are picked up automatically by ci/run_single_test.sh and pytest. export PATH="/tmp/cloudbuild_shims:${PATH}" export PYTHONPATH="/tmp/cloudbuild_shims${PYTHONPATH:+:${PYTHONPATH}}" export PYTEST_ADDOPTS="-p cloudbuild_pytest_plugin ${PYTEST_ADDOPTS:-}" + + # Open file descriptor 3 pointing to the container's live stdout so child workers + # spawned by xargs in ci/run_conditional_tests.sh can stream real-time logs. exec 3>&1 # Export PACKAGE_LIST and delegate execution to the existing monorepo test runner script. @@ -151,6 +202,8 @@ substitutions: _MAX_SHARDS: '1' _SHARD_INDEX: '0' _PARALLEL_WORKERS: '4' + _TIME_BUDGET_MINUTES: '20' + _HARD_TIMEOUT_MINUTES: '30' _BUILD_TYPE: 'presubmit' _BASE_BRANCH: 'main' _TEST_ALL_PACKAGES: 'false' From 8060742dbc71189a471d10b7a3b0eb1ee0eea96f Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 5 Nov 2026 20:09:22 +0000 Subject: [PATCH 07/29] fix build --- ci/cloudbuild/system-test.yaml | 36 +++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 41baa97a54e8..7ef6439c8b2d 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -165,6 +165,27 @@ steps: echo "[${PACKAGE_NAME}] ${BUDGET_MESSAGE}" >&3 echo "${BUDGET_MESSAGE}" + # Compare actual runtime in minutes against the configured weight in PACKAGE_WEIGHTS + # (defaulting to 1). Only propose an adjustment when the difference is at least 3 minutes + # (>= 15% of the 20-minute budget) to avoid noise from normal test runtime variance. + PROPOSED_WEIGHT=$(( (ELAPSED_SECONDS + 30) / 60 )) + if [[ ${PROPOSED_WEIGHT} -lt 1 ]]; then + PROPOSED_WEIGHT=1 + fi + CONFIGURED_WEIGHT=$(echo "${PACKAGE_WEIGHTS:-}" | tr ' ' '\n' | awk -F: -v package_name="${PACKAGE_NAME}" ' + $1 == package_name { gsub(/[[:space:]]/, "", $2); if ($2 != "") print $2 } + ' | tail -n 1) + CONFIGURED_WEIGHT="${CONFIGURED_WEIGHT:-1}" + WEIGHT_DIFFERENCE=$(( PROPOSED_WEIGHT - CONFIGURED_WEIGHT )) + if [[ ${WEIGHT_DIFFERENCE} -lt 0 ]]; then + WEIGHT_DIFFERENCE=$(( -WEIGHT_DIFFERENCE )) + fi + if [[ ${WEIGHT_DIFFERENCE} -ge 3 ]]; then + WEIGHT_MESSAGE="Weight adjustment needed for ${PACKAGE_NAME}: configured=${CONFIGURED_WEIGHT}, proposed=${PROPOSED_WEIGHT} (${PACKAGE_NAME}:${PROPOSED_WEIGHT})" + echo "[${PACKAGE_NAME}] ${WEIGHT_MESSAGE}" >&3 + echo "${WEIGHT_MESSAGE}" + fi + # Emit failure details to stdout (captured by run_conditional_tests.sh) only when tests fail # so passing runs do not print their logs a second time. if [[ ${EXIT_CODE} -ne 0 ]]; then @@ -207,10 +228,19 @@ substitutions: _BUILD_TYPE: 'presubmit' _BASE_BRANCH: 'main' _TEST_ALL_PACKAGES: 'false' - _ENABLED_PACKAGES: 'packages/google-cloud-bigquery-storage' + _ENABLED_PACKAGES: >- + packages/bigquery-magics + packages/google-cloud-bigquery-storage + packages/google-cloud-compute + packages/google-cloud-datastore + packages/google-cloud-dns + packages/google-cloud-error-reporting + packages/google-cloud-logging + packages/google-resumable-media + packages/pandas-gbq + packages/sqlalchemy-bigquery _NOX_SESSION: 'system-3.12' - _PACKAGE_WEIGHTS: >- - google-cloud-bigquery-storage:5 + _PACKAGE_WEIGHTS: '' options: machineType: 'E2_HIGHCPU_8' From c602e7b2d8f7b020b3046bd5e2e8f6e5ee58ca93 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 5 Nov 2026 20:24:38 +0000 Subject: [PATCH 08/29] run tests on every change to ci --- ci/cloudbuild/system-test.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 7ef6439c8b2d..0e5e924ee379 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -62,14 +62,12 @@ steps: sys.path.insert(0, "ci") import get_package_shards - # Only run system tests for packages directly modified in the pull request - # (unless TEST_ALL_PACKAGES=true on postsubmit), ignoring changes to .github/, - # ci/, or core dependency packages. - get_package_shards.CI_INFRASTRUCTURE_DIRS = set() - get_package_shards.CORE_PACKAGES = set() - # Normalize enabled package paths to include a trailing slash so they match # the directory format ("packages//") returned by get_package_shards.get_packages(). + # When CI infrastructure (ci/, .github/) or core dependencies change, + # get_package_shards.get_packages_to_test() automatically selects all packages + # returned by get_packages(). Passing handwritten_only=False to original_get_packages + # ensures every package explicitly listed in _ENABLED_PACKAGES is included. enabled_packages = { package_path.strip().rstrip("/") + "/" for package_path in os.environ.get("ENABLED_PACKAGES", "").replace(",", " ").split() @@ -80,7 +78,9 @@ steps: def filtered_get_packages(*args, **kwargs): filtered_packages = {} - for package_name, package_paths in original_get_packages(*args, **kwargs).items(): + for package_name, package_paths in original_get_packages( + handwritten_only=False + ).items(): kept_paths = [ path for path in package_paths if path in enabled_packages ] From 4a1ff5d081a863865ec1738458f1130774716fe6 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 17 Nov 2026 19:04:14 +0000 Subject: [PATCH 09/29] work in progress --- ci/cloudbuild/system-test.yaml | 20 +++++++++++++------ .../v2/test_system_logging_service_v2_v2.py | 11 ++++++---- .../tests/system/test_system.py | 19 +++++++++++------- .../tests/system/requests/conftest.py | 9 ++++++++- .../tests_async/system/requests/conftest.py | 9 ++++++++- 5 files changed, 49 insertions(+), 19 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 0e5e924ee379..d15cb77d301e 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -23,6 +23,7 @@ steps: # Pass sharding and diff parameters consumed by ci/get_package_shards.py. - 'MAX_SHARDS=${_MAX_SHARDS}' - 'SHARD_INDEX=${_SHARD_INDEX}' + - 'MIN_SHARD_WEIGHT=${_MIN_SHARD_WEIGHT}' - 'BUILD_TYPE=${_BUILD_TYPE}' - 'TARGET_BRANCH=${_BASE_BRANCH}' - 'TEST_ALL_PACKAGES=${_TEST_ALL_PACKAGES}' @@ -33,6 +34,7 @@ steps: - 'TEST_TYPE=${_NOX_SESSION}' # Configure parallel package execution, time budget/threshold, and unbuffered Python output. - 'PARALLEL_WORKERS=${_PARALLEL_WORKERS}' + - 'CONTINUE_ON_ERROR=true' - 'TIME_BUDGET_MINUTES=${_TIME_BUDGET_MINUTES}' - 'HARD_TIMEOUT_MINUTES=${_HARD_TIMEOUT_MINUTES}' - 'PYTHONUNBUFFERED=1' @@ -147,10 +149,14 @@ steps: TEMP_LOG_FILE=$(mktemp) START_TIME=$(date +%s) echo "[${PACKAGE_NAME}] Starting nox $*" >&3 - "$REAL_NOX" "$@" 2>&1 | awk -v package_prefix="[${PACKAGE_NAME}] " -v log_file="${TEMP_LOG_FILE}" '{ - print $0 > log_file; fflush(log_file); - print package_prefix $0 > "/dev/fd/3"; fflush("/dev/fd/3"); - }' + "$REAL_NOX" "$@" 2>&1 | python3 -u -c ' + import os, sys + prefix = f"[{sys.argv[1]}] " + with open(sys.argv[2], "w", buffering=1) as log_f, os.fdopen(3, "w", buffering=1, closefd=False) as live_f: + for line in sys.stdin: + log_f.write(line) + live_f.write(prefix + line) + ' "${PACKAGE_NAME}" "${TEMP_LOG_FILE}" EXIT_CODE=${PIPESTATUS[0]} # Calculate elapsed time and percentage of the target time budget spent on this package. @@ -172,8 +178,9 @@ steps: if [[ ${PROPOSED_WEIGHT} -lt 1 ]]; then PROPOSED_WEIGHT=1 fi - CONFIGURED_WEIGHT=$(echo "${PACKAGE_WEIGHTS:-}" | tr ' ' '\n' | awk -F: -v package_name="${PACKAGE_NAME}" ' - $1 == package_name { gsub(/[[:space:]]/, "", $2); if ($2 != "") print $2 } + CONFIGURED_WEIGHT=$(printf '%s\n' "${PACKAGE_WEIGHTS:-}" | awk -F: -v package_name="${PACKAGE_NAME}" ' + { gsub(/^[[:space:]]+|[[:space:]]+$/, "", $1); gsub(/[[:space:]]/, "", $2) } + $1 == package_name && $2 != "" { print $2 } ' | tail -n 1) CONFIGURED_WEIGHT="${CONFIGURED_WEIGHT:-1}" WEIGHT_DIFFERENCE=$(( PROPOSED_WEIGHT - CONFIGURED_WEIGHT )) @@ -222,6 +229,7 @@ timeout: 1800s substitutions: _MAX_SHARDS: '1' _SHARD_INDEX: '0' + _MIN_SHARD_WEIGHT: '1' _PARALLEL_WORKERS: '4' _TIME_BUDGET_MINUTES: '20' _HARD_TIMEOUT_MINUTES: '30' diff --git a/packages/google-cloud-logging/tests/system/gapic/v2/test_system_logging_service_v2_v2.py b/packages/google-cloud-logging/tests/system/gapic/v2/test_system_logging_service_v2_v2.py index d08703d48548..6fe019ae02ef 100644 --- a/packages/google-cloud-logging/tests/system/gapic/v2/test_system_logging_service_v2_v2.py +++ b/packages/google-cloud-logging/tests/system/gapic/v2/test_system_logging_service_v2_v2.py @@ -31,10 +31,13 @@ def test_write_log_entries(self): # Guard: Skip if project or credentials are missing # This prevents the "prerelease_deps" job from failing - if not project_id or "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ: - pytest.skip( - "System tests require GOOGLE_CLOUD_PROJECT and GOOGLE_APPLICATION_CREDENTIALS" - ) + if not project_id: + pytest.skip("System tests require GOOGLE_CLOUD_PROJECT or PROJECT_ID") + + try: + google.auth.default() + except google.auth.exceptions.DefaultCredentialsError: + pytest.skip("System tests require Application Default Credentials") # Instantiate the client # Since we've confirmed the env var exists, this call is now safe diff --git a/packages/google-cloud-logging/tests/system/test_system.py b/packages/google-cloud-logging/tests/system/test_system.py index 52280aa35ee9..2b29bcbb255c 100644 --- a/packages/google-cloud-logging/tests/system/test_system.py +++ b/packages/google-cloud-logging/tests/system/test_system.py @@ -105,18 +105,23 @@ class Config(object): def setUpModule(): + import google.auth + import google.auth.exceptions + # Use GOOGLE_CLOUD_PROJECT or PROJECT_ID project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") or os.environ.get("PROJECT_ID") - # Check if we have credentials (either via file or environment) - # google.auth.default() will check GOOGLE_APPLICATION_CREDENTIALS - has_creds = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") is not None + if not project_id: + raise unittest.SkipTest( + "System tests skipped: GOOGLE_CLOUD_PROJECT or PROJECT_ID must be set." + ) - # Guard: Skip the entire module if the environment is not ready - if not project_id or not has_creds: + # Check if we have credentials (either via GOOGLE_APPLICATION_CREDENTIALS or ADC) + try: + google.auth.default() + except google.auth.exceptions.DefaultCredentialsError: raise unittest.SkipTest( - "System tests skipped: GOOGLE_CLOUD_PROJECT and " - "GOOGLE_APPLICATION_CREDENTIALS must be set." + "System tests skipped: Application Default Credentials must be available." ) # Explicitly pass the project to the client diff --git a/packages/google-resumable-media/tests/system/requests/conftest.py b/packages/google-resumable-media/tests/system/requests/conftest.py index 54ae10a2b5ca..7ea27e473755 100644 --- a/packages/google-resumable-media/tests/system/requests/conftest.py +++ b/packages/google-resumable-media/tests/system/requests/conftest.py @@ -13,6 +13,8 @@ # limitations under the License. """py.test fixtures to be shared across multiple system test modules.""" +import os + import google.auth # type: ignore import google.auth.transport.requests as tr_requests # type: ignore import pytest # type: ignore @@ -24,7 +26,12 @@ def ensure_bucket(transport): get_response = transport.get(utils.BUCKET_URL) if get_response.status_code == 404: credentials = transport.credentials - query_params = {"project": credentials.project_id} + project_id = ( + getattr(credentials, "project_id", None) + or os.environ.get("GOOGLE_CLOUD_PROJECT") + or os.environ.get("PROJECT_ID") + ) + query_params = {"project": project_id} payload = {"name": utils.BUCKET_NAME} post_response = transport.post( utils.BUCKET_POST_URL, params=query_params, json=payload diff --git a/packages/google-resumable-media/tests_async/system/requests/conftest.py b/packages/google-resumable-media/tests_async/system/requests/conftest.py index f9acc483223e..68b1a62b1e28 100644 --- a/packages/google-resumable-media/tests_async/system/requests/conftest.py +++ b/packages/google-resumable-media/tests_async/system/requests/conftest.py @@ -13,6 +13,8 @@ # limitations under the License. """py.test fixtures to be shared across multiple system test modules.""" +import os + import google.auth.transport._aiohttp_requests as tr_requests # type: ignore import pytest # type: ignore from google.auth._default_async import default_async # type: ignore @@ -24,7 +26,12 @@ async def ensure_bucket(transport): get_response = await transport.request("GET", utils.BUCKET_URL) if get_response.status == 404: credentials = transport.credentials - query_params = {"project": credentials.project_id} + project_id = ( + getattr(credentials, "project_id", None) + or os.environ.get("GOOGLE_CLOUD_PROJECT") + or os.environ.get("PROJECT_ID") + ) + query_params = {"project": project_id} payload = {"name": utils.BUCKET_NAME} post_response = await transport.request( "POST", utils.BUCKET_POST_URL, params=query_params, json=payload From e5e421b79ac0c1d812ba94d3eb53f33f03d6c7ef Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 17 Nov 2026 20:03:53 +0000 Subject: [PATCH 10/29] work in progress --- ci/cloudbuild/system-test.yaml | 35 ++++++++++++++++--- .../samples/snippets/requirements.txt | 2 +- .../tests_async/system/requests/conftest.py | 8 +++++ 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index d15cb77d301e..9160597a373d 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -16,7 +16,9 @@ steps: - name: 'us-central1-docker.pkg.dev/client-library-releases/release-images-prod/python-multi@sha256:d529954c373542171f9fc411e0102cebe542b511e320a965edf8820285f4f122' id: 'run-system-tests-shard' env: - # Populate standard GCP project environment variables expected by system test suites. + # Populate standard GCP project and CI environment variables expected by system test suites. + - 'CI=true' + - 'RUN_COMPLIANCE_TESTS=false' - 'GOOGLE_CLOUD_PROJECT=${PROJECT_ID}' - 'GCLOUD_PROJECT=${PROJECT_ID}' - 'PROJECT_ID=${PROJECT_ID}' @@ -191,6 +193,7 @@ steps: WEIGHT_MESSAGE="Weight adjustment needed for ${PACKAGE_NAME}: configured=${CONFIGURED_WEIGHT}, proposed=${PROPOSED_WEIGHT} (${PACKAGE_NAME}:${PROPOSED_WEIGHT})" echo "[${PACKAGE_NAME}] ${WEIGHT_MESSAGE}" >&3 echo "${WEIGHT_MESSAGE}" + echo " ${PACKAGE_NAME}: ${PROPOSED_WEIGHT} # was ${CONFIGURED_WEIGHT} (actual: $(( ELAPSED_SECONDS / 60 ))m $(printf '%02d' $(( ELAPSED_SECONDS % 60 )))s)" >> /tmp/cloudbuild_weight_adjustments.txt fi # Emit failure details to stdout (captured by run_conditional_tests.sh) only when tests fail @@ -219,6 +222,20 @@ steps: exec 3>&1 # Export PACKAGE_LIST and delegate execution to the existing monorepo test runner script. + rm -f /tmp/cloudbuild_weight_adjustments.txt + print_weight_summary() { + echo "" + echo "============================================================" + if [[ -s /tmp/cloudbuild_weight_adjustments.txt ]]; then + echo "⚠️ WEIGHT ADJUSTMENTS NEEDED (drifted >= 3m from configured weight):" + sort -u /tmp/cloudbuild_weight_adjustments.txt + else + echo "✅ WEIGHT SUMMARY: No weight adjustments needed (all packages within 2m of configured weight)." + fi + echo "============================================================" + } + trap print_weight_summary EXIT + export PACKAGE_LIST echo "=== System Shard ${SHARD_INDEX}/${MAX_SHARDS} running ${NOX_SESSION} for: ${PACKAGE_LIST} ===" ./ci/run_conditional_tests.sh @@ -227,9 +244,9 @@ steps: timeout: 1800s substitutions: - _MAX_SHARDS: '1' + _MAX_SHARDS: '5' _SHARD_INDEX: '0' - _MIN_SHARD_WEIGHT: '1' + _MIN_SHARD_WEIGHT: '10' _PARALLEL_WORKERS: '4' _TIME_BUDGET_MINUTES: '20' _HARD_TIMEOUT_MINUTES: '30' @@ -248,7 +265,17 @@ substitutions: packages/pandas-gbq packages/sqlalchemy-bigquery _NOX_SESSION: 'system-3.12' - _PACKAGE_WEIGHTS: '' + _PACKAGE_WEIGHTS: | + bigquery-magics: 2 + google-cloud-bigquery-storage: 8 + google-cloud-compute: 8 + google-cloud-datastore: 8 + google-cloud-dns: 1 + google-cloud-error-reporting: 1 + google-cloud-logging: 10 + google-resumable-media: 1 + pandas-gbq: 12 + sqlalchemy-bigquery: 11 options: machineType: 'E2_HIGHCPU_8' diff --git a/packages/google-cloud-bigquery-storage/samples/snippets/requirements.txt b/packages/google-cloud-bigquery-storage/samples/snippets/requirements.txt index 5b613fdf002f..4f6a4e7b35ca 100644 --- a/packages/google-cloud-bigquery-storage/samples/snippets/requirements.txt +++ b/packages/google-cloud-bigquery-storage/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ google-cloud-bigquery-storage==2.43.0 -google-cloud-bigquery==3.46.2 +google-cloud-bigquery==3.47.0 pyarrow==26.0.0 pytest==9.1.1 diff --git a/packages/google-resumable-media/tests_async/system/requests/conftest.py b/packages/google-resumable-media/tests_async/system/requests/conftest.py index 68b1a62b1e28..72b16a42703a 100644 --- a/packages/google-resumable-media/tests_async/system/requests/conftest.py +++ b/packages/google-resumable-media/tests_async/system/requests/conftest.py @@ -51,6 +51,14 @@ async def cleanup_bucket(transport): def _get_authorized_transport(): credentials, project_id = default_async(scopes=(utils.GCS_RW_SCOPE,)) + if not hasattr(credentials, "project_id") or not credentials.token: + import google.auth.transport.requests + import google.oauth2.credentials + + sync_creds, sync_project = google.auth.default(scopes=(utils.GCS_RW_SCOPE,)) + sync_creds.refresh(google.auth.transport.requests.Request()) + credentials = google.oauth2.credentials.Credentials(token=sync_creds.token) + credentials.project_id = getattr(sync_creds, "project_id", None) or sync_project return tr_requests.AuthorizedSession(credentials) From 6868b6e912e36f1fcc2f80a1764bea284171029b Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 17 Nov 2026 20:40:50 +0000 Subject: [PATCH 11/29] work in progress --- .../snippets/query_and_wait_arrow_test.py | 16 +++++++++++++++- .../tests_async/system/requests/conftest.py | 13 ++++++++----- packages/pandas-gbq/tests/system/test_auth.py | 2 +- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/packages/google-cloud-bigquery-storage/samples/snippets/query_and_wait_arrow_test.py b/packages/google-cloud-bigquery-storage/samples/snippets/query_and_wait_arrow_test.py index ad7af73392b4..91ddd405d20e 100644 --- a/packages/google-cloud-bigquery-storage/samples/snippets/query_and_wait_arrow_test.py +++ b/packages/google-cloud-bigquery-storage/samples/snippets/query_and_wait_arrow_test.py @@ -12,12 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools + import pyarrow +from google.cloud import bigquery + from . import query_and_wait_arrow -def test_query_and_wait_arrow(): +def test_query_and_wait_arrow(monkeypatch): + # TODO: Remove use_query_cache=False workaround once google-cloud-bigquery > 3.47.0 + # (containing #18958) is released and pinned in requirements.txt. + monkeypatch.setattr( + query_and_wait_arrow.bigquery, + "Client", + functools.partial( + bigquery.Client, + default_query_job_config=bigquery.QueryJobConfig(use_query_cache=False), + ), + ) batches = query_and_wait_arrow.query_and_wait_arrow() total_rows = 0 diff --git a/packages/google-resumable-media/tests_async/system/requests/conftest.py b/packages/google-resumable-media/tests_async/system/requests/conftest.py index 72b16a42703a..ba2395707374 100644 --- a/packages/google-resumable-media/tests_async/system/requests/conftest.py +++ b/packages/google-resumable-media/tests_async/system/requests/conftest.py @@ -13,6 +13,7 @@ # limitations under the License. """py.test fixtures to be shared across multiple system test modules.""" +import inspect import os import google.auth.transport._aiohttp_requests as tr_requests # type: ignore @@ -51,20 +52,22 @@ async def cleanup_bucket(transport): def _get_authorized_transport(): credentials, project_id = default_async(scopes=(utils.GCS_RW_SCOPE,)) - if not hasattr(credentials, "project_id") or not credentials.token: + if not inspect.iscoroutinefunction(credentials.before_request): + import google.auth import google.auth.transport.requests - import google.oauth2.credentials + from google.oauth2 import _credentials_async sync_creds, sync_project = google.auth.default(scopes=(utils.GCS_RW_SCOPE,)) sync_creds.refresh(google.auth.transport.requests.Request()) - credentials = google.oauth2.credentials.Credentials(token=sync_creds.token) - credentials.project_id = getattr(sync_creds, "project_id", None) or sync_project + credentials = _credentials_async.Credentials(token=sync_creds.token) + credentials.project_id = ( + getattr(sync_creds, "project_id", None) or sync_project or project_id + ) return tr_requests.AuthorizedSession(credentials) @pytest.fixture(scope="module") async def authorized_transport(): - credentials, project_id = default_async(scopes=(utils.GCS_RW_SCOPE,)) yield _get_authorized_transport() diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py index 564625a92d73..264a63039864 100644 --- a/packages/pandas-gbq/tests/system/test_auth.py +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -11,7 +11,7 @@ from pandas_gbq import auth -IS_RUNNING_ON_CI = "KOKORO_BUILD_ID" in os.environ +IS_RUNNING_ON_CI = bool(os.environ.get("KOKORO_BUILD_ID") or os.environ.get("CI")) def mock_default_credentials(scopes=None, request=None): From 86b207827747580d3bb09de24bba31562c9ca93b Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 18 Nov 2026 15:32:46 +0000 Subject: [PATCH 12/29] fix build --- .../tests_async/system/requests/test_download.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/google-resumable-media/tests_async/system/requests/test_download.py b/packages/google-resumable-media/tests_async/system/requests/test_download.py index 5c5ac7d8582e..2602cfa6056f 100644 --- a/packages/google-resumable-media/tests_async/system/requests/test_download.py +++ b/packages/google-resumable-media/tests_async/system/requests/test_download.py @@ -23,7 +23,6 @@ import google.auth.transport._aiohttp_requests as tr_requests # type: ignore import multidict # type: ignore import pytest # type: ignore -from google.auth._default_async import default_async # type: ignore import google._async_resumable_media.requests as resumable_requests import google._async_resumable_media.requests.download as download_mod @@ -181,9 +180,8 @@ async def secret_file(authorized_transport, bucket): # Transport that returns corrupt data, so we can exercise checksum handling. @pytest.fixture(scope="module") -async def corrupting_transport(): - credentials, _ = default_async(scopes=(utils.GCS_RW_SCOPE,)) - yield CorruptingAuthorizedSession(credentials) +async def corrupting_transport(authorized_transport): + yield CorruptingAuthorizedSession(authorized_transport.credentials) @pytest.fixture(scope="module") From 00842a468e2b229ef6a9a205d6017c416dbb564a Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 18 Nov 2026 15:38:36 +0000 Subject: [PATCH 13/29] work in progress --- ci/cloudbuild/system-test.yaml | 59 +++++++++++++--------------------- 1 file changed, 22 insertions(+), 37 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 9160597a373d..fdaf26086e77 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -29,7 +29,7 @@ steps: - 'BUILD_TYPE=${_BUILD_TYPE}' - 'TARGET_BRANCH=${_BASE_BRANCH}' - 'TEST_ALL_PACKAGES=${_TEST_ALL_PACKAGES}' - - 'ENABLED_PACKAGES=${_ENABLED_PACKAGES}' + - 'DISABLED_PACKAGES=${_DISABLED_PACKAGES}' - 'PACKAGE_WEIGHTS=${_PACKAGE_WEIGHTS}' # Specify the nox session consumed by ci/run_conditional_tests.sh and ci/run_single_test.sh. - 'NOX_SESSION=${_NOX_SESSION}' @@ -58,7 +58,7 @@ steps: --depth=200 || true # Import ci/get_package_shards.py without modifying the script on disk. - # Filter get_packages() in memory to _ENABLED_PACKAGES so unmigrated packages are ignored. + # Filter get_packages() in memory to exclude _DISABLED_PACKAGES. PACKAGE_LIST=$(python3 -c ' import os import sys @@ -66,33 +66,28 @@ steps: sys.path.insert(0, "ci") import get_package_shards - # Normalize enabled package paths to include a trailing slash so they match - # the directory format ("packages//") returned by get_package_shards.get_packages(). - # When CI infrastructure (ci/, .github/) or core dependencies change, - # get_package_shards.get_packages_to_test() automatically selects all packages - # returned by get_packages(). Passing handwritten_only=False to original_get_packages - # ensures every package explicitly listed in _ENABLED_PACKAGES is included. - enabled_packages = { + # Normalize disabled package paths to include a trailing slash. + disabled_packages = { package_path.strip().rstrip("/") + "/" - for package_path in os.environ.get("ENABLED_PACKAGES", "").replace(",", " ").split() + for package_path in os.environ.get("DISABLED_PACKAGES", "").replace(",", " ").split() if package_path.strip() } - if enabled_packages: - original_get_packages = get_package_shards.get_packages - def filtered_get_packages(*args, **kwargs): - filtered_packages = {} - for package_name, package_paths in original_get_packages( - handwritten_only=False - ).items(): - kept_paths = [ - path for path in package_paths if path in enabled_packages - ] - if kept_paths: - filtered_packages[package_name] = kept_paths - return filtered_packages + original_get_packages = get_package_shards.get_packages + def filtered_get_packages(*args, **kwargs): + filtered_packages = {} + for package_name, package_paths in original_get_packages( + handwritten_only=False + ).items(): + # Keep only paths that are NOT in the disabled_packages set + kept_paths = [ + path for path in package_paths if path not in disabled_packages + ] + if kept_paths: + filtered_packages[package_name] = kept_paths + return filtered_packages - get_package_shards.get_packages = filtered_get_packages + get_package_shards.get_packages = filtered_get_packages # Compute the packages that need testing and distribute them across balanced shards. packages_to_test = get_package_shards.get_packages_to_test() @@ -102,9 +97,9 @@ steps: print(shards[shard_index].get("packages", "") if shard_index < len(shards) else "") ') - # Report immediate green (< 10s) if no enabled packages in this PR map to this shard index. + # Report immediate green (< 10s) if no packages in this PR map to this shard index. if [[ -z "${PACKAGE_LIST}" ]]; then - echo "No enabled packages assigned to shard ${SHARD_INDEX}. Exiting 0." + echo "No packages assigned to shard ${SHARD_INDEX}. Exiting 0." exit 0 fi @@ -253,17 +248,7 @@ substitutions: _BUILD_TYPE: 'presubmit' _BASE_BRANCH: 'main' _TEST_ALL_PACKAGES: 'false' - _ENABLED_PACKAGES: >- - packages/bigquery-magics - packages/google-cloud-bigquery-storage - packages/google-cloud-compute - packages/google-cloud-datastore - packages/google-cloud-dns - packages/google-cloud-error-reporting - packages/google-cloud-logging - packages/google-resumable-media - packages/pandas-gbq - packages/sqlalchemy-bigquery + _DISABLED_PACKAGES: '' _NOX_SESSION: 'system-3.12' _PACKAGE_WEIGHTS: | bigquery-magics: 2 From c1b24d411f1c0cda75fc41276948a50987e1918c Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 18 Nov 2026 16:13:39 +0000 Subject: [PATCH 14/29] work in progress --- ci/cloudbuild/system-test.yaml | 37 ++++++++++++++++--- .../system/requests/conftest.py | 9 ++++- .../tests/system/conftest.py | 2 +- .../tests/system/test_bucket.py | 1 - 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index fdaf26086e77..523986a6ffb5 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -73,15 +73,23 @@ steps: if package_path.strip() } + def has_system_tests(package_path): + return package_path.startswith("packages/") and any( + os.path.exists(os.path.join(package_path, rel_path)) + for rel_path in ("tests/system", "system_tests", "tests/system.py") + ) + original_get_packages = get_package_shards.get_packages def filtered_get_packages(*args, **kwargs): filtered_packages = {} for package_name, package_paths in original_get_packages( handwritten_only=False ).items(): - # Keep only paths that are NOT in the disabled_packages set + # Keep only packages/ paths with system tests that are NOT in disabled_packages kept_paths = [ - path for path in package_paths if path not in disabled_packages + path + for path in package_paths + if path not in disabled_packages and has_system_tests(path) ] if kept_paths: filtered_packages[package_name] = kept_paths @@ -106,15 +114,27 @@ steps: # Create runtime shims in /tmp/cloudbuild_shims so tracked repository files remain unmodified. mkdir -p /tmp/cloudbuild_shims - # 1. Pytest plugin (cloudbuild_pytest_plugin.py): + # 1. Python startup hook (sitecustomize.py) & Pytest plugin (cloudbuild_pytest_plugin.py): # - Package noxfile.py files skip the system session if GOOGLE_APPLICATION_CREDENTIALS # is unset. We export a placeholder value ("CLOUD_BUILD_ADC") to satisfy noxfile.py, - # then remove it here at pytest startup so google.auth.default() automatically uses - # the Cloud Build worker VM's Application Default Credentials (ADC). + # then remove it in sitecustomize.py for every non-nox Python subprocess (such as + # sqlalchemy-spanner's create_test_database.py and pytest) so google.auth.default() + # uses the Cloud Build worker VM's Application Default Credentials (ADC). # - Package noxfile.py files invoke `py.test --quiet`, which only outputs dots (".") # without newlines and causes Cloud Build logs to appear frozen for several minutes. # Overriding `quiet = 0` and `verbose = 1` makes pytest print one newline-terminated # status line as each individual test finishes. + cat << 'EOF' > /tmp/cloudbuild_shims/sitecustomize.py + import os + import sys + + if ( + os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") == "CLOUD_BUILD_ADC" + and os.path.basename(sys.argv[0]) != "nox" + ): + os.environ.pop("GOOGLE_APPLICATION_CREDENTIALS", None) + EOF + cat << 'EOF' > /tmp/cloudbuild_shims/cloudbuild_pytest_plugin.py import os @@ -248,16 +268,21 @@ substitutions: _BUILD_TYPE: 'presubmit' _BASE_BRANCH: 'main' _TEST_ALL_PACKAGES: 'false' - _DISABLED_PACKAGES: '' + _DISABLED_PACKAGES: 'packages/gapic-generator/ packages/google-auth/ packages/google-cloud-firestore/' _NOX_SESSION: 'system-3.12' _PACKAGE_WEIGHTS: | bigquery-magics: 2 + google-cloud-bigquery: 5 google-cloud-bigquery-storage: 8 google-cloud-compute: 8 google-cloud-datastore: 8 google-cloud-dns: 1 google-cloud-error-reporting: 1 google-cloud-logging: 10 + google-cloud-ndb: 12 + google-cloud-pubsub: 15 + google-cloud-spanner: 17 + google-cloud-storage: 5 google-resumable-media: 1 pandas-gbq: 12 sqlalchemy-bigquery: 11 diff --git a/packages/google-cloud-storage/tests/resumable_media/system/requests/conftest.py b/packages/google-cloud-storage/tests/resumable_media/system/requests/conftest.py index 67908795bded..8e3751db0833 100644 --- a/packages/google-cloud-storage/tests/resumable_media/system/requests/conftest.py +++ b/packages/google-cloud-storage/tests/resumable_media/system/requests/conftest.py @@ -13,6 +13,8 @@ # limitations under the License. """py.test fixtures to be shared across multiple system test modules.""" +import os + import google.auth # type: ignore import google.auth.transport.requests as tr_requests # type: ignore import pytest # type: ignore @@ -24,7 +26,12 @@ def ensure_bucket(transport): get_response = transport.get(utils.BUCKET_URL) if get_response.status_code == 404: credentials = transport.credentials - query_params = {"project": credentials.project_id} + project_id = ( + getattr(credentials, "project_id", None) + or os.environ.get("GOOGLE_CLOUD_PROJECT") + or os.environ.get("PROJECT_ID") + ) + query_params = {"project": project_id} payload = {"name": utils.BUCKET_NAME} post_response = transport.post( utils.BUCKET_POST_URL, params=query_params, json=payload diff --git a/packages/google-cloud-storage/tests/system/conftest.py b/packages/google-cloud-storage/tests/system/conftest.py index a74fd74f9652..827b1157367f 100644 --- a/packages/google-cloud-storage/tests/system/conftest.py +++ b/packages/google-cloud-storage/tests/system/conftest.py @@ -245,7 +245,7 @@ def signing_bucket_name(): @pytest.fixture(scope="session") -def signing_bucket(storage_client, signing_bucket_name): +def signing_bucket(storage_client, signing_bucket_name, service_account): bucket = storage_client.bucket(signing_bucket_name) # Create the bucket only if it doesn't yet exist. try: diff --git a/packages/google-cloud-storage/tests/system/test_bucket.py b/packages/google-cloud-storage/tests/system/test_bucket.py index ae194fe46793..39c65ae95237 100644 --- a/packages/google-cloud-storage/tests/system/test_bucket.py +++ b/packages/google-cloud-storage/tests/system/test_bucket.py @@ -456,7 +456,6 @@ def test_bucket_copy_blob_w_metageneration_match( payload = b"DEADBEEF" bucket_name = _helpers.unique_name("generation-match") bucket = storage_client.bucket(bucket_name) - bucket.requester_pays = True created = _helpers.retry_429_503(storage_client.create_bucket)(bucket) buckets_to_delete.append(created) assert created.name == bucket_name From d3583c865be32531545856d0b3f9227d16b83ee2 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 18 Nov 2026 16:46:24 +0000 Subject: [PATCH 15/29] work in progress --- .../bigtable-integration.yaml | 27 +++++++++++++ ci/cloudbuild/system-test.yaml | 6 +++ packages/google-cloud-bigtable/noxfile.py | 27 +++++++++++++ .../google-cloud-bigtable/samples/conftest.py | 5 +++ .../google-cloud-bigtable/samples/utils.py | 40 ++++++++++++++----- .../tests/system/data/test_metrics_async.py | 6 ++- .../tests/system/data/test_metrics_autogen.py | 6 ++- .../tests/system/v2_client/test_data_api.py | 12 +++--- 8 files changed, 111 insertions(+), 18 deletions(-) diff --git a/.librarian/generator-input/client-post-processing/bigtable-integration.yaml b/.librarian/generator-input/client-post-processing/bigtable-integration.yaml index 7bb45a26eaea..af3fcfcf9270 100644 --- a/.librarian/generator-input/client-post-processing/bigtable-integration.yaml +++ b/.librarian/generator-input/client-post-processing/bigtable-integration.yaml @@ -405,6 +405,7 @@ replacements: SYSTEM_TEST_STANDARD_DEPENDENCIES: List[str] = [ "mock", "pytest", + "pytest-xdist", "google-cloud-testutils", ] SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [ @@ -696,6 +697,19 @@ replacements: test_map[test_type](session) + def _has_xdist_arg(posargs): + return any( + arg == "-n" + or ( + arg.startswith("-n") + and (arg[2:].isdigit() or arg[2:] in ("auto", "logical")) + ) + or arg == "--numprocesses" + or arg.startswith("--numprocesses=") + for arg in posargs + ) + + def system_default(session): """Run the system test suite.""" constraints_path = str( @@ -729,9 +743,18 @@ replacements: *session.posargs, ) if system_test_folder_exists: + in_emulator = bool( + session.env.get("BIGTABLE_EMULATOR_HOST") + or os.environ.get("BIGTABLE_EMULATOR_HOST") + ) + has_xdist_arg = _has_xdist_arg(session.posargs) + xdist_args = ( + [] if (in_emulator or has_xdist_arg) else ["-n", "8", "--dist=loadfile"] + ) session.run( "py.test", "--quiet", + *xdist_args, f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, *session.posargs, @@ -752,6 +775,7 @@ replacements: "mock", "pytest", "pytest-asyncio", + "pytest-xdist", *req_args, ) # Install the library from this checkout last so the samples exercise the @@ -761,9 +785,12 @@ replacements: has_target = any(not arg.startswith("-") for arg in session.posargs) target_paths = [] if has_target else [os.path.join("samples")] + has_xdist_arg = _has_xdist_arg(session.posargs) + xdist_args = [] if has_xdist_arg else ["-n", "8", "--dist=loadfile"] session.run( "py.test", "--quiet", + *xdist_args, f"--junitxml=samples_{session.python}_sponge_log.xml", "--ignore=samples/legacy_client/beam", *target_paths, diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 523986a6ffb5..66cfff2a34e8 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -18,6 +18,8 @@ steps: env: # Populate standard GCP project and CI environment variables expected by system test suites. - 'CI=true' + - 'PIP_BREAK_SYSTEM_PACKAGES=1' + - 'UV_VENV_SEED=1' - 'RUN_COMPLIANCE_TESTS=false' - 'GOOGLE_CLOUD_PROJECT=${PROJECT_ID}' - 'GCLOUD_PROJECT=${PROJECT_ID}' @@ -111,6 +113,9 @@ steps: exit 0 fi + # Install uv so nox sessions use fast cached wheel installation instead of slow virtualenv + pip. + python3 -m pip install --quiet uv || true + # Create runtime shims in /tmp/cloudbuild_shims so tracked repository files remain unmodified. mkdir -p /tmp/cloudbuild_shims @@ -274,6 +279,7 @@ substitutions: bigquery-magics: 2 google-cloud-bigquery: 5 google-cloud-bigquery-storage: 8 + google-cloud-bigtable: 15 google-cloud-compute: 8 google-cloud-datastore: 8 google-cloud-dns: 1 diff --git a/packages/google-cloud-bigtable/noxfile.py b/packages/google-cloud-bigtable/noxfile.py index 045dcc166a38..dff330ad3691 100644 --- a/packages/google-cloud-bigtable/noxfile.py +++ b/packages/google-cloud-bigtable/noxfile.py @@ -54,6 +54,7 @@ SYSTEM_TEST_STANDARD_DEPENDENCIES: List[str] = [ "mock", "pytest", + "pytest-xdist", "google-cloud-testutils", ] SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [ @@ -345,6 +346,19 @@ def system(session, test_type): test_map[test_type](session) +def _has_xdist_arg(posargs): + return any( + arg == "-n" + or ( + arg.startswith("-n") + and (arg[2:].isdigit() or arg[2:] in ("auto", "logical")) + ) + or arg == "--numprocesses" + or arg.startswith("--numprocesses=") + for arg in posargs + ) + + def system_default(session): """Run the system test suite.""" constraints_path = str( @@ -378,9 +392,18 @@ def system_default(session): *session.posargs, ) if system_test_folder_exists: + in_emulator = bool( + session.env.get("BIGTABLE_EMULATOR_HOST") + or os.environ.get("BIGTABLE_EMULATOR_HOST") + ) + has_xdist_arg = _has_xdist_arg(session.posargs) + xdist_args = ( + [] if (in_emulator or has_xdist_arg) else ["-n", "8", "--dist=loadfile"] + ) session.run( "py.test", "--quiet", + *xdist_args, f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, *session.posargs, @@ -401,6 +424,7 @@ def samples(session): "mock", "pytest", "pytest-asyncio", + "pytest-xdist", *req_args, ) # Install the library from this checkout last so the samples exercise the @@ -410,9 +434,12 @@ def samples(session): has_target = any(not arg.startswith("-") for arg in session.posargs) target_paths = [] if has_target else [os.path.join("samples")] + has_xdist_arg = _has_xdist_arg(session.posargs) + xdist_args = [] if has_xdist_arg else ["-n", "8", "--dist=loadfile"] session.run( "py.test", "--quiet", + *xdist_args, f"--junitxml=samples_{session.python}_sponge_log.xml", "--ignore=samples/legacy_client/beam", *target_paths, diff --git a/packages/google-cloud-bigtable/samples/conftest.py b/packages/google-cloud-bigtable/samples/conftest.py index 4c23415ab7b4..9c4bcf05612d 100644 --- a/packages/google-cloud-bigtable/samples/conftest.py +++ b/packages/google-cloud-bigtable/samples/conftest.py @@ -76,6 +76,11 @@ def pytest_configure(config) -> None: ): return + # Under pytest-xdist, only the controller provisions and tears down the + # shared sample test instance; workers inherit BIGTABLE_INSTANCE via os.environ. + if hasattr(config, "workerinput"): + return + project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") if not project_id: raise pytest.UsageError( diff --git a/packages/google-cloud-bigtable/samples/utils.py b/packages/google-cloud-bigtable/samples/utils.py index a28eb58d80ed..23b0c3f2feec 100644 --- a/packages/google-cloud-bigtable/samples/utils.py +++ b/packages/google-cloud-bigtable/samples/utils.py @@ -14,11 +14,14 @@ Provides helper logic used across samples """ +import inspect + from google.api_core import exceptions from google.api_core.retry import Retry, if_exception_type from google.cloud import bigtable, bigtable_admin_v2 from google.cloud.bigtable.column_family import ColumnFamily +from google.cloud.bigtable.data import BigtableDataClient, ReadRowsQuery from google.cloud.bigtable_admin_v2.types import ColumnFamily as ColumnFamily_pb delete_retry = Retry( @@ -93,7 +96,7 @@ def create_table(project, instance_id, table_id, column_families={}): @Retry( - on_error=if_exception_type( + predicate=if_exception_type( exceptions.PreconditionFailed, exceptions.FailedPrecondition, exceptions.NotFound, @@ -109,13 +112,28 @@ def wait_for_table(table): """ if hasattr(table, "exists"): if not table.exists(): - raise exceptions.NotFound - else: - table_path = table.table_name if hasattr(table, "table_name") else str(table) - admin_client = bigtable_admin_v2.BigtableTableAdminClient() - admin_client.get_table( - request={ - "name": table_path, - "view": bigtable_admin_v2.Table.View.NAME_ONLY, - } - ) + raise exceptions.NotFound("Table does not exist yet") + return + + # Data-client Table / TableAsync instances and raw table_path strings (e.g. + # from create_table) do not expose .exists(), so verify via admin + data plane. + table_path = table.table_name if hasattr(table, "table_name") else str(table) + admin_client = bigtable_admin_v2.BigtableTableAdminClient() + admin_client.get_table( + request={ + "name": table_path, + "view": bigtable_admin_v2.Table.View.NAME_ONLY, + } + ) + + # Probe the data plane as well: get_table() can succeed while mutations + # still return FailedPrecondition ("table ... is still being created"). + if hasattr(table, "read_rows") and not inspect.iscoroutinefunction(table.read_rows): + table.read_rows(ReadRowsQuery(limit=1)) + return + + parts = table_path.split("/") + if len(parts) == 6 and parts[0] == "projects": + with BigtableDataClient(project=parts[1]) as data_client: + with data_client.get_table(parts[3], parts[5]) as data_table: + data_table.read_rows(ReadRowsQuery(limit=1)) diff --git a/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py b/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py index 34f6ccf3c2a9..34cd0d94ccec 100644 --- a/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py +++ b/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py @@ -124,6 +124,10 @@ async def __anext__(self): if not self._raised: self._raised = True if self._exc: + if callable(getattr(self._call, "cancel", None)): + self._call.cancel() + elif callable(getattr(self._call, "close", None)): + self._call.close() raise self._exc return await self._call.__anext__() @@ -917,7 +921,7 @@ async def test_read_rows_sharded_failure_timeout(self, table, temp_rows, handler query2 = ReadRowsQuery(row_keys=[b"b"]) handler.clear() with pytest.raises(ShardedReadRowsExceptionGroup) as e: - await table.read_rows_sharded([query1, query2], operation_timeout=0.005) + await table.read_rows_sharded([query1, query2], operation_timeout=0.0001) assert len(e.value.exceptions) == 2 for sub_exc in e.value.exceptions: assert isinstance(sub_exc.__cause__, DeadlineExceeded) diff --git a/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py b/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py index 7daf8ba2d03c..765d13e7a930 100644 --- a/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py +++ b/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py @@ -106,6 +106,10 @@ def __next__(self): if not self._raised: self._raised = True if self._exc: + if callable(getattr(self._call, "cancel", None)): + self._call.cancel() + elif callable(getattr(self._call, "close", None)): + self._call.close() raise self._exc return self._call.__next__() @@ -770,7 +774,7 @@ def test_read_rows_sharded_failure_timeout(self, table, temp_rows, handler): query2 = ReadRowsQuery(row_keys=[b"b"]) handler.clear() with pytest.raises(ShardedReadRowsExceptionGroup) as e: - table.read_rows_sharded([query1, query2], operation_timeout=0.005) + table.read_rows_sharded([query1, query2], operation_timeout=0.0001) assert len(e.value.exceptions) == 2 for sub_exc in e.value.exceptions: assert isinstance(sub_exc.__cause__, DeadlineExceeded) diff --git a/packages/google-cloud-bigtable/tests/system/v2_client/test_data_api.py b/packages/google-cloud-bigtable/tests/system/v2_client/test_data_api.py index 3a45224d390c..59bfe1915d25 100644 --- a/packages/google-cloud-bigtable/tests/system/v2_client/test_data_api.py +++ b/packages/google-cloud-bigtable/tests/system/v2_client/test_data_api.py @@ -356,7 +356,9 @@ def test_table_mutate_rows_retries_timeout(data_table, rows_to_delete): rows_to_delete.append(row_2) row_2.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL1, timestamp=timestamp) - statuses = data_table.mutate_rows([row, row_2]) + statuses = data_table.mutate_rows( + [row, row_2], retry=DEFAULT_RETRY.with_deadline(2.0) + ) assert statuses[0].code == Code.OK assert statuses[1].code == Code.DEADLINE_EXCEEDED @@ -940,10 +942,10 @@ def test_table_read_rows_retry_timeout_mid_stream( # Shorten the deadline so the timeout test is shorter. data_table_read_rows_retry_tests._table_impl.default_read_rows_operation_timeout = ( - 10.0 + 2.0 ) rows_data = data_table_read_rows_retry_tests.read_rows( - retry=DEFAULT_RETRY_READ_ROWS.with_deadline(10.0) + retry=DEFAULT_RETRY_READ_ROWS.with_deadline(2.0) ) with pytest.raises(exceptions.RetryError): rows_data.consume_all() @@ -971,10 +973,10 @@ def test_table_read_rows_retry_timeout_establishing_stream( # Shorten the deadline so the timeout test is shorter. data_table_read_rows_retry_tests._table_impl.default_read_rows_operation_timeout = ( - 10.0 + 2.0 ) rows_data = data_table_read_rows_retry_tests.read_rows( - retry=DEFAULT_RETRY_READ_ROWS.with_deadline(10.0) + retry=DEFAULT_RETRY_READ_ROWS.with_deadline(2.0) ) with pytest.raises(exceptions.RetryError): rows_data.consume_all() From abc1e8fa88061b6fb2ba9e8021fc517272681129 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 18 Nov 2026 17:09:13 +0000 Subject: [PATCH 16/29] work in progress --- ci/cloudbuild/system-test.yaml | 2 +- .../samples/snippets/authenticate_service_account_test.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 66cfff2a34e8..ba1eddf3f377 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -277,7 +277,7 @@ substitutions: _NOX_SESSION: 'system-3.12' _PACKAGE_WEIGHTS: | bigquery-magics: 2 - google-cloud-bigquery: 5 + google-cloud-bigquery: 8 google-cloud-bigquery-storage: 8 google-cloud-bigtable: 15 google-cloud-compute: 8 diff --git a/packages/google-cloud-bigquery/samples/snippets/authenticate_service_account_test.py b/packages/google-cloud-bigquery/samples/snippets/authenticate_service_account_test.py index fbdd2d0649ff..9c3ef374d304 100644 --- a/packages/google-cloud-bigquery/samples/snippets/authenticate_service_account_test.py +++ b/packages/google-cloud-bigquery/samples/snippets/authenticate_service_account_test.py @@ -24,9 +24,11 @@ def mock_credentials(*args: Any, **kwargs: Any) -> google.auth.credentials.Credentials: - credentials, _ = google.auth.default( + credentials, project_id = google.auth.default( ["https://www.googleapis.com/auth/cloud-platform"] ) + if not hasattr(credentials, "project_id"): + credentials.project_id = project_id # type: ignore[attr-defined] return credentials From f0ad6443ff1cf48eafbb0d38318eb6a98d73f48b Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 18 Nov 2026 18:20:06 +0000 Subject: [PATCH 17/29] work in progress --- .../tests/system/data/test_metrics_async.py | 56 +++++++++++++++---- .../tests/system/data/test_metrics_autogen.py | 45 +++++++++++---- 2 files changed, 81 insertions(+), 20 deletions(-) diff --git a/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py b/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py index 34cd0d94ccec..e2b7c3d6c84b 100644 --- a/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py +++ b/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py @@ -91,16 +91,52 @@ def clear(self): self._exc_list.clear() self.fail_mid_stream = False + def _make_deadline_exceeded(self): + if CrossSync.is_async: + return AioRpcError(StatusCode.DEADLINE_EXCEEDED, Metadata(), Metadata()) + else: + exc = RpcError(StatusCode.DEADLINE_EXCEEDED) + exc.trailing_metadata = lambda: [] + exc.initial_metadata = lambda: [] + exc.code = lambda: StatusCode.DEADLINE_EXCEEDED + exc.details = lambda: "Deadline Exceeded" + + def _result(): + raise exc + + exc.result = _result + return exc + @CrossSync.convert async def intercept_unary_unary(self, continuation, client_call_details, request): if self._exc_list: raise self._exc_list.pop(0) + # Deterministic client-side timeout for *_failure_timeout tests: + # When the CI worker and Bigtable cluster sit in the same GCP region + # (e.g. Cloud Build in us-central1 with <1ms RTT), tiny 1–5ms timeouts + # race against Bigtable returning server headers (setting cluster_id) on + # one hand, and expiring inside Python's local shard-dispatch loop before + # all shards start on the other. Using a 50ms timeout gives Python ample + # headroom to start all sub-requests under xdist CPU load, while sleeping + # past the deadline here guarantees DEADLINE_EXCEEDED with no server headers. + if ( + client_call_details.timeout is not None + and client_call_details.timeout <= 0.1 + ): + await CrossSync.sleep(client_call_details.timeout + 0.01) + raise self._make_deadline_exceeded() return await continuation(client_call_details, request) @CrossSync.convert async def intercept_unary_stream(self, continuation, client_call_details, request): if not self.fail_mid_stream and self._exc_list: raise self._exc_list.pop(0) + if ( + client_call_details.timeout is not None + and client_call_details.timeout <= 0.1 + ): + await CrossSync.sleep(client_call_details.timeout + 0.01) + raise self._make_deadline_exceeded() response = await continuation(client_call_details, request) @@ -333,7 +369,7 @@ async def test_read_rows_failure_timeout(self, table, temp_rows, handler): await temp_rows.add_row(b"row_key_1") handler.clear() with pytest.raises(GoogleAPICallError): - await table.read_rows(ReadRowsQuery(), operation_timeout=0.001) + await table.read_rows(ReadRowsQuery(), operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -525,7 +561,7 @@ async def test_read_rows_stream_failure_timeout(self, table, temp_rows, handler) await temp_rows.add_row(b"row_key_1") handler.clear() generator = await table.read_rows_stream( - ReadRowsQuery(), operation_timeout=0.001 + ReadRowsQuery(), operation_timeout=0.05 ) with pytest.raises(GoogleAPICallError): [_ async for _ in generator] @@ -752,7 +788,7 @@ async def test_read_row_failure_timeout(self, table, temp_rows, handler): await temp_rows.add_row(b"row_key_1") handler.clear() with pytest.raises(GoogleAPICallError): - await table.read_row(b"row_key_1", operation_timeout=0.001) + await table.read_row(b"row_key_1", operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -921,7 +957,7 @@ async def test_read_rows_sharded_failure_timeout(self, table, temp_rows, handler query2 = ReadRowsQuery(row_keys=[b"b"]) handler.clear() with pytest.raises(ShardedReadRowsExceptionGroup) as e: - await table.read_rows_sharded([query1, query2], operation_timeout=0.0001) + await table.read_rows_sharded([query1, query2], operation_timeout=0.05) assert len(e.value.exceptions) == 2 for sub_exc in e.value.exceptions: assert isinstance(sub_exc.__cause__, DeadlineExceeded) @@ -1163,7 +1199,7 @@ async def test_bulk_mutate_rows_failure_timeout(self, table, temp_rows, handler) handler.clear() with pytest.raises(MutationsExceptionGroup): - await table.bulk_mutate_rows([entry], operation_timeout=0.001) + await table.bulk_mutate_rows([entry], operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -1386,7 +1422,7 @@ async def test_mutate_rows_batcher_failure_timeout(self, table, temp_rows, handl with pytest.raises(MutationsExceptionGroup): async with table.mutations_batcher( - batch_operation_timeout=0.001 + batch_operation_timeout=0.05 ) as batcher: await batcher.append(entry) # validate counts @@ -1563,7 +1599,7 @@ async def test_mutate_row_failure_timeout(self, table, temp_rows, handler): mutation = SetCell(TEST_FAMILY, b"q", b"v") with pytest.raises(GoogleAPICallError): - await table.mutate_row(row_key, [mutation], operation_timeout=0.001) + await table.mutate_row(row_key, [mutation], operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -1805,7 +1841,7 @@ async def test_sample_row_keys_failure_timeout(self, table, handler): No grpc headers expected """ with pytest.raises(GoogleAPICallError): - await table.sample_row_keys(operation_timeout=0.001) + await table.sample_row_keys(operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -1971,7 +2007,7 @@ async def test_read_modify_write_failure_timeout(self, table, temp_rows, handler await temp_rows.add_row(row_key, value=0, family=family, qualifier=qualifier) rule = IncrementRule(family, qualifier, 1) with pytest.raises(GoogleAPICallError): - await table.read_modify_write_row(row_key, rule, operation_timeout=0.001) + await table.read_modify_write_row(row_key, rule, operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -2162,7 +2198,7 @@ async def test_check_and_mutate_row_failure_timeout( row_key, predicate=ValueRangeFilter(0, 2), true_case_mutations=true_mutation, - operation_timeout=0.001, + operation_timeout=0.05, ) # validate counts assert len(handler.completed_operations) == 1 diff --git a/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py b/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py index 765d13e7a930..d1d2cfe531fe 100644 --- a/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py +++ b/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py @@ -81,14 +81,39 @@ def clear(self): self._exc_list.clear() self.fail_mid_stream = False + def _make_deadline_exceeded(self): + exc = RpcError(StatusCode.DEADLINE_EXCEEDED) + exc.trailing_metadata = lambda: [] + exc.initial_metadata = lambda: [] + exc.code = lambda: StatusCode.DEADLINE_EXCEEDED + exc.details = lambda: "Deadline Exceeded" + + def _result(): + raise exc + + exc.result = _result + return exc + def intercept_unary_unary(self, continuation, client_call_details, request): if self._exc_list: raise self._exc_list.pop(0) + if ( + client_call_details.timeout is not None + and client_call_details.timeout <= 0.1 + ): + CrossSync._Sync_Impl.sleep(client_call_details.timeout + 0.01) + raise self._make_deadline_exceeded() return continuation(client_call_details, request) def intercept_unary_stream(self, continuation, client_call_details, request): if not self.fail_mid_stream and self._exc_list: raise self._exc_list.pop(0) + if ( + client_call_details.timeout is not None + and client_call_details.timeout <= 0.1 + ): + CrossSync._Sync_Impl.sleep(client_call_details.timeout + 0.01) + raise self._make_deadline_exceeded() response = continuation(client_call_details, request) if self.fail_mid_stream and self._exc_list: exc = self._exc_list.pop(0) @@ -280,7 +305,7 @@ def test_read_rows_failure_timeout(self, table, temp_rows, handler): temp_rows.add_row(b"row_key_1") handler.clear() with pytest.raises(GoogleAPICallError): - table.read_rows(ReadRowsQuery(), operation_timeout=0.001) + table.read_rows(ReadRowsQuery(), operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -439,7 +464,7 @@ def test_read_rows_stream_failure_timeout(self, table, temp_rows, handler): No grpc headers expected""" temp_rows.add_row(b"row_key_1") handler.clear() - generator = table.read_rows_stream(ReadRowsQuery(), operation_timeout=0.001) + generator = table.read_rows_stream(ReadRowsQuery(), operation_timeout=0.05) with pytest.raises(GoogleAPICallError): [_ for _ in generator] assert len(handler.completed_operations) == 1 @@ -628,7 +653,7 @@ def test_read_row_failure_timeout(self, table, temp_rows, handler): temp_rows.add_row(b"row_key_1") handler.clear() with pytest.raises(GoogleAPICallError): - table.read_row(b"row_key_1", operation_timeout=0.001) + table.read_row(b"row_key_1", operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -774,7 +799,7 @@ def test_read_rows_sharded_failure_timeout(self, table, temp_rows, handler): query2 = ReadRowsQuery(row_keys=[b"b"]) handler.clear() with pytest.raises(ShardedReadRowsExceptionGroup) as e: - table.read_rows_sharded([query1, query2], operation_timeout=0.0001) + table.read_rows_sharded([query1, query2], operation_timeout=0.05) assert len(e.value.exceptions) == 2 for sub_exc in e.value.exceptions: assert isinstance(sub_exc.__cause__, DeadlineExceeded) @@ -975,7 +1000,7 @@ def test_bulk_mutate_rows_failure_timeout(self, table, temp_rows, handler): entry = RowMutationEntry(row_key, [mutation]) handler.clear() with pytest.raises(MutationsExceptionGroup): - table.bulk_mutate_rows([entry], operation_timeout=0.001) + table.bulk_mutate_rows([entry], operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -1159,7 +1184,7 @@ def test_mutate_rows_batcher_failure_timeout(self, table, temp_rows, handler): mutation = SetCell(TEST_FAMILY, b"q", b"v") entry = RowMutationEntry(row_key, [mutation]) with pytest.raises(MutationsExceptionGroup): - with table.mutations_batcher(batch_operation_timeout=0.001) as batcher: + with table.mutations_batcher(batch_operation_timeout=0.05) as batcher: batcher.append(entry) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -1306,7 +1331,7 @@ def test_mutate_row_failure_timeout(self, table, temp_rows, handler): row_key = b"row_key_1" mutation = SetCell(TEST_FAMILY, b"q", b"v") with pytest.raises(GoogleAPICallError): - table.mutate_row(row_key, [mutation], operation_timeout=0.001) + table.mutate_row(row_key, [mutation], operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -1475,7 +1500,7 @@ def test_sample_row_keys_failure_timeout(self, table, handler): No grpc headers expected""" with pytest.raises(GoogleAPICallError): - table.sample_row_keys(operation_timeout=0.001) + table.sample_row_keys(operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -1571,7 +1596,7 @@ def test_read_modify_write_failure_timeout(self, table, temp_rows, handler): temp_rows.add_row(row_key, value=0, family=family, qualifier=qualifier) rule = IncrementRule(family, qualifier, 1) with pytest.raises(GoogleAPICallError): - table.read_modify_write_row(row_key, rule, operation_timeout=0.001) + table.read_modify_write_row(row_key, rule, operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -1684,7 +1709,7 @@ def test_check_and_mutate_row_failure_timeout(self, table, temp_rows, handler): row_key, predicate=ValueRangeFilter(0, 2), true_case_mutations=true_mutation, - operation_timeout=0.001, + operation_timeout=0.05, ) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 From 128ebfa7d826619430769308e636f47a758b114e Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 18 Nov 2026 18:22:32 +0000 Subject: [PATCH 18/29] lint --- .../tests/system/data/test_metrics_async.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py b/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py index e2b7c3d6c84b..f4e3be1d5820 100644 --- a/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py +++ b/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py @@ -1421,9 +1421,7 @@ async def test_mutate_rows_batcher_failure_timeout(self, table, temp_rows, handl entry = RowMutationEntry(row_key, [mutation]) with pytest.raises(MutationsExceptionGroup): - async with table.mutations_batcher( - batch_operation_timeout=0.05 - ) as batcher: + async with table.mutations_batcher(batch_operation_timeout=0.05) as batcher: await batcher.append(entry) # validate counts assert len(handler.completed_operations) == 1 From da522746450e0cdc045e4d21d01124d6d4bce721 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 18 Nov 2026 18:25:22 +0000 Subject: [PATCH 19/29] work in progress --- ci/cloudbuild/system-test.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index ba1eddf3f377..3f3dcab722cf 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -68,16 +68,17 @@ steps: sys.path.insert(0, "ci") import get_package_shards - # Normalize disabled package paths to include a trailing slash. + # Normalize disabled package paths by stripping trailing slashes. disabled_packages = { - package_path.strip().rstrip("/") + "/" + package_path.strip().rstrip("/") for package_path in os.environ.get("DISABLED_PACKAGES", "").replace(",", " ").split() if package_path.strip() } def has_system_tests(package_path): - return package_path.startswith("packages/") and any( - os.path.exists(os.path.join(package_path, rel_path)) + normalized = package_path.strip().rstrip("/") + return normalized.startswith("packages/") and any( + os.path.exists(os.path.join(normalized, rel_path)) for rel_path in ("tests/system", "system_tests", "tests/system.py") ) @@ -91,7 +92,8 @@ steps: kept_paths = [ path for path in package_paths - if path not in disabled_packages and has_system_tests(path) + if path.strip().rstrip("/") not in disabled_packages + and has_system_tests(path) ] if kept_paths: filtered_packages[package_name] = kept_paths From 6da817790915be62a043edc6f8d59cb10c409342 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 18 Nov 2026 18:57:06 +0000 Subject: [PATCH 20/29] work in progress --- .../tests/system/conftest.py | 28 +++++++++++++++++++ .../tests/system/utils.py | 19 +++++++++---- .../tests/system/v2_client/conftest.py | 2 +- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/packages/google-cloud-bigtable/tests/system/conftest.py b/packages/google-cloud-bigtable/tests/system/conftest.py index 7a7b315ec542..543f603404a0 100644 --- a/packages/google-cloud-bigtable/tests/system/conftest.py +++ b/packages/google-cloud-bigtable/tests/system/conftest.py @@ -20,3 +20,31 @@ script_path = os.path.dirname(os.path.realpath(__file__)) sys.path.append(script_path) + + +def pytest_configure(config) -> None: + """Clean up leaked test instances in the controller before workers start.""" + from google.cloud.environment_vars import BIGTABLE_EMULATOR + + if ( + os.getenv(BIGTABLE_EMULATOR) + or getattr(config.option, "help", False) + or getattr(config.option, "collectonly", False) + or hasattr(config, "workerinput") + ): + return + + project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") + if not project_id: + try: + import google.auth + + _, project_id = google.auth.default() + except Exception: + return + + if project_id: + from utils import clear_stale_instances + + # Purge leaked test instances older than 1 hour before allocating nodes. + clear_stale_instances(project_id, older_than_days=1 / 24, max_deletions=100) diff --git a/packages/google-cloud-bigtable/tests/system/utils.py b/packages/google-cloud-bigtable/tests/system/utils.py index fe3f3a2b865f..2824d4b247df 100644 --- a/packages/google-cloud-bigtable/tests/system/utils.py +++ b/packages/google-cloud-bigtable/tests/system/utils.py @@ -27,15 +27,23 @@ def clear_stale_instances( prefix: Union[str, Tuple[str, ...]] = ( "python-bigtable-tests", "g-c-p", + "ndef-", + "new-", + "dif-", + "dif-cmek-", + "ndif-", "admin-overlay-instance", "bt-samples-", + "metric-scale-test-", + "instanceadmin-", + "instance-admin-", ), - older_than_days: int = 1, - max_deletions: int = 5, + older_than_days: float = 1, + max_deletions: int = 50, ): """ Synchronously deletes any instances in the given project that are older - than older_than_days and whose name or display name matches the given prefix. + than older_than_days and whose name, display name, or test label matches. """ if os.getenv(BIGTABLE_EMULATOR): return @@ -57,11 +65,12 @@ def clear_stale_instances( break for instance in response.instances: - # Check if instance matches the prefix + # Check if instance matches the prefix or python-system test label display_name_matches = instance.display_name.startswith(prefix) name_matches = instance.name.split("/")[-1].startswith(prefix) + label_matches = "python-system" in getattr(instance, "labels", {}) - if display_name_matches or name_matches: + if display_name_matches or name_matches or label_matches: if instance.create_time: try: create_time = instance.create_time diff --git a/packages/google-cloud-bigtable/tests/system/v2_client/conftest.py b/packages/google-cloud-bigtable/tests/system/v2_client/conftest.py index 59e016f52af8..82ae486b2ad8 100644 --- a/packages/google-cloud-bigtable/tests/system/v2_client/conftest.py +++ b/packages/google-cloud-bigtable/tests/system/v2_client/conftest.py @@ -58,7 +58,7 @@ def location_id(): @pytest.fixture(scope="session") def serve_nodes(): - return 3 + return 1 @pytest.fixture(scope="session") From aca40cd27aeac89b8d9242de1546851dc40343e0 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 19 Nov 2026 17:00:30 +0000 Subject: [PATCH 21/29] work in progress --- ci/cloudbuild/system-test.yaml | 75 +++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 3f3dcab722cf..b516e0969d95 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -15,6 +15,11 @@ steps: - name: 'us-central1-docker.pkg.dev/client-library-releases/release-images-prod/python-multi@sha256:d529954c373542171f9fc411e0102cebe542b511e320a965edf8820285f4f122' id: 'run-system-tests-shard' + secretEnv: + - 'GAPIC_GENERATOR_ADS_ENV' + - 'GAPIC_GENERATOR_SERVICE_ACCOUNT_JSON' + - 'GOOGLE_AUTH_SERVICE_ACCOUNT_JSON' + - 'GOOGLE_AUTH_AUTHORIZED_USER_JSON' env: # Populate standard GCP project and CI environment variables expected by system test suites. - 'CI=true' @@ -24,6 +29,7 @@ steps: - 'GOOGLE_CLOUD_PROJECT=${PROJECT_ID}' - 'GCLOUD_PROJECT=${PROJECT_ID}' - 'PROJECT_ID=${PROJECT_ID}' + - 'FIRESTORE_PROJECT_ID=${_FIRESTORE_PROJECT_ID}' # Pass sharding and diff parameters consumed by ci/get_package_shards.py. - 'MAX_SHARDS=${_MAX_SHARDS}' - 'SHARD_INDEX=${_SHARD_INDEX}' @@ -79,7 +85,12 @@ steps: normalized = package_path.strip().rstrip("/") return normalized.startswith("packages/") and any( os.path.exists(os.path.join(normalized, rel_path)) - for rel_path in ("tests/system", "system_tests", "tests/system.py") + for rel_path in ( + "tests/system", + "tests/system_live", + "system_tests", + "tests/system.py", + ) ) original_get_packages = get_package_shards.get_packages @@ -172,8 +183,47 @@ steps: REAL_NOX=$(which -a nox | grep -v '/tmp/cloudbuild_shims' | head -n 1) TEMP_LOG_FILE=$(mktemp) START_TIME=$(date +%s) - echo "[${PACKAGE_NAME}] Starting nox $*" >&3 - "$REAL_NOX" "$@" 2>&1 | python3 -u -c ' + NOX_ARGS=("$@") + + # Per-package setup for google-cloud-firestore, google-auth, and gapic-generator. + if [[ "${PACKAGE_NAME}" == "google-cloud-firestore" && -n "${FIRESTORE_PROJECT_ID:-}" ]]; then + export GOOGLE_CLOUD_PROJECT="${FIRESTORE_PROJECT_ID}" + export GCLOUD_PROJECT="${FIRESTORE_PROJECT_ID}" + elif [[ "${PACKAGE_NAME}" == "google-auth" ]]; then + mkdir -p system_tests/data + printf '%s' "${GOOGLE_AUTH_SERVICE_ACCOUNT_JSON}" > system_tests/data/service_account.json + printf '%s' "${GOOGLE_AUTH_AUTHORIZED_USER_JSON}" > system_tests/data/authorized_user.json + export GOOGLE_APPLICATION_CREDENTIALS="${PWD}/system_tests/data/service_account.json" + NOX_ARGS=(--stop-on-first-error -f system_tests/noxfile.py) + elif [[ "${PACKAGE_NAME}" == "gapic-generator" ]]; then + ADS_ENV_FILE=$(mktemp) + ADS_SA_FILE=$(mktemp --suffix=.json) + trap 'rm -f "${ADS_ENV_FILE}" "${ADS_SA_FILE}" "${TEMP_LOG_FILE}"' EXIT + printf '%s\n' "${GAPIC_GENERATOR_ADS_ENV}" > "${ADS_ENV_FILE}" + printf '%s' "${GAPIC_GENERATOR_SERVICE_ACCOUNT_JSON}" > "${ADS_SA_FILE}" + set -a + source "${ADS_ENV_FILE}" + set +a + rm -f "${ADS_ENV_FILE}" + export GOOGLE_APPLICATION_CREDENTIALS="${ADS_SA_FILE}" + export GOOGLE_ADS_VIDEO_PATH="/tmp/gapic_generator_test_video.mp4" + if [[ ! -f "${GOOGLE_ADS_VIDEO_PATH}" ]]; then + if command -v ffmpeg >/dev/null 2>&1; then + ffmpeg -y -loglevel error -f lavfi -i testsrc=duration=8:size=1280x720:rate=30 -c:v libx264 -b:v 1M -pix_fmt yuv420p "${GOOGLE_ADS_VIDEO_PATH}" + else + python3 -c ' + import os, struct + ftyp = struct.pack(">I4s4sI4s", 20, b"ftyp", b"isom", 512, b"isom") + pad = b"\x00" * (600 * 1024) + free = struct.pack(">I4s", 8 + len(pad), b"free") + pad + open(os.environ["GOOGLE_ADS_VIDEO_PATH"], "wb").write(ftyp + free) + ' + fi + fi + fi + + echo "[${PACKAGE_NAME}] Starting nox ${NOX_ARGS[*]}" >&3 + "$REAL_NOX" "${NOX_ARGS[@]}" 2>&1 | python3 -u -c ' import os, sys prefix = f"[{sys.argv[1]}] " with open(sys.argv[2], "w", buffering=1) as log_f, os.fdopen(3, "w", buffering=1, closefd=False) as live_f: @@ -275,17 +325,21 @@ substitutions: _BUILD_TYPE: 'presubmit' _BASE_BRANCH: 'main' _TEST_ALL_PACKAGES: 'false' - _DISABLED_PACKAGES: 'packages/gapic-generator/ packages/google-auth/ packages/google-cloud-firestore/' + _DISABLED_PACKAGES: '' + _FIRESTORE_PROJECT_ID: '' _NOX_SESSION: 'system-3.12' _PACKAGE_WEIGHTS: | bigquery-magics: 2 + gapic-generator: 5 + google-auth: 5 google-cloud-bigquery: 8 google-cloud-bigquery-storage: 8 - google-cloud-bigtable: 15 + google-cloud-bigtable: 8 google-cloud-compute: 8 google-cloud-datastore: 8 google-cloud-dns: 1 google-cloud-error-reporting: 1 + google-cloud-firestore: 8 google-cloud-logging: 10 google-cloud-ndb: 12 google-cloud-pubsub: 15 @@ -298,3 +352,14 @@ substitutions: options: machineType: 'E2_HIGHCPU_8' logging: CLOUD_LOGGING_ONLY + +availableSecrets: + secretManager: + - versionName: projects/$PROJECT_ID/secrets/gapic-generator-ads-env/versions/latest + env: 'GAPIC_GENERATOR_ADS_ENV' + - versionName: projects/$PROJECT_ID/secrets/gapic-generator-service-account/versions/latest + env: 'GAPIC_GENERATOR_SERVICE_ACCOUNT_JSON' + - versionName: projects/$PROJECT_ID/secrets/google-auth-service-account/versions/latest + env: 'GOOGLE_AUTH_SERVICE_ACCOUNT_JSON' + - versionName: projects/$PROJECT_ID/secrets/google-auth-authorized-user/versions/latest + env: 'GOOGLE_AUTH_AUTHORIZED_USER_JSON' From 54a38396b2091984bb3dcddaf58b08eb60c93720 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 19 Nov 2026 17:03:57 +0000 Subject: [PATCH 22/29] Revert "work in progress" This reverts commit 06aa46e40e23637cf1b16d06a59a6b3108535c90. --- ci/cloudbuild/system-test.yaml | 75 +++------------------------------- 1 file changed, 5 insertions(+), 70 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index b516e0969d95..3f3dcab722cf 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -15,11 +15,6 @@ steps: - name: 'us-central1-docker.pkg.dev/client-library-releases/release-images-prod/python-multi@sha256:d529954c373542171f9fc411e0102cebe542b511e320a965edf8820285f4f122' id: 'run-system-tests-shard' - secretEnv: - - 'GAPIC_GENERATOR_ADS_ENV' - - 'GAPIC_GENERATOR_SERVICE_ACCOUNT_JSON' - - 'GOOGLE_AUTH_SERVICE_ACCOUNT_JSON' - - 'GOOGLE_AUTH_AUTHORIZED_USER_JSON' env: # Populate standard GCP project and CI environment variables expected by system test suites. - 'CI=true' @@ -29,7 +24,6 @@ steps: - 'GOOGLE_CLOUD_PROJECT=${PROJECT_ID}' - 'GCLOUD_PROJECT=${PROJECT_ID}' - 'PROJECT_ID=${PROJECT_ID}' - - 'FIRESTORE_PROJECT_ID=${_FIRESTORE_PROJECT_ID}' # Pass sharding and diff parameters consumed by ci/get_package_shards.py. - 'MAX_SHARDS=${_MAX_SHARDS}' - 'SHARD_INDEX=${_SHARD_INDEX}' @@ -85,12 +79,7 @@ steps: normalized = package_path.strip().rstrip("/") return normalized.startswith("packages/") and any( os.path.exists(os.path.join(normalized, rel_path)) - for rel_path in ( - "tests/system", - "tests/system_live", - "system_tests", - "tests/system.py", - ) + for rel_path in ("tests/system", "system_tests", "tests/system.py") ) original_get_packages = get_package_shards.get_packages @@ -183,47 +172,8 @@ steps: REAL_NOX=$(which -a nox | grep -v '/tmp/cloudbuild_shims' | head -n 1) TEMP_LOG_FILE=$(mktemp) START_TIME=$(date +%s) - NOX_ARGS=("$@") - - # Per-package setup for google-cloud-firestore, google-auth, and gapic-generator. - if [[ "${PACKAGE_NAME}" == "google-cloud-firestore" && -n "${FIRESTORE_PROJECT_ID:-}" ]]; then - export GOOGLE_CLOUD_PROJECT="${FIRESTORE_PROJECT_ID}" - export GCLOUD_PROJECT="${FIRESTORE_PROJECT_ID}" - elif [[ "${PACKAGE_NAME}" == "google-auth" ]]; then - mkdir -p system_tests/data - printf '%s' "${GOOGLE_AUTH_SERVICE_ACCOUNT_JSON}" > system_tests/data/service_account.json - printf '%s' "${GOOGLE_AUTH_AUTHORIZED_USER_JSON}" > system_tests/data/authorized_user.json - export GOOGLE_APPLICATION_CREDENTIALS="${PWD}/system_tests/data/service_account.json" - NOX_ARGS=(--stop-on-first-error -f system_tests/noxfile.py) - elif [[ "${PACKAGE_NAME}" == "gapic-generator" ]]; then - ADS_ENV_FILE=$(mktemp) - ADS_SA_FILE=$(mktemp --suffix=.json) - trap 'rm -f "${ADS_ENV_FILE}" "${ADS_SA_FILE}" "${TEMP_LOG_FILE}"' EXIT - printf '%s\n' "${GAPIC_GENERATOR_ADS_ENV}" > "${ADS_ENV_FILE}" - printf '%s' "${GAPIC_GENERATOR_SERVICE_ACCOUNT_JSON}" > "${ADS_SA_FILE}" - set -a - source "${ADS_ENV_FILE}" - set +a - rm -f "${ADS_ENV_FILE}" - export GOOGLE_APPLICATION_CREDENTIALS="${ADS_SA_FILE}" - export GOOGLE_ADS_VIDEO_PATH="/tmp/gapic_generator_test_video.mp4" - if [[ ! -f "${GOOGLE_ADS_VIDEO_PATH}" ]]; then - if command -v ffmpeg >/dev/null 2>&1; then - ffmpeg -y -loglevel error -f lavfi -i testsrc=duration=8:size=1280x720:rate=30 -c:v libx264 -b:v 1M -pix_fmt yuv420p "${GOOGLE_ADS_VIDEO_PATH}" - else - python3 -c ' - import os, struct - ftyp = struct.pack(">I4s4sI4s", 20, b"ftyp", b"isom", 512, b"isom") - pad = b"\x00" * (600 * 1024) - free = struct.pack(">I4s", 8 + len(pad), b"free") + pad - open(os.environ["GOOGLE_ADS_VIDEO_PATH"], "wb").write(ftyp + free) - ' - fi - fi - fi - - echo "[${PACKAGE_NAME}] Starting nox ${NOX_ARGS[*]}" >&3 - "$REAL_NOX" "${NOX_ARGS[@]}" 2>&1 | python3 -u -c ' + echo "[${PACKAGE_NAME}] Starting nox $*" >&3 + "$REAL_NOX" "$@" 2>&1 | python3 -u -c ' import os, sys prefix = f"[{sys.argv[1]}] " with open(sys.argv[2], "w", buffering=1) as log_f, os.fdopen(3, "w", buffering=1, closefd=False) as live_f: @@ -325,21 +275,17 @@ substitutions: _BUILD_TYPE: 'presubmit' _BASE_BRANCH: 'main' _TEST_ALL_PACKAGES: 'false' - _DISABLED_PACKAGES: '' - _FIRESTORE_PROJECT_ID: '' + _DISABLED_PACKAGES: 'packages/gapic-generator/ packages/google-auth/ packages/google-cloud-firestore/' _NOX_SESSION: 'system-3.12' _PACKAGE_WEIGHTS: | bigquery-magics: 2 - gapic-generator: 5 - google-auth: 5 google-cloud-bigquery: 8 google-cloud-bigquery-storage: 8 - google-cloud-bigtable: 8 + google-cloud-bigtable: 15 google-cloud-compute: 8 google-cloud-datastore: 8 google-cloud-dns: 1 google-cloud-error-reporting: 1 - google-cloud-firestore: 8 google-cloud-logging: 10 google-cloud-ndb: 12 google-cloud-pubsub: 15 @@ -352,14 +298,3 @@ substitutions: options: machineType: 'E2_HIGHCPU_8' logging: CLOUD_LOGGING_ONLY - -availableSecrets: - secretManager: - - versionName: projects/$PROJECT_ID/secrets/gapic-generator-ads-env/versions/latest - env: 'GAPIC_GENERATOR_ADS_ENV' - - versionName: projects/$PROJECT_ID/secrets/gapic-generator-service-account/versions/latest - env: 'GAPIC_GENERATOR_SERVICE_ACCOUNT_JSON' - - versionName: projects/$PROJECT_ID/secrets/google-auth-service-account/versions/latest - env: 'GOOGLE_AUTH_SERVICE_ACCOUNT_JSON' - - versionName: projects/$PROJECT_ID/secrets/google-auth-authorized-user/versions/latest - env: 'GOOGLE_AUTH_AUTHORIZED_USER_JSON' From b6b844634746d344c22a3f623eb1ccb60f272201 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 19 Nov 2026 17:36:11 +0000 Subject: [PATCH 23/29] work in progress --- ci/cancel_superseded_builds.py | 342 +++++++++++++++++++++++++++++++++ ci/cloudbuild/system-test.yaml | 21 +- 2 files changed, 361 insertions(+), 2 deletions(-) create mode 100644 ci/cancel_superseded_builds.py diff --git a/ci/cancel_superseded_builds.py b/ci/cancel_superseded_builds.py new file mode 100644 index 000000000000..8b160f3fade9 --- /dev/null +++ b/ci/cancel_superseded_builds.py @@ -0,0 +1,342 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Cancel superseded Cloud Build PR runs when a newer commit is pushed. + +Cloud Build triggers do not provide a native ``cancel-in-progress`` setting +(unlike GitHub Actions ``concurrency`` or Kokoro's +``stop_outdated_presubmit_builds``). When Cloud Build concurrency slots are +saturated, a newly pushed commit sits in ``QUEUED`` and cannot even start its +first build step until an older ``WORKING`` build exits. + +This script solves both cases using only the Python standard library and the +Cloud Build REST API (authenticated via the GCE metadata server): + +1. ``--cancel-older`` (run once at build startup): + Lists all ``QUEUED`` / ``WORKING`` builds for the same trigger and PR number + whose ``createTime`` is earlier than this build and cancels them. +2. ``--watch`` (run in the background while tests execute): + Polls every ``--interval`` seconds (default 30s) for any ``QUEUED`` / + ``WORKING`` build for the same trigger and PR number whose ``createTime`` is + newer than this build and targets a different ``COMMIT_SHA``. If found, this + build cancels itself via ``builds:cancel`` so its concurrency slot is freed + immediately for the newer commit. + +Requires ``PROJECT_ID``, ``LOCATION``, and ``BUILD_ID`` environment variables +(populated from Cloud Build built-in substitutions) and +``roles/cloudbuild.builds.editor`` on the build service account. + +Non-PR builds (e.g. postsubmit pushes on ``main`` where ``_PR_NUMBER`` is +absent) exit cleanly as a no-op. Any transient API/network errors fail open +with a warning so cancellation checks never break CI test execution. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + + +METADATA_TOKEN_URL = ( + "http://metadata.google.internal/computeMetadata/v1/" + "instance/service-accounts/default/token" +) + + +def _get_access_token() -> str: + """Fetch a short-lived OAuth2 access token from the GCE metadata server.""" + req = urllib.request.Request( + METADATA_TOKEN_URL, + headers={"Metadata-Flavor": "Google"}, + ) + with urllib.request.urlopen(req, timeout=10) as resp: + return json.loads(resp.read().decode("utf-8"))["access_token"] + + +def _api_request( + method: str, + url: str, + token: str, + body: bytes | None = None, +) -> dict: + """Execute an authenticated JSON HTTP request against the Cloud Build API.""" + headers = {"Authorization": f"Bearer {token}"} + if body is not None: + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=body, headers=headers, method=method) + with urllib.request.urlopen(req, timeout=15) as resp: + raw = resp.read().decode("utf-8") + return json.loads(raw) if raw else {} + + +def _parse_timestamp(raw_ts: str) -> datetime: + """Parse an RFC 3339 Cloud Build timestamp (e.g. ``2026-11-19T17:00:00.123Z``).""" + # Python 3.11+ fromisoformat handles trailing "Z", normalize for 3.10 compatibility. + normalized = raw_ts.replace("Z", "+00:00") + # Truncate sub-microsecond nanoseconds (9 digits) to 6 digits if present. + if "." in normalized: + date_part, rest = normalized.split(".", 1) + sign_idx = max(rest.find("+"), rest.find("-")) + frac = rest[:sign_idx][:6].ljust(6, "0") + tz = rest[sign_idx:] + normalized = f"{date_part}.{frac}{tz}" + return datetime.fromisoformat(normalized) + + +def _base_url(project_id: str, location: str) -> str: + """Return the regional Cloud Build v1 builds endpoint.""" + return ( + f"https://cloudbuild.googleapis.com/v1/" + f"projects/{project_id}/locations/{location}/builds" + ) + + +def _get_build(project_id: str, location: str, build_id: str, token: str) -> dict: + """Fetch metadata (trigger ID, substitutions, createTime) for a single build.""" + url = f"{_base_url(project_id, location)}/{build_id}" + return _api_request("GET", url, token) + + +def _list_active_trigger_builds( + project_id: str, + location: str, + trigger_id: str, + token: str, +) -> list[dict]: + """List all QUEUED or WORKING builds belonging to the same Cloud Build trigger.""" + filter_expr = f'trigger_id="{trigger_id}" AND (status="QUEUED" OR status="WORKING")' + builds: list[dict] = [] + page_token = "" + + while True: + params = {"filter": filter_expr, "pageSize": "100"} + if page_token: + params["pageToken"] = page_token + url = f"{_base_url(project_id, location)}?{urllib.parse.urlencode(params)}" + data = _api_request("GET", url, token) + builds.extend(data.get("builds", [])) + page_token = data.get("nextPageToken", "") + if not page_token: + break + + return builds + + +def _cancel_build(project_id: str, location: str, build_id: str, token: str) -> None: + """Issue a Cloud Build CancelBuild request for ``build_id``.""" + url = f"{_base_url(project_id, location)}/{build_id}:cancel" + _api_request("POST", url, token, body=b"{}") + + +def _extract_pr_context(build: dict) -> tuple[str, str, str, datetime] | None: + """Extract ``(trigger_id, pr_number, commit_sha, create_time)`` for a PR build. + + Returns ``None`` if the build is not associated with a trigger or pull request + (e.g. postsubmit continuous builds on ``main``). + """ + trigger_id = build.get("buildTriggerId", "") + substitutions = build.get("substitutions") or {} + pr_number = substitutions.get("_PR_NUMBER", "") + commit_sha = substitutions.get("COMMIT_SHA", "") + create_time_raw = build.get("createTime", "") + + if not trigger_id or not pr_number or not create_time_raw: + return None + + return trigger_id, pr_number, commit_sha, _parse_timestamp(create_time_raw) + + +def _cancel_older_builds( + project_id: str, + location: str, + my_build_id: str, + trigger_id: str, + pr_number: str, + my_commit_sha: str, + my_create_time: datetime, + token: str, +) -> None: + """Cancel any older QUEUED/WORKING builds for the same PR and trigger.""" + active_builds = _list_active_trigger_builds(project_id, location, trigger_id, token) + for other in active_builds: + other_id = other.get("id", "") + if not other_id or other_id == my_build_id: + continue + other_subs = other.get("substitutions") or {} + if other_subs.get("_PR_NUMBER") != pr_number: + continue + # Only cancel builds for an earlier commit so manual re-runs of the same + # commit or older timestamps are handled safely. + other_sha = other_subs.get("COMMIT_SHA", "") + if my_commit_sha and other_sha == my_commit_sha: + continue + other_create_raw = other.get("createTime", "") + if not other_create_raw: + continue + if _parse_timestamp(other_create_raw) < my_create_time: + print( + f"[cancel-superseded] Cancelling older {other.get('status')} build " + f"{other_id} (PR #{pr_number}, commit {other_sha[:8]}) superseded by " + f"{my_build_id} (commit {my_commit_sha[:8]}).", + flush=True, + ) + try: + _cancel_build(project_id, location, other_id, token) + except urllib.error.URLError as exc: + print( + f"[cancel-superseded] Warning: failed to cancel {other_id}: {exc}", + file=sys.stderr, + flush=True, + ) + + +def _watch_for_newer_build( + project_id: str, + location: str, + my_build_id: str, + trigger_id: str, + pr_number: str, + my_commit_sha: str, + my_create_time: datetime, + interval: int, +) -> None: + """Poll until a newer commit build for the same PR + trigger appears, then self-cancel.""" + while True: + time.sleep(interval) + try: + # Refresh metadata token each iteration so long-running builds never hit expiry. + token = _get_access_token() + active_builds = _list_active_trigger_builds( + project_id, location, trigger_id, token + ) + for other in active_builds: + other_id = other.get("id", "") + if not other_id or other_id == my_build_id: + continue + other_subs = other.get("substitutions") or {} + if other_subs.get("_PR_NUMBER") != pr_number: + continue + other_sha = other_subs.get("COMMIT_SHA", "") + if my_commit_sha and other_sha == my_commit_sha: + continue + other_create_raw = other.get("createTime", "") + if not other_create_raw: + continue + if _parse_timestamp(other_create_raw) > my_create_time: + print( + f"[cancel-superseded] Newer {other.get('status')} build {other_id} " + f"(PR #{pr_number}, commit {other_sha[:8]}) detected; " + f"self-cancelling {my_build_id} to free worker slot.", + flush=True, + ) + _cancel_build(project_id, location, my_build_id, token) + return + except Exception as exc: + # Fail open on transient metadata/API errors and keep watching. + print( + f"[cancel-superseded] Watchdog warning (will retry): {exc}", + file=sys.stderr, + flush=True, + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Cancel superseded Cloud Build PR runs for the same trigger." + ) + parser.add_argument( + "--cancel-older", + action="store_true", + help="Cancel older QUEUED/WORKING builds for the same PR and trigger.", + ) + parser.add_argument( + "--watch", + action="store_true", + help="Continuously watch for a newer commit on the same PR and self-cancel.", + ) + parser.add_argument( + "--interval", + type=int, + default=30, + help="Polling interval in seconds for --watch mode (default: 30).", + ) + args = parser.parse_args() + + project_id = os.environ.get("PROJECT_ID", "") + location = os.environ.get("LOCATION", "") + build_id = os.environ.get("BUILD_ID", "") + if not project_id or not location or not build_id: + return 0 + + try: + token = _get_access_token() + current_build = _get_build(project_id, location, build_id, token) + except Exception as exc: + print( + f"[cancel-superseded] Warning: unable to query build metadata ({exc}); skipping.", + file=sys.stderr, + flush=True, + ) + return 0 + + ctx = _extract_pr_context(current_build) + if ctx is None: + # Postsubmit push or manual build without _PR_NUMBER; nothing to cancel. + return 0 + + trigger_id, pr_number, my_commit_sha, my_create_time = ctx + + if args.cancel_older: + try: + _cancel_older_builds( + project_id, + location, + build_id, + trigger_id, + pr_number, + my_commit_sha, + my_create_time, + token, + ) + except Exception as exc: + print( + f"[cancel-superseded] Warning: failed to cancel older builds ({exc}).", + file=sys.stderr, + flush=True, + ) + + if args.watch: + _watch_for_newer_build( + project_id, + location, + build_id, + trigger_id, + pr_number, + my_commit_sha, + my_create_time, + args.interval, + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 3f3dcab722cf..6df7f6fc5f58 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -24,6 +24,9 @@ steps: - 'GOOGLE_CLOUD_PROJECT=${PROJECT_ID}' - 'GCLOUD_PROJECT=${PROJECT_ID}' - 'PROJECT_ID=${PROJECT_ID}' + # Cloud Build built-ins used by ci/cancel_superseded_builds.py to cancel outdated PR runs. + - 'LOCATION=${LOCATION}' + - 'BUILD_ID=${BUILD_ID}' # Pass sharding and diff parameters consumed by ci/get_package_shards.py. - 'MAX_SHARDS=${_MAX_SHARDS}' - 'SHARD_INDEX=${_SHARD_INDEX}' @@ -46,6 +49,10 @@ steps: #!/usr/bin/env bash set -eo pipefail + # Cancel any older QUEUED/WORKING builds for this PR and shard so newer pushes + # immediately take precedence instead of piling up in the Cloud Build queue. + python3 ci/cancel_superseded_builds.py --cancel-older || true + # Derive PY_VERSION (e.g. "3.12" from "system-3.12") required by ci/run_single_test.sh. export PY_VERSION="${NOX_SESSION#system-}" @@ -245,7 +252,12 @@ steps: # Export PACKAGE_LIST and delegate execution to the existing monorepo test runner script. rm -f /tmp/cloudbuild_weight_adjustments.txt - print_weight_summary() { + CANCEL_WATCHDOG_PID="" + cleanup_on_exit() { + # Stop the background cancellation watchdog so it never keeps the build container open. + if [[ -n "${CANCEL_WATCHDOG_PID}" ]]; then + kill "${CANCEL_WATCHDOG_PID}" 2>/dev/null || true + fi echo "" echo "============================================================" if [[ -s /tmp/cloudbuild_weight_adjustments.txt ]]; then @@ -256,7 +268,12 @@ steps: fi echo "============================================================" } - trap print_weight_summary EXIT + trap cleanup_on_exit EXIT + + # Run a background watchdog that self-cancels this build if a newer commit is pushed + # to the same PR while all Cloud Build worker slots are occupied. + python3 ci/cancel_superseded_builds.py --watch & + CANCEL_WATCHDOG_PID=$! export PACKAGE_LIST echo "=== System Shard ${SHARD_INDEX}/${MAX_SHARDS} running ${NOX_SESSION} for: ${PACKAGE_LIST} ===" From f2b7edb484ffbe349e720241dc1cc979c73fab9a Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 19 Nov 2026 18:12:42 +0000 Subject: [PATCH 24/29] Revert "work in progress" This reverts commit aa63f9d6eba6709a5dedcaa90abaeaba27d9bee9. --- ci/cancel_superseded_builds.py | 342 --------------------------------- ci/cloudbuild/system-test.yaml | 21 +- 2 files changed, 2 insertions(+), 361 deletions(-) delete mode 100644 ci/cancel_superseded_builds.py diff --git a/ci/cancel_superseded_builds.py b/ci/cancel_superseded_builds.py deleted file mode 100644 index 8b160f3fade9..000000000000 --- a/ci/cancel_superseded_builds.py +++ /dev/null @@ -1,342 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Cancel superseded Cloud Build PR runs when a newer commit is pushed. - -Cloud Build triggers do not provide a native ``cancel-in-progress`` setting -(unlike GitHub Actions ``concurrency`` or Kokoro's -``stop_outdated_presubmit_builds``). When Cloud Build concurrency slots are -saturated, a newly pushed commit sits in ``QUEUED`` and cannot even start its -first build step until an older ``WORKING`` build exits. - -This script solves both cases using only the Python standard library and the -Cloud Build REST API (authenticated via the GCE metadata server): - -1. ``--cancel-older`` (run once at build startup): - Lists all ``QUEUED`` / ``WORKING`` builds for the same trigger and PR number - whose ``createTime`` is earlier than this build and cancels them. -2. ``--watch`` (run in the background while tests execute): - Polls every ``--interval`` seconds (default 30s) for any ``QUEUED`` / - ``WORKING`` build for the same trigger and PR number whose ``createTime`` is - newer than this build and targets a different ``COMMIT_SHA``. If found, this - build cancels itself via ``builds:cancel`` so its concurrency slot is freed - immediately for the newer commit. - -Requires ``PROJECT_ID``, ``LOCATION``, and ``BUILD_ID`` environment variables -(populated from Cloud Build built-in substitutions) and -``roles/cloudbuild.builds.editor`` on the build service account. - -Non-PR builds (e.g. postsubmit pushes on ``main`` where ``_PR_NUMBER`` is -absent) exit cleanly as a no-op. Any transient API/network errors fail open -with a warning so cancellation checks never break CI test execution. -""" - -from __future__ import annotations - -import argparse -from datetime import datetime -import json -import os -import sys -import time -import urllib.error -import urllib.parse -import urllib.request - - -METADATA_TOKEN_URL = ( - "http://metadata.google.internal/computeMetadata/v1/" - "instance/service-accounts/default/token" -) - - -def _get_access_token() -> str: - """Fetch a short-lived OAuth2 access token from the GCE metadata server.""" - req = urllib.request.Request( - METADATA_TOKEN_URL, - headers={"Metadata-Flavor": "Google"}, - ) - with urllib.request.urlopen(req, timeout=10) as resp: - return json.loads(resp.read().decode("utf-8"))["access_token"] - - -def _api_request( - method: str, - url: str, - token: str, - body: bytes | None = None, -) -> dict: - """Execute an authenticated JSON HTTP request against the Cloud Build API.""" - headers = {"Authorization": f"Bearer {token}"} - if body is not None: - headers["Content-Type"] = "application/json" - req = urllib.request.Request(url, data=body, headers=headers, method=method) - with urllib.request.urlopen(req, timeout=15) as resp: - raw = resp.read().decode("utf-8") - return json.loads(raw) if raw else {} - - -def _parse_timestamp(raw_ts: str) -> datetime: - """Parse an RFC 3339 Cloud Build timestamp (e.g. ``2026-11-19T17:00:00.123Z``).""" - # Python 3.11+ fromisoformat handles trailing "Z", normalize for 3.10 compatibility. - normalized = raw_ts.replace("Z", "+00:00") - # Truncate sub-microsecond nanoseconds (9 digits) to 6 digits if present. - if "." in normalized: - date_part, rest = normalized.split(".", 1) - sign_idx = max(rest.find("+"), rest.find("-")) - frac = rest[:sign_idx][:6].ljust(6, "0") - tz = rest[sign_idx:] - normalized = f"{date_part}.{frac}{tz}" - return datetime.fromisoformat(normalized) - - -def _base_url(project_id: str, location: str) -> str: - """Return the regional Cloud Build v1 builds endpoint.""" - return ( - f"https://cloudbuild.googleapis.com/v1/" - f"projects/{project_id}/locations/{location}/builds" - ) - - -def _get_build(project_id: str, location: str, build_id: str, token: str) -> dict: - """Fetch metadata (trigger ID, substitutions, createTime) for a single build.""" - url = f"{_base_url(project_id, location)}/{build_id}" - return _api_request("GET", url, token) - - -def _list_active_trigger_builds( - project_id: str, - location: str, - trigger_id: str, - token: str, -) -> list[dict]: - """List all QUEUED or WORKING builds belonging to the same Cloud Build trigger.""" - filter_expr = f'trigger_id="{trigger_id}" AND (status="QUEUED" OR status="WORKING")' - builds: list[dict] = [] - page_token = "" - - while True: - params = {"filter": filter_expr, "pageSize": "100"} - if page_token: - params["pageToken"] = page_token - url = f"{_base_url(project_id, location)}?{urllib.parse.urlencode(params)}" - data = _api_request("GET", url, token) - builds.extend(data.get("builds", [])) - page_token = data.get("nextPageToken", "") - if not page_token: - break - - return builds - - -def _cancel_build(project_id: str, location: str, build_id: str, token: str) -> None: - """Issue a Cloud Build CancelBuild request for ``build_id``.""" - url = f"{_base_url(project_id, location)}/{build_id}:cancel" - _api_request("POST", url, token, body=b"{}") - - -def _extract_pr_context(build: dict) -> tuple[str, str, str, datetime] | None: - """Extract ``(trigger_id, pr_number, commit_sha, create_time)`` for a PR build. - - Returns ``None`` if the build is not associated with a trigger or pull request - (e.g. postsubmit continuous builds on ``main``). - """ - trigger_id = build.get("buildTriggerId", "") - substitutions = build.get("substitutions") or {} - pr_number = substitutions.get("_PR_NUMBER", "") - commit_sha = substitutions.get("COMMIT_SHA", "") - create_time_raw = build.get("createTime", "") - - if not trigger_id or not pr_number or not create_time_raw: - return None - - return trigger_id, pr_number, commit_sha, _parse_timestamp(create_time_raw) - - -def _cancel_older_builds( - project_id: str, - location: str, - my_build_id: str, - trigger_id: str, - pr_number: str, - my_commit_sha: str, - my_create_time: datetime, - token: str, -) -> None: - """Cancel any older QUEUED/WORKING builds for the same PR and trigger.""" - active_builds = _list_active_trigger_builds(project_id, location, trigger_id, token) - for other in active_builds: - other_id = other.get("id", "") - if not other_id or other_id == my_build_id: - continue - other_subs = other.get("substitutions") or {} - if other_subs.get("_PR_NUMBER") != pr_number: - continue - # Only cancel builds for an earlier commit so manual re-runs of the same - # commit or older timestamps are handled safely. - other_sha = other_subs.get("COMMIT_SHA", "") - if my_commit_sha and other_sha == my_commit_sha: - continue - other_create_raw = other.get("createTime", "") - if not other_create_raw: - continue - if _parse_timestamp(other_create_raw) < my_create_time: - print( - f"[cancel-superseded] Cancelling older {other.get('status')} build " - f"{other_id} (PR #{pr_number}, commit {other_sha[:8]}) superseded by " - f"{my_build_id} (commit {my_commit_sha[:8]}).", - flush=True, - ) - try: - _cancel_build(project_id, location, other_id, token) - except urllib.error.URLError as exc: - print( - f"[cancel-superseded] Warning: failed to cancel {other_id}: {exc}", - file=sys.stderr, - flush=True, - ) - - -def _watch_for_newer_build( - project_id: str, - location: str, - my_build_id: str, - trigger_id: str, - pr_number: str, - my_commit_sha: str, - my_create_time: datetime, - interval: int, -) -> None: - """Poll until a newer commit build for the same PR + trigger appears, then self-cancel.""" - while True: - time.sleep(interval) - try: - # Refresh metadata token each iteration so long-running builds never hit expiry. - token = _get_access_token() - active_builds = _list_active_trigger_builds( - project_id, location, trigger_id, token - ) - for other in active_builds: - other_id = other.get("id", "") - if not other_id or other_id == my_build_id: - continue - other_subs = other.get("substitutions") or {} - if other_subs.get("_PR_NUMBER") != pr_number: - continue - other_sha = other_subs.get("COMMIT_SHA", "") - if my_commit_sha and other_sha == my_commit_sha: - continue - other_create_raw = other.get("createTime", "") - if not other_create_raw: - continue - if _parse_timestamp(other_create_raw) > my_create_time: - print( - f"[cancel-superseded] Newer {other.get('status')} build {other_id} " - f"(PR #{pr_number}, commit {other_sha[:8]}) detected; " - f"self-cancelling {my_build_id} to free worker slot.", - flush=True, - ) - _cancel_build(project_id, location, my_build_id, token) - return - except Exception as exc: - # Fail open on transient metadata/API errors and keep watching. - print( - f"[cancel-superseded] Watchdog warning (will retry): {exc}", - file=sys.stderr, - flush=True, - ) - - -def main() -> int: - parser = argparse.ArgumentParser( - description="Cancel superseded Cloud Build PR runs for the same trigger." - ) - parser.add_argument( - "--cancel-older", - action="store_true", - help="Cancel older QUEUED/WORKING builds for the same PR and trigger.", - ) - parser.add_argument( - "--watch", - action="store_true", - help="Continuously watch for a newer commit on the same PR and self-cancel.", - ) - parser.add_argument( - "--interval", - type=int, - default=30, - help="Polling interval in seconds for --watch mode (default: 30).", - ) - args = parser.parse_args() - - project_id = os.environ.get("PROJECT_ID", "") - location = os.environ.get("LOCATION", "") - build_id = os.environ.get("BUILD_ID", "") - if not project_id or not location or not build_id: - return 0 - - try: - token = _get_access_token() - current_build = _get_build(project_id, location, build_id, token) - except Exception as exc: - print( - f"[cancel-superseded] Warning: unable to query build metadata ({exc}); skipping.", - file=sys.stderr, - flush=True, - ) - return 0 - - ctx = _extract_pr_context(current_build) - if ctx is None: - # Postsubmit push or manual build without _PR_NUMBER; nothing to cancel. - return 0 - - trigger_id, pr_number, my_commit_sha, my_create_time = ctx - - if args.cancel_older: - try: - _cancel_older_builds( - project_id, - location, - build_id, - trigger_id, - pr_number, - my_commit_sha, - my_create_time, - token, - ) - except Exception as exc: - print( - f"[cancel-superseded] Warning: failed to cancel older builds ({exc}).", - file=sys.stderr, - flush=True, - ) - - if args.watch: - _watch_for_newer_build( - project_id, - location, - build_id, - trigger_id, - pr_number, - my_commit_sha, - my_create_time, - args.interval, - ) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 6df7f6fc5f58..3f3dcab722cf 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -24,9 +24,6 @@ steps: - 'GOOGLE_CLOUD_PROJECT=${PROJECT_ID}' - 'GCLOUD_PROJECT=${PROJECT_ID}' - 'PROJECT_ID=${PROJECT_ID}' - # Cloud Build built-ins used by ci/cancel_superseded_builds.py to cancel outdated PR runs. - - 'LOCATION=${LOCATION}' - - 'BUILD_ID=${BUILD_ID}' # Pass sharding and diff parameters consumed by ci/get_package_shards.py. - 'MAX_SHARDS=${_MAX_SHARDS}' - 'SHARD_INDEX=${_SHARD_INDEX}' @@ -49,10 +46,6 @@ steps: #!/usr/bin/env bash set -eo pipefail - # Cancel any older QUEUED/WORKING builds for this PR and shard so newer pushes - # immediately take precedence instead of piling up in the Cloud Build queue. - python3 ci/cancel_superseded_builds.py --cancel-older || true - # Derive PY_VERSION (e.g. "3.12" from "system-3.12") required by ci/run_single_test.sh. export PY_VERSION="${NOX_SESSION#system-}" @@ -252,12 +245,7 @@ steps: # Export PACKAGE_LIST and delegate execution to the existing monorepo test runner script. rm -f /tmp/cloudbuild_weight_adjustments.txt - CANCEL_WATCHDOG_PID="" - cleanup_on_exit() { - # Stop the background cancellation watchdog so it never keeps the build container open. - if [[ -n "${CANCEL_WATCHDOG_PID}" ]]; then - kill "${CANCEL_WATCHDOG_PID}" 2>/dev/null || true - fi + print_weight_summary() { echo "" echo "============================================================" if [[ -s /tmp/cloudbuild_weight_adjustments.txt ]]; then @@ -268,12 +256,7 @@ steps: fi echo "============================================================" } - trap cleanup_on_exit EXIT - - # Run a background watchdog that self-cancels this build if a newer commit is pushed - # to the same PR while all Cloud Build worker slots are occupied. - python3 ci/cancel_superseded_builds.py --watch & - CANCEL_WATCHDOG_PID=$! + trap print_weight_summary EXIT export PACKAGE_LIST echo "=== System Shard ${SHARD_INDEX}/${MAX_SHARDS} running ${NOX_SESSION} for: ${PACKAGE_LIST} ===" From c476009cd91370ba4e52ed5ee939fbd841370a1d Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 19 Nov 2026 18:31:42 +0000 Subject: [PATCH 25/29] work in progress --- ci/cloudbuild/system-test.yaml | 4 +- .../instanceadmin/instanceadmin.py | 10 +- .../instanceadmin/test_instanceadmin.py | 45 +++++- .../instanceadmin/instanceadmin_async.py | 10 +- .../instanceadmin/test_instanceadmin_async.py | 45 +++++- .../google-cloud-bigtable/samples/conftest.py | 52 +++++-- .../instanceadmin/instanceadmin.py | 8 +- .../instanceadmin/test_instanceadmin.py | 72 +++++++++- .../tests/conformance/_utils.py | 135 ++++++++++++++++++ .../tests/conformance/conftest.py | 52 ++++++- .../tests/conformance/test_bidi_reads.py | 49 ++----- .../tests/conformance/test_bidi_writes.py | 50 +++---- .../tests/conformance/test_conformance.py | 57 +++----- 13 files changed, 443 insertions(+), 146 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 3f3dcab722cf..91b566e115bb 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -281,7 +281,7 @@ substitutions: bigquery-magics: 2 google-cloud-bigquery: 8 google-cloud-bigquery-storage: 8 - google-cloud-bigtable: 15 + google-cloud-bigtable: 12 google-cloud-compute: 8 google-cloud-datastore: 8 google-cloud-dns: 1 @@ -290,7 +290,7 @@ substitutions: google-cloud-ndb: 12 google-cloud-pubsub: 15 google-cloud-spanner: 17 - google-cloud-storage: 5 + google-cloud-storage: 7 google-resumable-media: 1 pandas-gbq: 12 sqlalchemy-bigquery: 11 diff --git a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py index 04a164a0f9ce..70c3c98e28d9 100644 --- a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py +++ b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py @@ -23,7 +23,9 @@ import argparse -def run_instance_operations(project_id, instance_id, cluster_id): +def run_instance_operations( + project_id, instance_id, cluster_id, location_id="us-central1-f" +): # [START bigtable_check_instance_exists] from google.api_core.exceptions import NotFound @@ -49,7 +51,7 @@ def run_instance_operations(project_id, instance_id, cluster_id): instance_path = client.instance_path(project_id, instance_id) cluster = bigtable_admin_v2.Cluster( - location=client.common_location_path(project_id, "us-central1-f"), + location=client.common_location_path(project_id, location_id), serve_nodes=1, default_storage_type=bigtable_admin_v2.StorageType.SSD, ) @@ -128,7 +130,7 @@ def delete_instance(project_id, instance_id): # [END bigtable_delete_instance] -def add_cluster(project_id, instance_id, cluster_id): +def add_cluster(project_id, instance_id, cluster_id, location_id="us-central1-b"): # [START bigtable_create_cluster] from google.api_core.exceptions import NotFound @@ -144,7 +146,7 @@ def add_cluster(project_id, instance_id, cluster_id): print(cluster.name.split("/")[-1]) new_cluster = bigtable_admin_v2.Cluster( - location=client.common_location_path(project_id, "us-central1-a"), + location=client.common_location_path(project_id, location_id), serve_nodes=1, default_storage_type=bigtable_admin_v2.StorageType.SSD, ) diff --git a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py index 1c5203b1a680..eb6158b66429 100644 --- a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py +++ b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py @@ -28,18 +28,59 @@ INSTANCE_ID = f"instance-admin-{str(uuid.uuid4())[:8]}" CLUSTER_ID = f"cluster-admin-{str(uuid.uuid4())[:8]}" NEW_CLUSTER_ID = f"cluster-add-{str(uuid.uuid4())[:8]}" +FALLBACK_ZONES = ( + "us-central1-f", + "us-central1-b", + "us-central1-c", + "us-east1-b", + "us-west1-b", +) + + +def _is_zone_stockout(exc): + message = str(exc).lower() + return "sold out" in message or "not enough servers" in message + + +def _run_with_zone_fallback(fn, zones, capsys, *args): + last_exc = None + for zone in zones: + capsys.readouterr() + try: + fn(*args, location_id=zone) + return zone + except exceptions.GoogleAPICallError as exc: + if not _is_zone_stockout(exc): + raise + last_exc = exc + raise last_exc def test_instance_operations(capsys): try: - run_instance_operations(PROJECT, INSTANCE_ID, CLUSTER_ID) + primary_zone = _run_with_zone_fallback( + run_instance_operations, + FALLBACK_ZONES, + capsys, + PROJECT, + INSTANCE_ID, + CLUSTER_ID, + ) out, _ = capsys.readouterr() assert f"Created instance: {INSTANCE_ID}" in out assert "Listing instances:" in out assert "Listing clusters..." in out backoff_503 = backoff.on_exception(backoff.expo, exceptions.ServiceUnavailable) - backoff_503(add_cluster)(PROJECT, INSTANCE_ID, NEW_CLUSTER_ID) + secondary_zones = tuple(z for z in FALLBACK_ZONES if z != primary_zone) + _run_with_zone_fallback( + backoff_503(add_cluster), + secondary_zones, + capsys, + PROJECT, + INSTANCE_ID, + NEW_CLUSTER_ID, + ) out, _ = capsys.readouterr() assert f"Cluster created: {NEW_CLUSTER_ID}" in out diff --git a/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/instanceadmin_async.py b/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/instanceadmin_async.py index daf96c8c5bd9..0712ab3f7a1d 100644 --- a/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/instanceadmin_async.py +++ b/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/instanceadmin_async.py @@ -24,7 +24,9 @@ import asyncio -async def run_instance_operations(project_id, instance_id, cluster_id): +async def run_instance_operations( + project_id, instance_id, cluster_id, location_id="us-central1-f" +): # [START bigtable_async_check_instance_exists] from google.api_core.exceptions import NotFound @@ -50,7 +52,7 @@ async def run_instance_operations(project_id, instance_id, cluster_id): instance_path = client.instance_path(project_id, instance_id) cluster = bigtable_admin_v2.Cluster( - location=client.common_location_path(project_id, "us-central1-f"), + location=client.common_location_path(project_id, location_id), serve_nodes=1, default_storage_type=bigtable_admin_v2.StorageType.SSD, ) @@ -129,7 +131,7 @@ async def delete_instance(project_id, instance_id): # [END bigtable_async_delete_instance] -async def add_cluster(project_id, instance_id, cluster_id): +async def add_cluster(project_id, instance_id, cluster_id, location_id="us-central1-b"): # [START bigtable_async_create_cluster] from google.api_core.exceptions import NotFound @@ -145,7 +147,7 @@ async def add_cluster(project_id, instance_id, cluster_id): print(cluster.name.split("/")[-1]) new_cluster = bigtable_admin_v2.Cluster( - location=client.common_location_path(project_id, "us-central1-a"), + location=client.common_location_path(project_id, location_id), serve_nodes=1, default_storage_type=bigtable_admin_v2.StorageType.SSD, ) diff --git a/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/test_instanceadmin_async.py b/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/test_instanceadmin_async.py index 3ec71138d30d..d4965cf08390 100644 --- a/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/test_instanceadmin_async.py +++ b/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/test_instanceadmin_async.py @@ -29,19 +29,60 @@ INSTANCE_ID = f"instance-admin-async-{str(uuid.uuid4())[:8]}" CLUSTER_ID = f"cluster-admin-async-{str(uuid.uuid4())[:8]}" NEW_CLUSTER_ID = f"cluster-add-async-{str(uuid.uuid4())[:8]}" +FALLBACK_ZONES = ( + "us-central1-f", + "us-central1-b", + "us-central1-c", + "us-east1-b", + "us-west1-b", +) + + +def _is_zone_stockout(exc): + message = str(exc).lower() + return "sold out" in message or "not enough servers" in message + + +async def _run_with_zone_fallback(fn, zones, capsys, *args): + last_exc = None + for zone in zones: + capsys.readouterr() + try: + await fn(*args, location_id=zone) + return zone + except exceptions.GoogleAPICallError as exc: + if not _is_zone_stockout(exc): + raise + last_exc = exc + raise last_exc @pytest.mark.asyncio async def test_instance_operations(capsys): try: - await run_instance_operations(PROJECT, INSTANCE_ID, CLUSTER_ID) + primary_zone = await _run_with_zone_fallback( + run_instance_operations, + FALLBACK_ZONES, + capsys, + PROJECT, + INSTANCE_ID, + CLUSTER_ID, + ) out, _ = capsys.readouterr() assert f"Created instance: {INSTANCE_ID}" in out assert "Listing instances:" in out assert "Listing clusters..." in out backoff_503 = backoff.on_exception(backoff.expo, exceptions.ServiceUnavailable) - await backoff_503(add_cluster)(PROJECT, INSTANCE_ID, NEW_CLUSTER_ID) + secondary_zones = tuple(z for z in FALLBACK_ZONES if z != primary_zone) + await _run_with_zone_fallback( + backoff_503(add_cluster), + secondary_zones, + capsys, + PROJECT, + INSTANCE_ID, + NEW_CLUSTER_ID, + ) out, _ = capsys.readouterr() assert f"Cluster created: {NEW_CLUSTER_ID}" in out diff --git a/packages/google-cloud-bigtable/samples/conftest.py b/packages/google-cloud-bigtable/samples/conftest.py index 9c4bcf05612d..d43dff2e3b71 100644 --- a/packages/google-cloud-bigtable/samples/conftest.py +++ b/packages/google-cloud-bigtable/samples/conftest.py @@ -111,23 +111,45 @@ def pytest_configure(config) -> None: signal.signal(signal.SIGTERM, _sigterm_handler) print(f"\nCreating temporary sample test instance: {instance_name}") + fallback_zones = ( + "us-central1-b", + "us-central1-c", + "us-central1-f", + "us-east1-b", + "us-west1-b", + ) try: - operation = client.create_instance( - parent=project_path, - instance_id=instance_id, - instance=bigtable_admin.Instance( - display_name=instance_id, - type_=bigtable_admin.Instance.Type.PRODUCTION, - ), - clusters={ - f"{instance_id}-c1": bigtable_admin.Cluster( - location=client.common_location_path(project_id, "us-central1-b"), - serve_nodes=1, - default_storage_type=bigtable_admin.StorageType.SSD, + last_exc = None + for zone in fallback_zones: + try: + operation = client.create_instance( + parent=project_path, + instance_id=instance_id, + instance=bigtable_admin.Instance( + display_name=instance_id, + type_=bigtable_admin.Instance.Type.PRODUCTION, + ), + clusters={ + f"{instance_id}-c1": bigtable_admin.Cluster( + location=client.common_location_path(project_id, zone), + serve_nodes=1, + default_storage_type=bigtable_admin.StorageType.SSD, + ) + }, ) - }, - ) - operation.result(timeout=240) + operation.result(timeout=240) + last_exc = None + break + except exceptions.GoogleAPICallError as exc: + message = str(exc).lower() + if "sold out" not in message and "not enough servers" not in message: + raise + print( + f"Zone {zone} temporarily sold out for {instance_id}, trying next zone..." + ) + last_exc = exc + if last_exc is not None: + raise last_exc os.environ["BIGTABLE_INSTANCE"] = instance_id print(f"Created temporary sample test instance: {instance_id}") except Exception: diff --git a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py index 53f1bfbb1906..e64ff46200ff 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py @@ -36,7 +36,9 @@ from google.cloud.bigtable import enums -def run_instance_operations(project_id, instance_id, cluster_id): +def run_instance_operations( + project_id, instance_id, cluster_id, location_id="us-central1-f" +): """Check Instance exists. Creates a Production instance with default Cluster. List instances in a project. @@ -49,7 +51,6 @@ def run_instance_operations(project_id, instance_id, cluster_id): :param instance_id: Instance of the client. """ client = bigtable.Client(project=project_id, admin=True) - location_id = "us-central1-f" serve_nodes = 1 storage_type = enums.StorageType.SSD labels = {"prod-label": "prod-label"} @@ -121,7 +122,7 @@ def delete_instance(project_id, instance_id): # [END bigtable_delete_instance_legacy] -def add_cluster(project_id, instance_id, cluster_id): +def add_cluster(project_id, instance_id, cluster_id, location_id="us-central1-b"): """Add Cluster :type project_id: str @@ -136,7 +137,6 @@ def add_cluster(project_id, instance_id, cluster_id): client = bigtable.Client(project=project_id, admin=True) instance = client.instance(instance_id) - location_id = "us-central1-a" serve_nodes = 1 storage_type = enums.StorageType.SSD diff --git a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/test_instanceadmin.py b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/test_instanceadmin.py index 669280967733..b46c862f6cd7 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/test_instanceadmin.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/test_instanceadmin.py @@ -33,6 +33,32 @@ INSTANCE = INSTANCE_ID_FORMAT.format(random.randrange(ID_RANGE), int(time.time())) CLUSTER1 = CLUSTER_ID_FORMAT.format(random.randrange(ID_RANGE)) CLUSTER2 = CLUSTER_ID_FORMAT.format(random.randrange(ID_RANGE)) +FALLBACK_ZONES = ( + "us-central1-f", + "us-central1-b", + "us-central1-c", + "us-east1-b", + "us-west1-b", +) + + +def _is_zone_stockout(exc): + message = str(exc).lower() + return "sold out" in message or "not enough servers" in message + + +def _run_with_zone_fallback(fn, zones, capsys, *args): + last_exc = None + for zone in zones: + capsys.readouterr() + try: + fn(*args, location_id=zone) + return zone + except exceptions.GoogleAPICallError as exc: + if not _is_zone_stockout(exc): + raise + last_exc = exc + raise last_exc @pytest.fixture(scope="module", autouse=True) @@ -73,7 +99,14 @@ def disposal(instance): def test_run_instance_operations(capsys, dispose_of): dispose_of(INSTANCE) - instanceadmin.run_instance_operations(PROJECT, INSTANCE, CLUSTER1) + primary_zone = _run_with_zone_fallback( + instanceadmin.run_instance_operations, + FALLBACK_ZONES, + capsys, + PROJECT, + INSTANCE, + CLUSTER1, + ) out = capsys.readouterr().out assert f"Instance {INSTANCE} does not exist." in out assert "Creating an instance" in out @@ -85,7 +118,9 @@ def test_run_instance_operations(capsys, dispose_of): assert "Listing clusters..." in out assert f"\n{CLUSTER1}\n" in out - instanceadmin.run_instance_operations(PROJECT, INSTANCE, CLUSTER1) + instanceadmin.run_instance_operations( + PROJECT, INSTANCE, CLUSTER1, location_id=primary_zone + ) out = capsys.readouterr().out assert f"Instance {INSTANCE} already exists." in out assert "Listing instances" in out @@ -110,7 +145,14 @@ def _set_up_instance(): assert f"Instance {INSTANCE} does not exist" in out # Ok, create it then - instanceadmin.run_instance_operations(PROJECT, INSTANCE, CLUSTER1) + _run_with_zone_fallback( + instanceadmin.run_instance_operations, + FALLBACK_ZONES, + capsys, + PROJECT, + INSTANCE, + CLUSTER1, + ) capsys.readouterr() # throw away output _set_up_instance() @@ -135,17 +177,33 @@ def _set_up_instance(): assert f"Instance {INSTANCE} does not exist" in out # Get the instance created - instanceadmin.run_instance_operations(PROJECT, INSTANCE, CLUSTER1) + primary_zone = _run_with_zone_fallback( + instanceadmin.run_instance_operations, + FALLBACK_ZONES, + capsys, + PROJECT, + INSTANCE, + CLUSTER1, + ) capsys.readouterr() # throw away output + return primary_zone - _set_up_instance() + primary_zone = _set_up_instance() # Add a cluster to that instance # Avoid failing for "instance is currently being changed" by # applying an exponential backoff backoff_503 = backoff.on_exception(backoff.expo, exceptions.ServiceUnavailable) - backoff_503(instanceadmin.add_cluster)(PROJECT, INSTANCE, CLUSTER2) + secondary_zones = tuple(z for z in FALLBACK_ZONES if z != primary_zone) + secondary_zone = _run_with_zone_fallback( + backoff_503(instanceadmin.add_cluster), + secondary_zones, + capsys, + PROJECT, + INSTANCE, + CLUSTER2, + ) out = capsys.readouterr().out assert f"Adding cluster to instance {INSTANCE}" in out assert "Listing clusters..." in out @@ -153,7 +211,7 @@ def _set_up_instance(): assert f"Cluster created: {CLUSTER2}" in out # Try to add the same cluster again, won't work - instanceadmin.add_cluster(PROJECT, INSTANCE, CLUSTER2) + instanceadmin.add_cluster(PROJECT, INSTANCE, CLUSTER2, location_id=secondary_zone) out = capsys.readouterr().out assert "Listing clusters..." in out assert f"\n{CLUSTER1}\n" in out diff --git a/packages/google-cloud-storage/tests/conformance/_utils.py b/packages/google-cloud-storage/tests/conformance/_utils.py index 1857a46f34ef..96479f5776bf 100644 --- a/packages/google-cloud-storage/tests/conformance/_utils.py +++ b/packages/google-cloud-storage/tests/conformance/_utils.py @@ -1,8 +1,143 @@ +import os +import socket +import subprocess import time import traceback +from typing import Optional, Tuple import requests +DEFAULT_TESTBENCH_IMAGE = ( + "gcr.io/cloud-devrel-public-resources/storage-testbench:latest" +) + + +def get_container_docker_network() -> Optional[str]: + """Return a shared user-defined Docker network (e.g. 'cloudbuild') when + running inside a container attached to one, or None on a regular host. + """ + if not os.path.exists("/.dockerenv"): + return None + hostname = socket.gethostname() + try: + result = subprocess.run( + [ + "docker", + "inspect", + "-f", + "{{range $k, $v := .NetworkSettings.Networks}}{{$k}} {{end}}", + hostname, + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + if result.returncode == 0: + networks = [ + net + for net in result.stdout.strip().split() + if net and net not in ("bridge", "host", "none") + ] + if "cloudbuild" in networks: + return "cloudbuild" + if networks: + return networks[0] + # Fallback if container hostname was overridden from container ID. + cb_check = subprocess.run( + ["docker", "network", "inspect", "cloudbuild"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + check=False, + ) + if cb_check.returncode == 0: + return "cloudbuild" + except (OSError, subprocess.SubprocessError): + return None + return None + + +def stop_testbench(container_name: str) -> None: + """Force-remove a running storage-testbench container, ignoring errors.""" + subprocess.run( + ["docker", "rm", "-f", container_name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + + +def start_testbench( + container_name: str, + host_http_port: int, + host_grpc_port: Optional[int] = None, + image: str = DEFAULT_TESTBENCH_IMAGE, + ready_timeout: float = 30.0, +) -> Tuple[str, Optional[str]]: + """Pull and start storage-testbench, returning (http_endpoint, grpc_endpoint). + + When running inside a container on a Docker network (such as Cloud Build's + 'cloudbuild' network), attaches the testbench to that same network and + returns container-DNS endpoints; otherwise publishes host ports on localhost. + """ + subprocess.run(["docker", "pull", image], check=True) + stop_testbench(container_name) + + network = get_container_docker_network() + if network: + cmd = [ + "docker", + "run", + "--rm", + "-d", + "--name", + container_name, + "--network", + network, + image, + ] + http_endpoint = f"http://{container_name}:9000" + grpc_endpoint = ( + f"{container_name}:{host_grpc_port}" if host_grpc_port is not None else None + ) + else: + cmd = [ + "docker", + "run", + "--rm", + "-d", + "--name", + container_name, + "-p", + f"{host_http_port}:9000", + ] + if host_grpc_port is not None: + cmd.extend(["-p", f"{host_grpc_port}:{host_grpc_port}"]) + cmd.append(image) + http_endpoint = f"http://localhost:{host_http_port}" + grpc_endpoint = ( + f"localhost:{host_grpc_port}" if host_grpc_port is not None else None + ) + + subprocess.run(cmd, check=True) + + deadline = time.time() + ready_timeout + last_err = None + while time.time() < deadline: + try: + requests.get(f"{http_endpoint}/", timeout=2) + return http_endpoint, grpc_endpoint + except requests.exceptions.RequestException as exc: + last_err = exc + time.sleep(0.5) + + stop_testbench(container_name) + raise RuntimeError( + f"storage-testbench ({container_name}) did not become ready at " + f"{http_endpoint} within {ready_timeout}s: {last_err}" + ) + def start_grpc_server(grpc_endpoint, http_endpoint): """Starts the testbench gRPC server if it's not already running. diff --git a/packages/google-cloud-storage/tests/conformance/conftest.py b/packages/google-cloud-storage/tests/conformance/conftest.py index fb553ff1cbc3..a9bd31023323 100644 --- a/packages/google-cloud-storage/tests/conformance/conftest.py +++ b/packages/google-cloud-storage/tests/conformance/conftest.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import atexit import os import random import uuid +from typing import Optional import pytest from google.auth.credentials import AnonymousCredentials @@ -22,8 +24,55 @@ from google.cloud import storage +from ._utils import start_testbench, stop_testbench + """Environment variable or default host for Storage testbench emulator.""" _HOST = os.environ.get("STORAGE_EMULATOR_HOST", "http://localhost:9000") +_TESTBENCH_CONTAINER: Optional[str] = None + + +def _cleanup_testbench() -> None: + global _TESTBENCH_CONTAINER + container = _TESTBENCH_CONTAINER + if container: + _TESTBENCH_CONTAINER = None + stop_testbench(container) + + +def _running_retry_conformance(config) -> bool: + args = [str(arg) for arg in (config.args or [])] + if not args: + return True + return any("test_conformance" in arg or os.path.isdir(arg) for arg in args) + + +def pytest_configure(config) -> None: + """Start a single shared storage-testbench container for retry conformance tests. + + Under pytest-xdist, only the controller starts and stops the container; + workers inherit STORAGE_EMULATOR_HOST via os.environ. + """ + global _TESTBENCH_CONTAINER + if getattr(config.option, "help", False) or getattr( + config.option, "collectonly", False + ): + return + if hasattr(config, "workerinput"): + return + if os.environ.get("STORAGE_EMULATOR_HOST"): + return + if not _running_retry_conformance(config): + return + + container_name = f"storage-conformance-{uuid.uuid4().hex[:8]}" + http_endpoint, _ = start_testbench(container_name, host_http_port=9000) + _TESTBENCH_CONTAINER = container_name + atexit.register(_cleanup_testbench) + os.environ["STORAGE_EMULATOR_HOST"] = http_endpoint + + +def pytest_unconfigure(config) -> None: + _cleanup_testbench() """Emulated project information for the storage testbench.""" @@ -57,10 +106,11 @@ def _create_block(desired_kib): @pytest.fixture def client(): + host = os.environ.get("STORAGE_EMULATOR_HOST", _HOST) client = storage.Client( project=_CONF_TEST_PROJECT_ID, credentials=AnonymousCredentials(), - client_options={"api_endpoint": _HOST}, + client_options={"api_endpoint": host}, ) return client diff --git a/packages/google-cloud-storage/tests/conformance/test_bidi_reads.py b/packages/google-cloud-storage/tests/conformance/test_bidi_reads.py index 272da47bc034..3dace4d79c96 100644 --- a/packages/google-cloud-storage/tests/conformance/test_bidi_reads.py +++ b/packages/google-cloud-storage/tests/conformance/test_bidi_reads.py @@ -1,8 +1,5 @@ import io -import subprocess -import time import traceback -import urllib import uuid import pytest @@ -17,7 +14,7 @@ from google.cloud.storage.asyncio.async_multi_range_downloader import ( AsyncMultiRangeDownloader, ) -from tests.conformance._utils import start_grpc_server +from tests.conformance._utils import start_grpc_server, start_testbench, stop_testbench # --- Configuration --- @@ -25,45 +22,29 @@ TEST_BENCH_ENDPOINT = ( "http://localhost:9001" # 9000 in VM is taken by test_conformance.py ) -_PORT = urllib.parse.urlsplit(TEST_BENCH_ENDPOINT).port +_HTTP_PORT = 9001 _GRPC_PORT = 8888 PROJECT_NUMBER = "12345" # A dummy project number is fine for the testbench. GRPC_ENDPOINT = f"localhost:{_GRPC_PORT}" CONTENT_LENGTH = 1024 * 10 # 10 KB -_DEFAULT_IMAGE_NAME = "gcr.io/cloud-devrel-public-resources/storage-testbench" -_DEFAULT_IMAGE_TAG = "latest" -_DOCKER_IMAGE = f"{_DEFAULT_IMAGE_NAME}:{_DEFAULT_IMAGE_TAG}" -_PULL_CMD = ["docker", "pull", _DOCKER_IMAGE] -_RUN_CMD = [ - "docker", - "run", - "--name", - "bidi_reads_container", - "--rm", - "-d", - "-p", - f"{_PORT}:9000", - "-p", - f"{_GRPC_PORT}:{_GRPC_PORT}", - _DOCKER_IMAGE, -] -_DOCKER_STOP_CMD = [ - "docker", - "stop", - "bidi_reads_container", -] - @pytest.fixture(scope="module") def testbench(): - subprocess.run(_PULL_CMD) - proc = subprocess.Popen(_RUN_CMD) - time.sleep(10) - yield GRPC_ENDPOINT, TEST_BENCH_ENDPOINT - subprocess.run(_DOCKER_STOP_CMD) - proc.kill() + global TEST_BENCH_ENDPOINT, GRPC_ENDPOINT + container_name = f"bidi-reads-{uuid.uuid4().hex[:8]}" + http_endpoint, grpc_endpoint = start_testbench( + container_name, + host_http_port=_HTTP_PORT, + host_grpc_port=_GRPC_PORT, + ) + TEST_BENCH_ENDPOINT = http_endpoint + GRPC_ENDPOINT = grpc_endpoint + try: + yield GRPC_ENDPOINT, TEST_BENCH_ENDPOINT + finally: + stop_testbench(container_name) def _is_retriable(exc): diff --git a/packages/google-cloud-storage/tests/conformance/test_bidi_writes.py b/packages/google-cloud-storage/tests/conformance/test_bidi_writes.py index 5490a419ec84..aa423a97632c 100644 --- a/packages/google-cloud-storage/tests/conformance/test_bidi_writes.py +++ b/packages/google-cloud-storage/tests/conformance/test_bidi_writes.py @@ -1,6 +1,3 @@ -import subprocess -import time -import urllib import uuid import grpc @@ -15,51 +12,36 @@ AsyncAppendableObjectWriter, ) from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient -from tests.conformance._utils import start_grpc_server +from tests.conformance._utils import start_grpc_server, start_testbench, stop_testbench # --- Configuration --- TEST_BENCH_ENDPOINT = ( "http://localhost:9002" # 9000 in VM is taken by test_conformance.py, 9001 by reads ) -_PORT = urllib.parse.urlsplit(TEST_BENCH_ENDPOINT).port +_HTTP_PORT = 9002 _GRPC_PORT = 8888 PROJECT_NUMBER = "12345" # A dummy project number is fine for the testbench. GRPC_ENDPOINT = f"localhost:{_GRPC_PORT}" HTTP_ENDPOINT = TEST_BENCH_ENDPOINT -_DEFAULT_IMAGE_NAME = "gcr.io/cloud-devrel-public-resources/storage-testbench" -_DEFAULT_IMAGE_TAG = "latest" -_DOCKER_IMAGE = f"{_DEFAULT_IMAGE_NAME}:{_DEFAULT_IMAGE_TAG}" -_PULL_CMD = ["docker", "pull", _DOCKER_IMAGE] -_RUN_CMD = [ - "docker", - "run", - "--name", - "bidi_writes_container", - "--rm", - "-d", - "-p", - f"{_PORT}:9000", - "-p", - f"{_GRPC_PORT}:8888", - _DOCKER_IMAGE, -] -_DOCKER_STOP_CMD = [ - "docker", - "stop", - "bidi_writes_container", -] - @pytest.fixture(scope="module") def testbench(): - subprocess.run(_PULL_CMD) - proc = subprocess.Popen(_RUN_CMD) - time.sleep(10) - yield GRPC_ENDPOINT, HTTP_ENDPOINT - subprocess.run(_DOCKER_STOP_CMD) - proc.kill() + global TEST_BENCH_ENDPOINT, HTTP_ENDPOINT, GRPC_ENDPOINT + container_name = f"bidi-writes-{uuid.uuid4().hex[:8]}" + http_endpoint, grpc_endpoint = start_testbench( + container_name, + host_http_port=_HTTP_PORT, + host_grpc_port=_GRPC_PORT, + ) + TEST_BENCH_ENDPOINT = http_endpoint + HTTP_ENDPOINT = http_endpoint + GRPC_ENDPOINT = grpc_endpoint + try: + yield GRPC_ENDPOINT, HTTP_ENDPOINT + finally: + stop_testbench(container_name) CONTENT = b"A" * 1024 * 1024 * 10 # 10 MiB diff --git a/packages/google-cloud-storage/tests/conformance/test_conformance.py b/packages/google-cloud-storage/tests/conformance/test_conformance.py index 2263429acac4..fc1d662b28ed 100644 --- a/packages/google-cloud-storage/tests/conformance/test_conformance.py +++ b/packages/google-cloud-storage/tests/conformance/test_conformance.py @@ -17,10 +17,7 @@ import functools import logging import os -import subprocess import tempfile -import time -import urllib import uuid import pytest @@ -36,14 +33,6 @@ """Environment variable or default host for Storage testbench emulator.""" _HOST = os.environ.get("STORAGE_EMULATOR_HOST", "http://localhost:9000") -_PORT = urllib.parse.urlsplit(_HOST).port - -"""The storage testbench docker image info and commands.""" -_DEFAULT_IMAGE_NAME = "gcr.io/cloud-devrel-public-resources/storage-testbench" -_DEFAULT_IMAGE_TAG = "latest" -_DOCKER_IMAGE = f"{_DEFAULT_IMAGE_NAME}:{_DEFAULT_IMAGE_TAG}" -_PULL_CMD = ["docker", "pull", _DOCKER_IMAGE] -_RUN_CMD = ["docker", "run", "--rm", "-d", "-p", f"{_PORT}:9000", _DOCKER_IMAGE] _CONF_TEST_PROJECT_ID = "my-project-id" _CONF_TEST_SERVICE_ACCOUNT_EMAIL = ( @@ -922,6 +911,7 @@ def run_test_case( hmac_key, file_data, ): + host = host or os.environ.get("STORAGE_EMULATOR_HOST", _HOST) scenario = _CONFORMANCE_TESTS[scenario_id - 1] expect_success = scenario["expectSuccess"] precondition_provided = scenario["preconditionProvided"] @@ -979,29 +969,22 @@ def run_test_case( ### Run Conformance Tests for Retry Strategy ########################################################################################### ######################################################################################################################################## -# Pull storage-testbench docker image -subprocess.run(_PULL_CMD) -time.sleep(5) - -# Run docker image to start storage-testbench -with subprocess.Popen(_RUN_CMD) as proc: - # Run retry conformance tests - for scenario in _CONFORMANCE_TESTS: - id = scenario["id"] - methods = scenario["methods"] - cases = scenario["cases"] - for i, c in enumerate(cases): - for m in methods: - method_name = m["name"] - method_group = m["group"] if m.get("group", None) else m["name"] - if method_group not in method_mapping: - logging.info(f"No tests for operation {method_name}") - continue - - for lib_func in method_mapping[method_group]: - test_name = f"test-S{id}-{method_name}-{lib_func.__name__}-{i}" - globals()[test_name] = functools.partial( - run_test_case, id, m, c, lib_func, _HOST - ) - time.sleep(5) - proc.kill() +# Register retry conformance tests (storage-testbench lifecycle is managed +# once by conftest.pytest_configure / pytest_unconfigure). +for scenario in _CONFORMANCE_TESTS: + id = scenario["id"] + methods = scenario["methods"] + cases = scenario["cases"] + for i, c in enumerate(cases): + for m in methods: + method_name = m["name"] + method_group = m["group"] if m.get("group", None) else m["name"] + if method_group not in method_mapping: + logging.info(f"No tests for operation {method_name}") + continue + + for lib_func in method_mapping[method_group]: + test_name = f"test-S{id}-{method_name}-{lib_func.__name__}-{i}" + globals()[test_name] = functools.partial( + run_test_case, id, m, c, lib_func, None + ) From d358a96624a38d2bd6c843f56cb4484ed94ebf12 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 20 Nov 2026 18:49:27 +0000 Subject: [PATCH 26/29] Reapply "work in progress" This reverts commit f382dad269735163131431b9e2e129ee8cc5b5e2. --- ci/cancel_superseded_builds.py | 342 +++++++++++++++++++++++++++++++++ ci/cloudbuild/system-test.yaml | 21 +- 2 files changed, 361 insertions(+), 2 deletions(-) create mode 100644 ci/cancel_superseded_builds.py diff --git a/ci/cancel_superseded_builds.py b/ci/cancel_superseded_builds.py new file mode 100644 index 000000000000..8b160f3fade9 --- /dev/null +++ b/ci/cancel_superseded_builds.py @@ -0,0 +1,342 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Cancel superseded Cloud Build PR runs when a newer commit is pushed. + +Cloud Build triggers do not provide a native ``cancel-in-progress`` setting +(unlike GitHub Actions ``concurrency`` or Kokoro's +``stop_outdated_presubmit_builds``). When Cloud Build concurrency slots are +saturated, a newly pushed commit sits in ``QUEUED`` and cannot even start its +first build step until an older ``WORKING`` build exits. + +This script solves both cases using only the Python standard library and the +Cloud Build REST API (authenticated via the GCE metadata server): + +1. ``--cancel-older`` (run once at build startup): + Lists all ``QUEUED`` / ``WORKING`` builds for the same trigger and PR number + whose ``createTime`` is earlier than this build and cancels them. +2. ``--watch`` (run in the background while tests execute): + Polls every ``--interval`` seconds (default 30s) for any ``QUEUED`` / + ``WORKING`` build for the same trigger and PR number whose ``createTime`` is + newer than this build and targets a different ``COMMIT_SHA``. If found, this + build cancels itself via ``builds:cancel`` so its concurrency slot is freed + immediately for the newer commit. + +Requires ``PROJECT_ID``, ``LOCATION``, and ``BUILD_ID`` environment variables +(populated from Cloud Build built-in substitutions) and +``roles/cloudbuild.builds.editor`` on the build service account. + +Non-PR builds (e.g. postsubmit pushes on ``main`` where ``_PR_NUMBER`` is +absent) exit cleanly as a no-op. Any transient API/network errors fail open +with a warning so cancellation checks never break CI test execution. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + + +METADATA_TOKEN_URL = ( + "http://metadata.google.internal/computeMetadata/v1/" + "instance/service-accounts/default/token" +) + + +def _get_access_token() -> str: + """Fetch a short-lived OAuth2 access token from the GCE metadata server.""" + req = urllib.request.Request( + METADATA_TOKEN_URL, + headers={"Metadata-Flavor": "Google"}, + ) + with urllib.request.urlopen(req, timeout=10) as resp: + return json.loads(resp.read().decode("utf-8"))["access_token"] + + +def _api_request( + method: str, + url: str, + token: str, + body: bytes | None = None, +) -> dict: + """Execute an authenticated JSON HTTP request against the Cloud Build API.""" + headers = {"Authorization": f"Bearer {token}"} + if body is not None: + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=body, headers=headers, method=method) + with urllib.request.urlopen(req, timeout=15) as resp: + raw = resp.read().decode("utf-8") + return json.loads(raw) if raw else {} + + +def _parse_timestamp(raw_ts: str) -> datetime: + """Parse an RFC 3339 Cloud Build timestamp (e.g. ``2026-11-19T17:00:00.123Z``).""" + # Python 3.11+ fromisoformat handles trailing "Z", normalize for 3.10 compatibility. + normalized = raw_ts.replace("Z", "+00:00") + # Truncate sub-microsecond nanoseconds (9 digits) to 6 digits if present. + if "." in normalized: + date_part, rest = normalized.split(".", 1) + sign_idx = max(rest.find("+"), rest.find("-")) + frac = rest[:sign_idx][:6].ljust(6, "0") + tz = rest[sign_idx:] + normalized = f"{date_part}.{frac}{tz}" + return datetime.fromisoformat(normalized) + + +def _base_url(project_id: str, location: str) -> str: + """Return the regional Cloud Build v1 builds endpoint.""" + return ( + f"https://cloudbuild.googleapis.com/v1/" + f"projects/{project_id}/locations/{location}/builds" + ) + + +def _get_build(project_id: str, location: str, build_id: str, token: str) -> dict: + """Fetch metadata (trigger ID, substitutions, createTime) for a single build.""" + url = f"{_base_url(project_id, location)}/{build_id}" + return _api_request("GET", url, token) + + +def _list_active_trigger_builds( + project_id: str, + location: str, + trigger_id: str, + token: str, +) -> list[dict]: + """List all QUEUED or WORKING builds belonging to the same Cloud Build trigger.""" + filter_expr = f'trigger_id="{trigger_id}" AND (status="QUEUED" OR status="WORKING")' + builds: list[dict] = [] + page_token = "" + + while True: + params = {"filter": filter_expr, "pageSize": "100"} + if page_token: + params["pageToken"] = page_token + url = f"{_base_url(project_id, location)}?{urllib.parse.urlencode(params)}" + data = _api_request("GET", url, token) + builds.extend(data.get("builds", [])) + page_token = data.get("nextPageToken", "") + if not page_token: + break + + return builds + + +def _cancel_build(project_id: str, location: str, build_id: str, token: str) -> None: + """Issue a Cloud Build CancelBuild request for ``build_id``.""" + url = f"{_base_url(project_id, location)}/{build_id}:cancel" + _api_request("POST", url, token, body=b"{}") + + +def _extract_pr_context(build: dict) -> tuple[str, str, str, datetime] | None: + """Extract ``(trigger_id, pr_number, commit_sha, create_time)`` for a PR build. + + Returns ``None`` if the build is not associated with a trigger or pull request + (e.g. postsubmit continuous builds on ``main``). + """ + trigger_id = build.get("buildTriggerId", "") + substitutions = build.get("substitutions") or {} + pr_number = substitutions.get("_PR_NUMBER", "") + commit_sha = substitutions.get("COMMIT_SHA", "") + create_time_raw = build.get("createTime", "") + + if not trigger_id or not pr_number or not create_time_raw: + return None + + return trigger_id, pr_number, commit_sha, _parse_timestamp(create_time_raw) + + +def _cancel_older_builds( + project_id: str, + location: str, + my_build_id: str, + trigger_id: str, + pr_number: str, + my_commit_sha: str, + my_create_time: datetime, + token: str, +) -> None: + """Cancel any older QUEUED/WORKING builds for the same PR and trigger.""" + active_builds = _list_active_trigger_builds(project_id, location, trigger_id, token) + for other in active_builds: + other_id = other.get("id", "") + if not other_id or other_id == my_build_id: + continue + other_subs = other.get("substitutions") or {} + if other_subs.get("_PR_NUMBER") != pr_number: + continue + # Only cancel builds for an earlier commit so manual re-runs of the same + # commit or older timestamps are handled safely. + other_sha = other_subs.get("COMMIT_SHA", "") + if my_commit_sha and other_sha == my_commit_sha: + continue + other_create_raw = other.get("createTime", "") + if not other_create_raw: + continue + if _parse_timestamp(other_create_raw) < my_create_time: + print( + f"[cancel-superseded] Cancelling older {other.get('status')} build " + f"{other_id} (PR #{pr_number}, commit {other_sha[:8]}) superseded by " + f"{my_build_id} (commit {my_commit_sha[:8]}).", + flush=True, + ) + try: + _cancel_build(project_id, location, other_id, token) + except urllib.error.URLError as exc: + print( + f"[cancel-superseded] Warning: failed to cancel {other_id}: {exc}", + file=sys.stderr, + flush=True, + ) + + +def _watch_for_newer_build( + project_id: str, + location: str, + my_build_id: str, + trigger_id: str, + pr_number: str, + my_commit_sha: str, + my_create_time: datetime, + interval: int, +) -> None: + """Poll until a newer commit build for the same PR + trigger appears, then self-cancel.""" + while True: + time.sleep(interval) + try: + # Refresh metadata token each iteration so long-running builds never hit expiry. + token = _get_access_token() + active_builds = _list_active_trigger_builds( + project_id, location, trigger_id, token + ) + for other in active_builds: + other_id = other.get("id", "") + if not other_id or other_id == my_build_id: + continue + other_subs = other.get("substitutions") or {} + if other_subs.get("_PR_NUMBER") != pr_number: + continue + other_sha = other_subs.get("COMMIT_SHA", "") + if my_commit_sha and other_sha == my_commit_sha: + continue + other_create_raw = other.get("createTime", "") + if not other_create_raw: + continue + if _parse_timestamp(other_create_raw) > my_create_time: + print( + f"[cancel-superseded] Newer {other.get('status')} build {other_id} " + f"(PR #{pr_number}, commit {other_sha[:8]}) detected; " + f"self-cancelling {my_build_id} to free worker slot.", + flush=True, + ) + _cancel_build(project_id, location, my_build_id, token) + return + except Exception as exc: + # Fail open on transient metadata/API errors and keep watching. + print( + f"[cancel-superseded] Watchdog warning (will retry): {exc}", + file=sys.stderr, + flush=True, + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Cancel superseded Cloud Build PR runs for the same trigger." + ) + parser.add_argument( + "--cancel-older", + action="store_true", + help="Cancel older QUEUED/WORKING builds for the same PR and trigger.", + ) + parser.add_argument( + "--watch", + action="store_true", + help="Continuously watch for a newer commit on the same PR and self-cancel.", + ) + parser.add_argument( + "--interval", + type=int, + default=30, + help="Polling interval in seconds for --watch mode (default: 30).", + ) + args = parser.parse_args() + + project_id = os.environ.get("PROJECT_ID", "") + location = os.environ.get("LOCATION", "") + build_id = os.environ.get("BUILD_ID", "") + if not project_id or not location or not build_id: + return 0 + + try: + token = _get_access_token() + current_build = _get_build(project_id, location, build_id, token) + except Exception as exc: + print( + f"[cancel-superseded] Warning: unable to query build metadata ({exc}); skipping.", + file=sys.stderr, + flush=True, + ) + return 0 + + ctx = _extract_pr_context(current_build) + if ctx is None: + # Postsubmit push or manual build without _PR_NUMBER; nothing to cancel. + return 0 + + trigger_id, pr_number, my_commit_sha, my_create_time = ctx + + if args.cancel_older: + try: + _cancel_older_builds( + project_id, + location, + build_id, + trigger_id, + pr_number, + my_commit_sha, + my_create_time, + token, + ) + except Exception as exc: + print( + f"[cancel-superseded] Warning: failed to cancel older builds ({exc}).", + file=sys.stderr, + flush=True, + ) + + if args.watch: + _watch_for_newer_build( + project_id, + location, + build_id, + trigger_id, + pr_number, + my_commit_sha, + my_create_time, + args.interval, + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 91b566e115bb..70beea7fb37b 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -24,6 +24,9 @@ steps: - 'GOOGLE_CLOUD_PROJECT=${PROJECT_ID}' - 'GCLOUD_PROJECT=${PROJECT_ID}' - 'PROJECT_ID=${PROJECT_ID}' + # Cloud Build built-ins used by ci/cancel_superseded_builds.py to cancel outdated PR runs. + - 'LOCATION=${LOCATION}' + - 'BUILD_ID=${BUILD_ID}' # Pass sharding and diff parameters consumed by ci/get_package_shards.py. - 'MAX_SHARDS=${_MAX_SHARDS}' - 'SHARD_INDEX=${_SHARD_INDEX}' @@ -46,6 +49,10 @@ steps: #!/usr/bin/env bash set -eo pipefail + # Cancel any older QUEUED/WORKING builds for this PR and shard so newer pushes + # immediately take precedence instead of piling up in the Cloud Build queue. + python3 ci/cancel_superseded_builds.py --cancel-older || true + # Derive PY_VERSION (e.g. "3.12" from "system-3.12") required by ci/run_single_test.sh. export PY_VERSION="${NOX_SESSION#system-}" @@ -245,7 +252,12 @@ steps: # Export PACKAGE_LIST and delegate execution to the existing monorepo test runner script. rm -f /tmp/cloudbuild_weight_adjustments.txt - print_weight_summary() { + CANCEL_WATCHDOG_PID="" + cleanup_on_exit() { + # Stop the background cancellation watchdog so it never keeps the build container open. + if [[ -n "${CANCEL_WATCHDOG_PID}" ]]; then + kill "${CANCEL_WATCHDOG_PID}" 2>/dev/null || true + fi echo "" echo "============================================================" if [[ -s /tmp/cloudbuild_weight_adjustments.txt ]]; then @@ -256,7 +268,12 @@ steps: fi echo "============================================================" } - trap print_weight_summary EXIT + trap cleanup_on_exit EXIT + + # Run a background watchdog that self-cancels this build if a newer commit is pushed + # to the same PR while all Cloud Build worker slots are occupied. + python3 ci/cancel_superseded_builds.py --watch & + CANCEL_WATCHDOG_PID=$! export PACKAGE_LIST echo "=== System Shard ${SHARD_INDEX}/${MAX_SHARDS} running ${NOX_SESSION} for: ${PACKAGE_LIST} ===" From c1fe2f115c0abb0ae94ce56ed1b40b38016b9a24 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 20 Nov 2026 18:50:29 +0000 Subject: [PATCH 27/29] Reapply "work in progress" This reverts commit 53b63ceaa3e8d07c20507002d7e6a3c3de00fa37. --- ci/cloudbuild/system-test.yaml | 75 +++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 70beea7fb37b..7c0d65939370 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -15,6 +15,11 @@ steps: - name: 'us-central1-docker.pkg.dev/client-library-releases/release-images-prod/python-multi@sha256:d529954c373542171f9fc411e0102cebe542b511e320a965edf8820285f4f122' id: 'run-system-tests-shard' + secretEnv: + - 'GAPIC_GENERATOR_ADS_ENV' + - 'GAPIC_GENERATOR_SERVICE_ACCOUNT_JSON' + - 'GOOGLE_AUTH_SERVICE_ACCOUNT_JSON' + - 'GOOGLE_AUTH_AUTHORIZED_USER_JSON' env: # Populate standard GCP project and CI environment variables expected by system test suites. - 'CI=true' @@ -27,6 +32,7 @@ steps: # Cloud Build built-ins used by ci/cancel_superseded_builds.py to cancel outdated PR runs. - 'LOCATION=${LOCATION}' - 'BUILD_ID=${BUILD_ID}' + - 'FIRESTORE_PROJECT_ID=${_FIRESTORE_PROJECT_ID}' # Pass sharding and diff parameters consumed by ci/get_package_shards.py. - 'MAX_SHARDS=${_MAX_SHARDS}' - 'SHARD_INDEX=${_SHARD_INDEX}' @@ -86,7 +92,12 @@ steps: normalized = package_path.strip().rstrip("/") return normalized.startswith("packages/") and any( os.path.exists(os.path.join(normalized, rel_path)) - for rel_path in ("tests/system", "system_tests", "tests/system.py") + for rel_path in ( + "tests/system", + "tests/system_live", + "system_tests", + "tests/system.py", + ) ) original_get_packages = get_package_shards.get_packages @@ -179,8 +190,47 @@ steps: REAL_NOX=$(which -a nox | grep -v '/tmp/cloudbuild_shims' | head -n 1) TEMP_LOG_FILE=$(mktemp) START_TIME=$(date +%s) - echo "[${PACKAGE_NAME}] Starting nox $*" >&3 - "$REAL_NOX" "$@" 2>&1 | python3 -u -c ' + NOX_ARGS=("$@") + + # Per-package setup for google-cloud-firestore, google-auth, and gapic-generator. + if [[ "${PACKAGE_NAME}" == "google-cloud-firestore" && -n "${FIRESTORE_PROJECT_ID:-}" ]]; then + export GOOGLE_CLOUD_PROJECT="${FIRESTORE_PROJECT_ID}" + export GCLOUD_PROJECT="${FIRESTORE_PROJECT_ID}" + elif [[ "${PACKAGE_NAME}" == "google-auth" ]]; then + mkdir -p system_tests/data + printf '%s' "${GOOGLE_AUTH_SERVICE_ACCOUNT_JSON}" > system_tests/data/service_account.json + printf '%s' "${GOOGLE_AUTH_AUTHORIZED_USER_JSON}" > system_tests/data/authorized_user.json + export GOOGLE_APPLICATION_CREDENTIALS="${PWD}/system_tests/data/service_account.json" + NOX_ARGS=(--stop-on-first-error -f system_tests/noxfile.py) + elif [[ "${PACKAGE_NAME}" == "gapic-generator" ]]; then + ADS_ENV_FILE=$(mktemp) + ADS_SA_FILE=$(mktemp --suffix=.json) + trap 'rm -f "${ADS_ENV_FILE}" "${ADS_SA_FILE}" "${TEMP_LOG_FILE}"' EXIT + printf '%s\n' "${GAPIC_GENERATOR_ADS_ENV}" > "${ADS_ENV_FILE}" + printf '%s' "${GAPIC_GENERATOR_SERVICE_ACCOUNT_JSON}" > "${ADS_SA_FILE}" + set -a + source "${ADS_ENV_FILE}" + set +a + rm -f "${ADS_ENV_FILE}" + export GOOGLE_APPLICATION_CREDENTIALS="${ADS_SA_FILE}" + export GOOGLE_ADS_VIDEO_PATH="/tmp/gapic_generator_test_video.mp4" + if [[ ! -f "${GOOGLE_ADS_VIDEO_PATH}" ]]; then + if command -v ffmpeg >/dev/null 2>&1; then + ffmpeg -y -loglevel error -f lavfi -i testsrc=duration=8:size=1280x720:rate=30 -c:v libx264 -b:v 1M -pix_fmt yuv420p "${GOOGLE_ADS_VIDEO_PATH}" + else + python3 -c ' + import os, struct + ftyp = struct.pack(">I4s4sI4s", 20, b"ftyp", b"isom", 512, b"isom") + pad = b"\x00" * (600 * 1024) + free = struct.pack(">I4s", 8 + len(pad), b"free") + pad + open(os.environ["GOOGLE_ADS_VIDEO_PATH"], "wb").write(ftyp + free) + ' + fi + fi + fi + + echo "[${PACKAGE_NAME}] Starting nox ${NOX_ARGS[*]}" >&3 + "$REAL_NOX" "${NOX_ARGS[@]}" 2>&1 | python3 -u -c ' import os, sys prefix = f"[{sys.argv[1]}] " with open(sys.argv[2], "w", buffering=1) as log_f, os.fdopen(3, "w", buffering=1, closefd=False) as live_f: @@ -292,17 +342,21 @@ substitutions: _BUILD_TYPE: 'presubmit' _BASE_BRANCH: 'main' _TEST_ALL_PACKAGES: 'false' - _DISABLED_PACKAGES: 'packages/gapic-generator/ packages/google-auth/ packages/google-cloud-firestore/' + _DISABLED_PACKAGES: '' + _FIRESTORE_PROJECT_ID: '' _NOX_SESSION: 'system-3.12' _PACKAGE_WEIGHTS: | bigquery-magics: 2 + gapic-generator: 5 + google-auth: 5 google-cloud-bigquery: 8 google-cloud-bigquery-storage: 8 - google-cloud-bigtable: 12 + google-cloud-bigtable: 8 google-cloud-compute: 8 google-cloud-datastore: 8 google-cloud-dns: 1 google-cloud-error-reporting: 1 + google-cloud-firestore: 8 google-cloud-logging: 10 google-cloud-ndb: 12 google-cloud-pubsub: 15 @@ -315,3 +369,14 @@ substitutions: options: machineType: 'E2_HIGHCPU_8' logging: CLOUD_LOGGING_ONLY + +availableSecrets: + secretManager: + - versionName: projects/$PROJECT_ID/secrets/gapic-generator-ads-env/versions/latest + env: 'GAPIC_GENERATOR_ADS_ENV' + - versionName: projects/$PROJECT_ID/secrets/gapic-generator-service-account/versions/latest + env: 'GAPIC_GENERATOR_SERVICE_ACCOUNT_JSON' + - versionName: projects/$PROJECT_ID/secrets/google-auth-service-account/versions/latest + env: 'GOOGLE_AUTH_SERVICE_ACCOUNT_JSON' + - versionName: projects/$PROJECT_ID/secrets/google-auth-authorized-user/versions/latest + env: 'GOOGLE_AUTH_AUTHORIZED_USER_JSON' From f14f5280d0eed93357a20c237cdde8b93b100d0d Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 20 Nov 2026 19:56:15 +0000 Subject: [PATCH 28/29] work in progress --- .../bigtable/data/_sync_autogen/client.py | 4 +- .../tests/unit/data/_async/test_client.py | 56 ++++++++++++++++--- .../unit/data/_sync_autogen/test_client.py | 31 ++++++---- 3 files changed, 68 insertions(+), 23 deletions(-) diff --git a/packages/google-cloud-bigtable/google/cloud/bigtable/data/_sync_autogen/client.py b/packages/google-cloud-bigtable/google/cloud/bigtable/data/_sync_autogen/client.py index a485b53f78c4..e485894419cf 100644 --- a/packages/google-cloud-bigtable/google/cloud/bigtable/data/_sync_autogen/client.py +++ b/packages/google-cloud-bigtable/google/cloud/bigtable/data/_sync_autogen/client.py @@ -1424,9 +1424,7 @@ def mutate_row( def execute_rpc(): return self.client._gapic_client.mutate_row( - request=request, - timeout=next(attempt_timeout_gen), - retry=None, + request=request, timeout=next(attempt_timeout_gen), retry=None ) return tracked_retry( diff --git a/packages/google-cloud-bigtable/tests/unit/data/_async/test_client.py b/packages/google-cloud-bigtable/tests/unit/data/_async/test_client.py index 3eaffac1e639..ff6126fc65bf 100644 --- a/packages/google-cloud-bigtable/tests/unit/data/_async/test_client.py +++ b/packages/google-cloud-bigtable/tests/unit/data/_async/test_client.py @@ -908,6 +908,12 @@ async def test__multiple_table_registration(self): async with self._make_client(project="project-id") as client: async with client.get_table("instance_1", "table_1") as table_1: + assert table_1._register_instance_future is not None + if not CrossSync.is_async: + # TODO(https://github.com/googleapis/google-cloud-python/issues/19158): + # Wait for background registration until client._register_instance + # records ownership synchronously. + table_1._register_instance_future.result() instance_1_path = client._gapic_client.instance_path( client.project, "instance_1" ) @@ -1052,7 +1058,12 @@ async def test_get_api_surface(self, method): ) else: raise TypeError(f"unexpected method: {method}") - await CrossSync.yield_to_event_loop() + # TODO(https://github.com/googleapis/google-cloud-python/issues/19158): + # Wait for the background registration task instead of yield_to_event_loop() + # (a no-op in sync mode) until client._register_instance records ownership + # synchronously. + assert surface._register_instance_future is not None + await CrossSync.wait([surface._register_instance_future]) if method != "get_materialized_view": assert surface.table_id == expected_table_id assert ( @@ -1167,7 +1178,12 @@ async def test_api_surface_context_manager(self, method): else: raise TypeError(f"unexpected method: {method}") async with fn() as table: - await CrossSync.yield_to_event_loop() + # TODO(https://github.com/googleapis/google-cloud-python/issues/19158): + # Wait for the background registration task instead of + # yield_to_event_loop() (a no-op in sync mode) until + # client._register_instance records ownership synchronously. + assert table._register_instance_future is not None + await CrossSync.wait([table._register_instance_future]) assert isinstance(table, surface_type) if method != "get_materialized_view": assert table.table_id == expected_table_id @@ -1407,6 +1423,10 @@ async def test_ctor(self, use_emulator): expected_mutate_rows_attempt_timeout = 0.75 client = self._make_client(use_emulator=use_emulator) assert not client._active_instances + if not use_emulator: + # Avoid opening a real PingAndWarm RPC so background registration + # completes deterministically in sync and async modes. + client._ping_and_warm_instances = CrossSync.Mock() table = self._get_target_class()( client, @@ -1420,7 +1440,12 @@ async def test_ctor(self, use_emulator): default_mutate_rows_operation_timeout=expected_mutate_rows_operation_timeout, default_mutate_rows_attempt_timeout=expected_mutate_rows_attempt_timeout, ) - await CrossSync.yield_to_event_loop() + # TODO(https://github.com/googleapis/google-cloud-python/issues/19158): + # Wait for the background registration task instead of yield_to_event_loop() + # (a no-op in sync mode) until client._register_instance records ownership + # synchronously. + assert table._register_instance_future is not None + await CrossSync.wait([table._register_instance_future]) assert table.table_id == expected_table_id assert table.instance_id == expected_instance_id assert ( @@ -1464,7 +1489,6 @@ async def test_ctor(self, use_emulator): == expected_mutate_rows_attempt_timeout ) if use_emulator: - await table._register_instance_future assert table._register_instance_future.done() assert not table._register_instance_future.cancelled() assert table._register_instance_future.exception() is None @@ -1853,6 +1877,10 @@ async def test_ctor(self, use_emulator): expected_mutate_rows_attempt_timeout = 0.75 client = self._make_client(use_emulator=use_emulator) assert not client._active_instances + if not use_emulator: + # Avoid opening a real PingAndWarm RPC so background registration + # completes deterministically in sync and async modes. + client._ping_and_warm_instances = CrossSync.Mock() view = self._get_target_class()( client, @@ -1867,7 +1895,12 @@ async def test_ctor(self, use_emulator): default_mutate_rows_operation_timeout=expected_mutate_rows_operation_timeout, default_mutate_rows_attempt_timeout=expected_mutate_rows_attempt_timeout, ) - await CrossSync.yield_to_event_loop() + # TODO(https://github.com/googleapis/google-cloud-python/issues/19158): + # Wait for the background registration task instead of yield_to_event_loop() + # (a no-op in sync mode) until client._register_instance records ownership + # synchronously. + assert view._register_instance_future is not None + await CrossSync.wait([view._register_instance_future]) assert view.table_id == expected_table_id assert ( view.table_name @@ -1915,7 +1948,6 @@ async def test_ctor(self, use_emulator): == expected_mutate_rows_attempt_timeout ) if use_emulator: - await view._register_instance_future assert view._register_instance_future.done() assert not view._register_instance_future.cancelled() assert view._register_instance_future.exception() is None @@ -1982,6 +2014,10 @@ async def test_ctor(self, use_emulator): expected_mutate_rows_attempt_timeout = 0.75 client = self._make_client(use_emulator=use_emulator) assert not client._active_instances + if not use_emulator: + # Avoid opening a real PingAndWarm RPC so background registration + # completes deterministically in sync and async modes. + client._ping_and_warm_instances = CrossSync.Mock() view = self._get_target_class()( client, @@ -1995,7 +2031,12 @@ async def test_ctor(self, use_emulator): default_mutate_rows_operation_timeout=expected_mutate_rows_operation_timeout, default_mutate_rows_attempt_timeout=expected_mutate_rows_attempt_timeout, ) - await CrossSync.yield_to_event_loop() + # TODO(https://github.com/googleapis/google-cloud-python/issues/19158): + # Wait for the background registration task instead of yield_to_event_loop() + # (a no-op in sync mode) until client._register_instance records ownership + # synchronously. + assert view._register_instance_future is not None + await CrossSync.wait([view._register_instance_future]) assert view.instance_id == expected_instance_id assert ( view.instance_name @@ -2038,7 +2079,6 @@ async def test_ctor(self, use_emulator): == expected_mutate_rows_attempt_timeout ) if use_emulator: - await view._register_instance_future assert view._register_instance_future.done() assert not view._register_instance_future.cancelled() assert view._register_instance_future.exception() is None diff --git a/packages/google-cloud-bigtable/tests/unit/data/_sync_autogen/test_client.py b/packages/google-cloud-bigtable/tests/unit/data/_sync_autogen/test_client.py index 76910fea8852..2654f8445a4f 100644 --- a/packages/google-cloud-bigtable/tests/unit/data/_sync_autogen/test_client.py +++ b/packages/google-cloud-bigtable/tests/unit/data/_sync_autogen/test_client.py @@ -759,6 +759,8 @@ def test__multiple_table_registration(self): with self._make_client(project="project-id") as client: with client.get_table("instance_1", "table_1") as table_1: + assert table_1._register_instance_future is not None + table_1._register_instance_future.result() instance_1_path = client._gapic_client.instance_path( client.project, "instance_1" ) @@ -883,7 +885,8 @@ def test_get_api_surface(self, method): ) else: raise TypeError(f"unexpected method: {method}") - CrossSync._Sync_Impl.yield_to_event_loop() + assert surface._register_instance_future is not None + CrossSync._Sync_Impl.wait([surface._register_instance_future]) if method != "get_materialized_view": assert surface.table_id == expected_table_id assert ( @@ -983,7 +986,8 @@ def test_api_surface_context_manager(self, method): else: raise TypeError(f"unexpected method: {method}") with fn() as table: - CrossSync._Sync_Impl.yield_to_event_loop() + assert table._register_instance_future is not None + CrossSync._Sync_Impl.wait([table._register_instance_future]) assert isinstance(table, surface_type) if method != "get_materialized_view": assert table.table_id == expected_table_id @@ -1176,6 +1180,8 @@ def test_ctor(self, use_emulator): expected_mutate_rows_attempt_timeout = 0.75 client = self._make_client(use_emulator=use_emulator) assert not client._active_instances + if not use_emulator: + client._ping_and_warm_instances = CrossSync._Sync_Impl.Mock() table = self._get_target_class()( client, expected_instance_id, @@ -1188,7 +1194,8 @@ def test_ctor(self, use_emulator): default_mutate_rows_operation_timeout=expected_mutate_rows_operation_timeout, default_mutate_rows_attempt_timeout=expected_mutate_rows_attempt_timeout, ) - CrossSync._Sync_Impl.yield_to_event_loop() + assert table._register_instance_future is not None + CrossSync._Sync_Impl.wait([table._register_instance_future]) assert table.table_id == expected_table_id assert table.instance_id == expected_instance_id assert ( @@ -1232,7 +1239,6 @@ def test_ctor(self, use_emulator): == expected_mutate_rows_attempt_timeout ) if use_emulator: - table._register_instance_future assert table._register_instance_future.done() assert not table._register_instance_future.cancelled() assert table._register_instance_future.exception() is None @@ -1531,6 +1537,8 @@ def test_ctor(self, use_emulator): expected_mutate_rows_attempt_timeout = 0.75 client = self._make_client(use_emulator=use_emulator) assert not client._active_instances + if not use_emulator: + client._ping_and_warm_instances = CrossSync._Sync_Impl.Mock() view = self._get_target_class()( client, expected_instance_id, @@ -1544,7 +1552,8 @@ def test_ctor(self, use_emulator): default_mutate_rows_operation_timeout=expected_mutate_rows_operation_timeout, default_mutate_rows_attempt_timeout=expected_mutate_rows_attempt_timeout, ) - CrossSync._Sync_Impl.yield_to_event_loop() + assert view._register_instance_future is not None + CrossSync._Sync_Impl.wait([view._register_instance_future]) assert view.table_id == expected_table_id assert ( view.table_name @@ -1592,7 +1601,6 @@ def test_ctor(self, use_emulator): == expected_mutate_rows_attempt_timeout ) if use_emulator: - view._register_instance_future assert view._register_instance_future.done() assert not view._register_instance_future.cancelled() assert view._register_instance_future.exception() is None @@ -1654,6 +1662,8 @@ def test_ctor(self, use_emulator): expected_mutate_rows_attempt_timeout = 0.75 client = self._make_client(use_emulator=use_emulator) assert not client._active_instances + if not use_emulator: + client._ping_and_warm_instances = CrossSync._Sync_Impl.Mock() view = self._get_target_class()( client, expected_instance_id, @@ -1666,7 +1676,8 @@ def test_ctor(self, use_emulator): default_mutate_rows_operation_timeout=expected_mutate_rows_operation_timeout, default_mutate_rows_attempt_timeout=expected_mutate_rows_attempt_timeout, ) - CrossSync._Sync_Impl.yield_to_event_loop() + assert view._register_instance_future is not None + CrossSync._Sync_Impl.wait([view._register_instance_future]) assert view.instance_id == expected_instance_id assert ( view.instance_name @@ -1709,7 +1720,6 @@ def test_ctor(self, use_emulator): == expected_mutate_rows_attempt_timeout ) if use_emulator: - view._register_instance_future assert view._register_instance_future.done() assert not view._register_instance_future.cancelled() assert view._register_instance_future.exception() is None @@ -2788,10 +2798,7 @@ def test_mutate_row_attempt_timeout_per_retry(self): ] mutation = mutations.DeleteAllFromRow() table.mutate_row( - "row_key", - mutation, - operation_timeout=10.0, - attempt_timeout=5.0, + "row_key", mutation, operation_timeout=10.0, attempt_timeout=5.0 ) mock_timeout_gen.assert_called_once_with(5.0, 10.0) assert mock_gapic.call_count == 2 From 5110ab0458f9f505f550dd4b48361dc07620d735 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 20 Nov 2026 20:09:21 +0000 Subject: [PATCH 29/29] work in progress --- ci/cancel_superseded_builds.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/ci/cancel_superseded_builds.py b/ci/cancel_superseded_builds.py index 8b160f3fade9..ad28a0aba426 100644 --- a/ci/cancel_superseded_builds.py +++ b/ci/cancel_superseded_builds.py @@ -29,9 +29,9 @@ 2. ``--watch`` (run in the background while tests execute): Polls every ``--interval`` seconds (default 30s) for any ``QUEUED`` / ``WORKING`` build for the same trigger and PR number whose ``createTime`` is - newer than this build and targets a different ``COMMIT_SHA``. If found, this - build cancels itself via ``builds:cancel`` so its concurrency slot is freed - immediately for the newer commit. + newer than this build (whether from a new commit or a duplicate re-run). If + found, this build cancels itself via ``builds:cancel`` so its concurrency + slot is freed immediately for the newer run. Requires ``PROJECT_ID``, ``LOCATION``, and ``BUILD_ID`` environment variables (populated from Cloud Build built-in substitutions) and @@ -183,15 +183,17 @@ def _cancel_older_builds( other_subs = other.get("substitutions") or {} if other_subs.get("_PR_NUMBER") != pr_number: continue - # Only cancel builds for an earlier commit so manual re-runs of the same - # commit or older timestamps are handled safely. other_sha = other_subs.get("COMMIT_SHA", "") - if my_commit_sha and other_sha == my_commit_sha: - continue other_create_raw = other.get("createTime", "") if not other_create_raw: continue - if _parse_timestamp(other_create_raw) < my_create_time: + # Cancel any earlier build for the same PR and trigger — including + # duplicate runs / manual re-runs of the same commit SHA — using + # build ID as a deterministic tie-breaker if timestamps match. + if (_parse_timestamp(other_create_raw), other_id) < ( + my_create_time, + my_build_id, + ): print( f"[cancel-superseded] Cancelling older {other.get('status')} build " f"{other_id} (PR #{pr_number}, commit {other_sha[:8]}) superseded by " @@ -235,12 +237,13 @@ def _watch_for_newer_build( if other_subs.get("_PR_NUMBER") != pr_number: continue other_sha = other_subs.get("COMMIT_SHA", "") - if my_commit_sha and other_sha == my_commit_sha: - continue other_create_raw = other.get("createTime", "") if not other_create_raw: continue - if _parse_timestamp(other_create_raw) > my_create_time: + if (_parse_timestamp(other_create_raw), other_id) > ( + my_create_time, + my_build_id, + ): print( f"[cancel-superseded] Newer {other.get('status')} build {other_id} " f"(PR #{pr_number}, commit {other_sha[:8]}) detected; "