diff --git a/.librarian/generator-input/client-post-processing/bigtable-integration.yaml b/.librarian/generator-input/client-post-processing/bigtable-integration.yaml index 7bb45a26eaea..af3fcfcf9270 100644 --- a/.librarian/generator-input/client-post-processing/bigtable-integration.yaml +++ b/.librarian/generator-input/client-post-processing/bigtable-integration.yaml @@ -405,6 +405,7 @@ replacements: SYSTEM_TEST_STANDARD_DEPENDENCIES: List[str] = [ "mock", "pytest", + "pytest-xdist", "google-cloud-testutils", ] SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [ @@ -696,6 +697,19 @@ replacements: test_map[test_type](session) + def _has_xdist_arg(posargs): + return any( + arg == "-n" + or ( + arg.startswith("-n") + and (arg[2:].isdigit() or arg[2:] in ("auto", "logical")) + ) + or arg == "--numprocesses" + or arg.startswith("--numprocesses=") + for arg in posargs + ) + + def system_default(session): """Run the system test suite.""" constraints_path = str( @@ -729,9 +743,18 @@ replacements: *session.posargs, ) if system_test_folder_exists: + in_emulator = bool( + session.env.get("BIGTABLE_EMULATOR_HOST") + or os.environ.get("BIGTABLE_EMULATOR_HOST") + ) + has_xdist_arg = _has_xdist_arg(session.posargs) + xdist_args = ( + [] if (in_emulator or has_xdist_arg) else ["-n", "8", "--dist=loadfile"] + ) session.run( "py.test", "--quiet", + *xdist_args, f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, *session.posargs, @@ -752,6 +775,7 @@ replacements: "mock", "pytest", "pytest-asyncio", + "pytest-xdist", *req_args, ) # Install the library from this checkout last so the samples exercise the @@ -761,9 +785,12 @@ replacements: has_target = any(not arg.startswith("-") for arg in session.posargs) target_paths = [] if has_target else [os.path.join("samples")] + has_xdist_arg = _has_xdist_arg(session.posargs) + xdist_args = [] if has_xdist_arg else ["-n", "8", "--dist=loadfile"] session.run( "py.test", "--quiet", + *xdist_args, f"--junitxml=samples_{session.python}_sponge_log.xml", "--ignore=samples/legacy_client/beam", *target_paths, diff --git a/ci/cloudbuild/lint-docs.yaml b/ci/cloudbuild/lint-docs.yaml new file mode 100644 index 000000000000..22ee97867df4 --- /dev/null +++ b/ci/cloudbuild/lint-docs.yaml @@ -0,0 +1,266 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - name: 'us-central1-docker.pkg.dev/client-library-releases/release-images-prod/python-multi@sha256:d529954c373542171f9fc411e0102cebe542b511e320a965edf8820285f4f122' + id: 'run-lint-docs-shard' + env: + - 'CI=true' + - 'PIP_BREAK_SYSTEM_PACKAGES=1' + # Use uv for fast isolated virtualenvs per package worker without shared-env lock races. + - 'NOX_DEFAULT_VENV_BACKEND=uv' + - 'UV_VENV_SEED=1' + - 'UV_CONCURRENT_BUILDS=1' + # Pass sharding and diff parameters consumed by ci/get_package_shards.py. + - 'MAX_SHARDS=${_MAX_SHARDS}' + - 'SHARD_INDEX=${_SHARD_INDEX}' + - 'MIN_SHARD_WEIGHT=${_MIN_SHARD_WEIGHT}' + - 'BUILD_TYPE=${_BUILD_TYPE}' + - 'TARGET_BRANCH=${_BASE_BRANCH}' + - 'TEST_ALL_PACKAGES=${_TEST_ALL_PACKAGES}' + - 'DISABLED_PACKAGES=${_DISABLED_PACKAGES}' + - 'PACKAGE_WEIGHTS=${_PACKAGE_WEIGHTS}' + # Accept _NOX_SESSION from Cloud Build trigger substitutions while running all configured _TEST_TYPES. + - 'TRIGGER_NOX_SESSION=${_NOX_SESSION}' + - 'TEST_TYPES=${_TEST_TYPES}' + # Configure parallel package execution, time budget/threshold, and unbuffered Python output. + - 'PARALLEL_WORKERS=${_PARALLEL_WORKERS}' + - 'CONTINUE_ON_ERROR=true' + - 'TIME_BUDGET_MINUTES=${_TIME_BUDGET_MINUTES}' + - 'HARD_TIMEOUT_MINUTES=${_HARD_TIMEOUT_MINUTES}' + - 'PYTHONUNBUFFERED=1' + script: | + #!/usr/bin/env bash + set -eo pipefail + + # ci/run_single_test.sh requires PY_VERSION to be non-empty even for unversioned sessions. + export PY_VERSION="unused" + + # Deepen both the checked-out PR commit and target branch history so git diff in + # ci/get_package_shards.py finds the merge base in Cloud Build's shallow checkout, + # and populate local refs/heads/${TARGET_BRANCH} so ci/run_conditional_tests.sh's + # local insteadOf fetch succeeds silently. + git fetch --no-tags --quiet origin \ + "$(git rev-parse HEAD)" \ + "+refs/heads/${TARGET_BRANCH}:refs/remotes/origin/${TARGET_BRANCH}" \ + "+refs/heads/${TARGET_BRANCH}:refs/heads/${TARGET_BRANCH}" \ + --depth=200 || true + + # Import ci/get_package_shards.py without modifying the script on disk. + # Filter get_packages() in memory to exclude _DISABLED_PACKAGES. + PACKAGE_LIST=$(python3 -c ' + import os + import sys + + sys.path.insert(0, "ci") + import get_package_shards + + disabled_packages = { + package_path.strip().rstrip("/") + for package_path in os.environ.get("DISABLED_PACKAGES", "").replace(",", " ").split() + if package_path.strip() + } + + original_get_packages = get_package_shards.get_packages + def filtered_get_packages(*args, **kwargs): + filtered_packages = {} + for package_name, package_paths in original_get_packages( + handwritten_only=False + ).items(): + kept_paths = [ + path + for path in package_paths + if path.strip().rstrip("/") not in disabled_packages + ] + if kept_paths: + filtered_packages[package_name] = kept_paths + return filtered_packages + + get_package_shards.get_packages = filtered_get_packages + + packages_to_test = get_package_shards.get_packages_to_test() + shards = get_package_shards.group_packages(packages_to_test) + + shard_index = int(os.environ.get("SHARD_INDEX", "0")) + print(shards[shard_index].get("packages", "") if shard_index < len(shards) else "") + ') + + # Report immediate green (< 10s) if no packages in this PR map to this shard index. + if [[ -z "${PACKAGE_LIST}" ]]; then + echo "No packages assigned to shard ${SHARD_INDEX}. Exiting 0." + exit 0 + fi + + echo "Installing build, twine, nox, and uv..." + python3 -m pip install --quiet --upgrade setuptools pip wheel build twine nox uv + + # Create runtime shims in /tmp/cloudbuild_shims so tracked repository files remain unmodified. + mkdir -p /tmp/cloudbuild_shims /tmp/cloudbuild_pkg_times + + # Nox wrapper (/tmp/cloudbuild_shims/nox): + # - Streams live output prefixed with `[:]` to fd 3 so Cloud Build + # logs show real-time progress across parallel workers. + # - Records per-session elapsed seconds in /tmp/cloudbuild_pkg_times/ so the + # EXIT summary evaluates cumulative package runtime across all lint/docs checks. + # - Emits full session output on failure so ruff/flake8/mypy/sphinx errors appear in the + # end-of-phase failure summary from ci/run_conditional_tests.sh. + cat << 'EOF' > /tmp/cloudbuild_shims/nox + #!/usr/bin/env bash + PACKAGE_NAME=$(basename "$PWD") + SESSION_LABEL="${TEST_TYPE:-nox}" + TEMP_LOG_FILE=$(mktemp) + START_TIME=$(date +%s) + echo "[${PACKAGE_NAME}:${SESSION_LABEL}] Starting nox $*" >&3 + python3 -m nox "$@" 2>&1 | python3 -u -c ' + import os, sys + sys.stdin.reconfigure(encoding="utf-8", errors="replace") + prefix = f"[{sys.argv[1]}] " + with open(sys.argv[2], "w", buffering=1, encoding="utf-8", errors="replace") as log_f, \ + os.fdopen(3, "w", buffering=1, closefd=False, encoding="utf-8", errors="replace") as live_f: + for line in sys.stdin: + log_f.write(line) + live_f.write(prefix + line) + ' "${PACKAGE_NAME}:${SESSION_LABEL}" "${TEMP_LOG_FILE}" + EXIT_CODE=${PIPESTATUS[0]} + + ELAPSED_SECONDS=$(( $(date +%s) - START_TIME )) + echo "${ELAPSED_SECONDS}" >> "/tmp/cloudbuild_pkg_times/${PACKAGE_NAME}" + echo "[${PACKAGE_NAME}:${SESSION_LABEL}] Completed in $(( ELAPSED_SECONDS / 60 ))m $(printf '%02d' $(( ELAPSED_SECONDS % 60 )))s (exit=${EXIT_CODE})" >&3 + + if [[ ${EXIT_CODE} -ne 0 ]]; then + cat "${TEMP_LOG_FILE}" + fi + rm -f "${TEMP_LOG_FILE}" + exit ${EXIT_CODE} + EOF + chmod +x /tmp/cloudbuild_shims/nox + + export PATH="/tmp/cloudbuild_shims:${PATH}" + exec 3>&1 + + print_weight_summary() { + python3 -c ' + import glob + import os + + budget_minutes = int(os.environ.get("TIME_BUDGET_MINUTES", "20")) + hard_minutes = int(os.environ.get("HARD_TIMEOUT_MINUTES", "30")) + budget_seconds = budget_minutes * 60 + + weights = {} + for line in os.environ.get("PACKAGE_WEIGHTS", "").splitlines(): + line = line.strip() + if line and not line.startswith("#") and ":" in line: + k, v = line.split(":", 1) + try: + weights[k.strip()] = int(v.strip()) + except ValueError: + pass + + adjustments = [] + print("\n============================================================") + print("Per-package cumulative lint & docs runtime:") + for path in sorted(glob.glob("/tmp/cloudbuild_pkg_times/*")): + pkg = os.path.basename(path) + with open(path) as f: + elapsed = sum(int(x.strip()) for x in f if x.strip().isdigit()) + pct = (elapsed * 100 + budget_seconds // 2) // budget_seconds + mins, secs = divmod(elapsed, 60) + configured = weights.get(pkg, 1) + proposed = max(1, (elapsed + 30) // 60) + print( + f" {pkg}: {mins}m {secs:02d}s ({pct}% of {budget_minutes}m budget, " + f"{hard_minutes}m hard threshold; weight={configured})" + ) + if abs(proposed - configured) >= 3: + adjustments.append( + f" {pkg}: {proposed} # was {configured} (actual: {mins}m {secs:02d}s)" + ) + + print("------------------------------------------------------------") + if adjustments: + print("⚠️ WEIGHT ADJUSTMENTS NEEDED (drifted >= 3m from configured weight):") + for item in adjustments: + print(item) + else: + print("✅ WEIGHT SUMMARY: No weight adjustments needed (all packages within 2m of configured weight).") + print("============================================================") + ' + } + trap print_weight_summary EXIT + + export PACKAGE_LIST + echo "=== Lint & Docs Shard ${SHARD_INDEX}/${MAX_SHARDS} running [${TEST_TYPES}] for: ${PACKAGE_LIST} ===" + + # Pre-build local gcp-sphinx-docfx-yaml wheel if docfx is included in TEST_TYPES + # (mirrors .github/workflows/docs.yml so local changes to gcp-sphinx-docfx-yaml are tested). + if [[ " ${TEST_TYPES} " == *" docfx "* ]] && [[ -d "packages/gcp-sphinx-docfx-yaml" ]]; then + mkdir -p /tmp/local_wheels + python3 -m pip wheel --quiet --no-deps packages/gcp-sphinx-docfx-yaml -w /tmp/local_wheels + WHEEL_PATHS=(/tmp/local_wheels/gcp_sphinx_docfx_yaml-*.whl) + if [[ -f "${WHEEL_PATHS[0]}" ]]; then + echo "gcp-sphinx-docfx-yaml @ file://${WHEEL_PATHS[0]}" > /tmp/uv_overrides.txt + else + echo "ERROR: Expected local wheel /tmp/local_wheels/gcp_sphinx_docfx_yaml-*.whl not found." >&2 + exit 1 + fi + fi + + OVERALL_EXIT=0 + for test_type in ${TEST_TYPES}; do + echo "" + echo ">>> Starting phase: TEST_TYPE=${test_type} <<<" + export TEST_TYPE="${test_type}" + if [[ "${test_type}" == "docfx" ]] && [[ -f /tmp/uv_overrides.txt ]]; then + UV_OVERRIDE=/tmp/uv_overrides.txt \ + UV_FIND_LINKS=/tmp/local_wheels \ + PIP_FIND_LINKS=/tmp/local_wheels \ + ./ci/run_conditional_tests.sh || OVERALL_EXIT=1 + else + ./ci/run_conditional_tests.sh || OVERALL_EXIT=1 + fi + done + + exit ${OVERALL_EXIT} + +# Hard 30-minute (1800s) CI timeout ceiling. +timeout: 1800s + +substitutions: + _MAX_SHARDS: '5' + _SHARD_INDEX: '0' + _MIN_SHARD_WEIGHT: '10' + _PARALLEL_WORKERS: '4' + _TIME_BUDGET_MINUTES: '20' + _HARD_TIMEOUT_MINUTES: '30' + _BUILD_TYPE: 'presubmit' + _BASE_BRANCH: 'main' + _TEST_ALL_PACKAGES: 'false' + _DISABLED_PACKAGES: '' + _NOX_SESSION: 'lint' + _TEST_TYPES: 'lint lint_setup_py twine_check mypy docs docfx' + # Initial weights seeded from .github/workflows/docs.yml for large autodoc workloads. + _PACKAGE_WEIGHTS: | + google-ads-admanager: 9 + google-cloud-compute: 50 + google-cloud-compute-v1beta: 120 + google-cloud-dialogflow: 9 + google-cloud-dialogflow-cx: 9 + google-cloud-discoveryengine: 15 + google-cloud-network-security: 9 + google-cloud-retail: 9 + +options: + machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml new file mode 100644 index 000000000000..91b566e115bb --- /dev/null +++ b/ci/cloudbuild/system-test.yaml @@ -0,0 +1,300 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - name: 'us-central1-docker.pkg.dev/client-library-releases/release-images-prod/python-multi@sha256:d529954c373542171f9fc411e0102cebe542b511e320a965edf8820285f4f122' + id: 'run-system-tests-shard' + env: + # Populate standard GCP project and CI environment variables expected by system test suites. + - 'CI=true' + - 'PIP_BREAK_SYSTEM_PACKAGES=1' + - 'UV_VENV_SEED=1' + - 'RUN_COMPLIANCE_TESTS=false' + - 'GOOGLE_CLOUD_PROJECT=${PROJECT_ID}' + - 'GCLOUD_PROJECT=${PROJECT_ID}' + - 'PROJECT_ID=${PROJECT_ID}' + # Pass sharding and diff parameters consumed by ci/get_package_shards.py. + - 'MAX_SHARDS=${_MAX_SHARDS}' + - 'SHARD_INDEX=${_SHARD_INDEX}' + - 'MIN_SHARD_WEIGHT=${_MIN_SHARD_WEIGHT}' + - 'BUILD_TYPE=${_BUILD_TYPE}' + - 'TARGET_BRANCH=${_BASE_BRANCH}' + - 'TEST_ALL_PACKAGES=${_TEST_ALL_PACKAGES}' + - 'DISABLED_PACKAGES=${_DISABLED_PACKAGES}' + - 'PACKAGE_WEIGHTS=${_PACKAGE_WEIGHTS}' + # Specify the nox session consumed by ci/run_conditional_tests.sh and ci/run_single_test.sh. + - 'NOX_SESSION=${_NOX_SESSION}' + - 'TEST_TYPE=${_NOX_SESSION}' + # Configure parallel package execution, time budget/threshold, and unbuffered Python output. + - 'PARALLEL_WORKERS=${_PARALLEL_WORKERS}' + - 'CONTINUE_ON_ERROR=true' + - 'TIME_BUDGET_MINUTES=${_TIME_BUDGET_MINUTES}' + - 'HARD_TIMEOUT_MINUTES=${_HARD_TIMEOUT_MINUTES}' + - 'PYTHONUNBUFFERED=1' + script: | + #!/usr/bin/env bash + set -eo pipefail + + # Derive PY_VERSION (e.g. "3.12" from "system-3.12") required by ci/run_single_test.sh. + export PY_VERSION="${NOX_SESSION#system-}" + + # Deepen both the checked-out PR commit and target branch history so git diff in + # ci/get_package_shards.py finds the merge base in Cloud Build's shallow checkout, + # and populate local refs/heads/${TARGET_BRANCH} so ci/run_conditional_tests.sh's + # local insteadOf fetch succeeds silently. + git fetch --no-tags --quiet origin \ + "$(git rev-parse HEAD)" \ + "+refs/heads/${TARGET_BRANCH}:refs/remotes/origin/${TARGET_BRANCH}" \ + "+refs/heads/${TARGET_BRANCH}:refs/heads/${TARGET_BRANCH}" \ + --depth=200 || true + + # Import ci/get_package_shards.py without modifying the script on disk. + # Filter get_packages() in memory to exclude _DISABLED_PACKAGES. + PACKAGE_LIST=$(python3 -c ' + import os + import sys + + sys.path.insert(0, "ci") + import get_package_shards + + # Normalize disabled package paths by stripping trailing slashes. + disabled_packages = { + package_path.strip().rstrip("/") + for package_path in os.environ.get("DISABLED_PACKAGES", "").replace(",", " ").split() + if package_path.strip() + } + + def has_system_tests(package_path): + normalized = package_path.strip().rstrip("/") + return normalized.startswith("packages/") and any( + os.path.exists(os.path.join(normalized, rel_path)) + for rel_path in ("tests/system", "system_tests", "tests/system.py") + ) + + original_get_packages = get_package_shards.get_packages + def filtered_get_packages(*args, **kwargs): + filtered_packages = {} + for package_name, package_paths in original_get_packages( + handwritten_only=False + ).items(): + # Keep only packages/ paths with system tests that are NOT in disabled_packages + kept_paths = [ + path + for path in package_paths + if path.strip().rstrip("/") not in disabled_packages + and has_system_tests(path) + ] + if kept_paths: + filtered_packages[package_name] = kept_paths + return filtered_packages + + get_package_shards.get_packages = filtered_get_packages + + # Compute the packages that need testing and distribute them across balanced shards. + packages_to_test = get_package_shards.get_packages_to_test() + shards = get_package_shards.group_packages(packages_to_test) + + shard_index = int(os.environ.get("SHARD_INDEX", "0")) + print(shards[shard_index].get("packages", "") if shard_index < len(shards) else "") + ') + + # Report immediate green (< 10s) if no packages in this PR map to this shard index. + if [[ -z "${PACKAGE_LIST}" ]]; then + echo "No packages assigned to shard ${SHARD_INDEX}. Exiting 0." + exit 0 + fi + + # Install uv so nox sessions use fast cached wheel installation instead of slow virtualenv + pip. + python3 -m pip install --quiet uv || true + + # Create runtime shims in /tmp/cloudbuild_shims so tracked repository files remain unmodified. + mkdir -p /tmp/cloudbuild_shims + + # 1. Python startup hook (sitecustomize.py) & Pytest plugin (cloudbuild_pytest_plugin.py): + # - Package noxfile.py files skip the system session if GOOGLE_APPLICATION_CREDENTIALS + # is unset. We export a placeholder value ("CLOUD_BUILD_ADC") to satisfy noxfile.py, + # then remove it in sitecustomize.py for every non-nox Python subprocess (such as + # sqlalchemy-spanner's create_test_database.py and pytest) so google.auth.default() + # uses the Cloud Build worker VM's Application Default Credentials (ADC). + # - Package noxfile.py files invoke `py.test --quiet`, which only outputs dots (".") + # without newlines and causes Cloud Build logs to appear frozen for several minutes. + # Overriding `quiet = 0` and `verbose = 1` makes pytest print one newline-terminated + # status line as each individual test finishes. + cat << 'EOF' > /tmp/cloudbuild_shims/sitecustomize.py + import os + import sys + + if ( + os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") == "CLOUD_BUILD_ADC" + and os.path.basename(sys.argv[0]) != "nox" + ): + os.environ.pop("GOOGLE_APPLICATION_CREDENTIALS", None) + EOF + + cat << 'EOF' > /tmp/cloudbuild_shims/cloudbuild_pytest_plugin.py + import os + + if os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") == "CLOUD_BUILD_ADC": + os.environ.pop("GOOGLE_APPLICATION_CREDENTIALS", None) + + + def pytest_configure(config): + config.option.quiet = 0 + config.option.verbose = max(config.option.verbose, 1) + EOF + + # 2. Nox wrapper (/tmp/cloudbuild_shims/nox): + # - When PARALLEL_WORKERS > 1, `ci/run_conditional_tests.sh` redirects each worker's + # stdout/stderr (file descriptors 1 and 2) to a temporary log file, which otherwise + # produces no console output until the entire package finishes. + # - Before invoking `ci/run_conditional_tests.sh`, we duplicate the container's live + # stdout to file descriptor 3 (`exec 3>&1`). This wrapper streams every output line + # prefixed with `[]` directly to file descriptor 3 in real time while + # also saving the raw output to a temporary file (`TEMP_LOG_FILE`). + # - At completion, it calculates how much of the per-package time budget was used + # (e.g., 18 minutes out of a 20-minute budget = 90%) and logs the summary. + # - If the nox session fails, it outputs the `=== FAILURES ===` section to stdout so + # `ci/run_conditional_tests.sh` prints a clean failure summary at the end of the run. + cat << 'EOF' > /tmp/cloudbuild_shims/nox + #!/usr/bin/env bash + PACKAGE_NAME=$(basename "$PWD") + REAL_NOX=$(which -a nox | grep -v '/tmp/cloudbuild_shims' | head -n 1) + TEMP_LOG_FILE=$(mktemp) + START_TIME=$(date +%s) + echo "[${PACKAGE_NAME}] Starting nox $*" >&3 + "$REAL_NOX" "$@" 2>&1 | python3 -u -c ' + import os, sys + prefix = f"[{sys.argv[1]}] " + with open(sys.argv[2], "w", buffering=1) as log_f, os.fdopen(3, "w", buffering=1, closefd=False) as live_f: + for line in sys.stdin: + log_f.write(line) + live_f.write(prefix + line) + ' "${PACKAGE_NAME}" "${TEMP_LOG_FILE}" + EXIT_CODE=${PIPESTATUS[0]} + + # Calculate elapsed time and percentage of the target time budget spent on this package. + ELAPSED_SECONDS=$(( $(date +%s) - START_TIME )) + BUDGET_MINUTES="${TIME_BUDGET_MINUTES:-20}" + HARD_LIMIT_MINUTES="${HARD_TIMEOUT_MINUTES:-30}" + BUDGET_SECONDS=$(( BUDGET_MINUTES * 60 )) + BUDGET_PERCENTAGE=$(( (ELAPSED_SECONDS * 100 + BUDGET_SECONDS / 2) / BUDGET_SECONDS )) + BUDGET_MESSAGE=$(printf "Time budget usage: %d%% (%dm %02ds / %dm budget, %dm hard threshold)" \ + "${BUDGET_PERCENTAGE}" "$(( ELAPSED_SECONDS / 60 ))" "$(( ELAPSED_SECONDS % 60 ))" \ + "${BUDGET_MINUTES}" "${HARD_LIMIT_MINUTES}") + echo "[${PACKAGE_NAME}] ${BUDGET_MESSAGE}" >&3 + echo "${BUDGET_MESSAGE}" + + # Compare actual runtime in minutes against the configured weight in PACKAGE_WEIGHTS + # (defaulting to 1). Only propose an adjustment when the difference is at least 3 minutes + # (>= 15% of the 20-minute budget) to avoid noise from normal test runtime variance. + PROPOSED_WEIGHT=$(( (ELAPSED_SECONDS + 30) / 60 )) + if [[ ${PROPOSED_WEIGHT} -lt 1 ]]; then + PROPOSED_WEIGHT=1 + fi + CONFIGURED_WEIGHT=$(printf '%s\n' "${PACKAGE_WEIGHTS:-}" | awk -F: -v package_name="${PACKAGE_NAME}" ' + { gsub(/^[[:space:]]+|[[:space:]]+$/, "", $1); gsub(/[[:space:]]/, "", $2) } + $1 == package_name && $2 != "" { print $2 } + ' | tail -n 1) + CONFIGURED_WEIGHT="${CONFIGURED_WEIGHT:-1}" + WEIGHT_DIFFERENCE=$(( PROPOSED_WEIGHT - CONFIGURED_WEIGHT )) + if [[ ${WEIGHT_DIFFERENCE} -lt 0 ]]; then + WEIGHT_DIFFERENCE=$(( -WEIGHT_DIFFERENCE )) + fi + if [[ ${WEIGHT_DIFFERENCE} -ge 3 ]]; then + WEIGHT_MESSAGE="Weight adjustment needed for ${PACKAGE_NAME}: configured=${CONFIGURED_WEIGHT}, proposed=${PROPOSED_WEIGHT} (${PACKAGE_NAME}:${PROPOSED_WEIGHT})" + echo "[${PACKAGE_NAME}] ${WEIGHT_MESSAGE}" >&3 + echo "${WEIGHT_MESSAGE}" + echo " ${PACKAGE_NAME}: ${PROPOSED_WEIGHT} # was ${CONFIGURED_WEIGHT} (actual: $(( ELAPSED_SECONDS / 60 ))m $(printf '%02d' $(( ELAPSED_SECONDS % 60 )))s)" >> /tmp/cloudbuild_weight_adjustments.txt + fi + + # Emit failure details to stdout (captured by run_conditional_tests.sh) only when tests fail + # so passing runs do not print their logs a second time. + if [[ ${EXIT_CODE} -ne 0 ]]; then + sed -n '/=== FAILURES ===/,$p' "${TEMP_LOG_FILE}" + fi + rm -f "${TEMP_LOG_FILE}" + exit ${EXIT_CODE} + EOF + chmod +x /tmp/cloudbuild_shims/nox + + # Set placeholder credentials for noxfile.py if not already set. + if [[ -z "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then + export GOOGLE_APPLICATION_CREDENTIALS="CLOUD_BUILD_ADC" + fi + + # Prepend /tmp/cloudbuild_shims to PATH and PYTHONPATH so our nox wrapper and pytest + # plugin are picked up automatically by ci/run_single_test.sh and pytest. + export PATH="/tmp/cloudbuild_shims:${PATH}" + export PYTHONPATH="/tmp/cloudbuild_shims${PYTHONPATH:+:${PYTHONPATH}}" + export PYTEST_ADDOPTS="-p cloudbuild_pytest_plugin ${PYTEST_ADDOPTS:-}" + + # Open file descriptor 3 pointing to the container's live stdout so child workers + # spawned by xargs in ci/run_conditional_tests.sh can stream real-time logs. + exec 3>&1 + + # Export PACKAGE_LIST and delegate execution to the existing monorepo test runner script. + rm -f /tmp/cloudbuild_weight_adjustments.txt + print_weight_summary() { + echo "" + echo "============================================================" + if [[ -s /tmp/cloudbuild_weight_adjustments.txt ]]; then + echo "⚠️ WEIGHT ADJUSTMENTS NEEDED (drifted >= 3m from configured weight):" + sort -u /tmp/cloudbuild_weight_adjustments.txt + else + echo "✅ WEIGHT SUMMARY: No weight adjustments needed (all packages within 2m of configured weight)." + fi + echo "============================================================" + } + trap print_weight_summary EXIT + + export PACKAGE_LIST + echo "=== System Shard ${SHARD_INDEX}/${MAX_SHARDS} running ${NOX_SESSION} for: ${PACKAGE_LIST} ===" + ./ci/run_conditional_tests.sh + +# Hard 30-minute (1800s) CI timeout ceiling. +timeout: 1800s + +substitutions: + _MAX_SHARDS: '5' + _SHARD_INDEX: '0' + _MIN_SHARD_WEIGHT: '10' + _PARALLEL_WORKERS: '4' + _TIME_BUDGET_MINUTES: '20' + _HARD_TIMEOUT_MINUTES: '30' + _BUILD_TYPE: 'presubmit' + _BASE_BRANCH: 'main' + _TEST_ALL_PACKAGES: 'false' + _DISABLED_PACKAGES: 'packages/gapic-generator/ packages/google-auth/ packages/google-cloud-firestore/' + _NOX_SESSION: 'system-3.12' + _PACKAGE_WEIGHTS: | + bigquery-magics: 2 + google-cloud-bigquery: 8 + google-cloud-bigquery-storage: 8 + google-cloud-bigtable: 12 + google-cloud-compute: 8 + google-cloud-datastore: 8 + google-cloud-dns: 1 + google-cloud-error-reporting: 1 + google-cloud-logging: 10 + google-cloud-ndb: 12 + google-cloud-pubsub: 15 + google-cloud-spanner: 17 + google-cloud-storage: 7 + google-resumable-media: 1 + pandas-gbq: 12 + sqlalchemy-bigquery: 11 + +options: + machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY diff --git a/packages/google-cloud-bigquery-storage/samples/snippets/query_and_wait_arrow_test.py b/packages/google-cloud-bigquery-storage/samples/snippets/query_and_wait_arrow_test.py index ad7af73392b4..91ddd405d20e 100644 --- a/packages/google-cloud-bigquery-storage/samples/snippets/query_and_wait_arrow_test.py +++ b/packages/google-cloud-bigquery-storage/samples/snippets/query_and_wait_arrow_test.py @@ -12,12 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools + import pyarrow +from google.cloud import bigquery + from . import query_and_wait_arrow -def test_query_and_wait_arrow(): +def test_query_and_wait_arrow(monkeypatch): + # TODO: Remove use_query_cache=False workaround once google-cloud-bigquery > 3.47.0 + # (containing #18958) is released and pinned in requirements.txt. + monkeypatch.setattr( + query_and_wait_arrow.bigquery, + "Client", + functools.partial( + bigquery.Client, + default_query_job_config=bigquery.QueryJobConfig(use_query_cache=False), + ), + ) batches = query_and_wait_arrow.query_and_wait_arrow() total_rows = 0 diff --git a/packages/google-cloud-bigquery-storage/samples/snippets/requirements.txt b/packages/google-cloud-bigquery-storage/samples/snippets/requirements.txt index 5b613fdf002f..4f6a4e7b35ca 100644 --- a/packages/google-cloud-bigquery-storage/samples/snippets/requirements.txt +++ b/packages/google-cloud-bigquery-storage/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ google-cloud-bigquery-storage==2.43.0 -google-cloud-bigquery==3.46.2 +google-cloud-bigquery==3.47.0 pyarrow==26.0.0 pytest==9.1.1 diff --git a/packages/google-cloud-bigquery/samples/snippets/authenticate_service_account_test.py b/packages/google-cloud-bigquery/samples/snippets/authenticate_service_account_test.py index fbdd2d0649ff..9c3ef374d304 100644 --- a/packages/google-cloud-bigquery/samples/snippets/authenticate_service_account_test.py +++ b/packages/google-cloud-bigquery/samples/snippets/authenticate_service_account_test.py @@ -24,9 +24,11 @@ def mock_credentials(*args: Any, **kwargs: Any) -> google.auth.credentials.Credentials: - credentials, _ = google.auth.default( + credentials, project_id = google.auth.default( ["https://www.googleapis.com/auth/cloud-platform"] ) + if not hasattr(credentials, "project_id"): + credentials.project_id = project_id # type: ignore[attr-defined] return credentials diff --git a/packages/google-cloud-bigtable/noxfile.py b/packages/google-cloud-bigtable/noxfile.py index 045dcc166a38..dff330ad3691 100644 --- a/packages/google-cloud-bigtable/noxfile.py +++ b/packages/google-cloud-bigtable/noxfile.py @@ -54,6 +54,7 @@ SYSTEM_TEST_STANDARD_DEPENDENCIES: List[str] = [ "mock", "pytest", + "pytest-xdist", "google-cloud-testutils", ] SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [ @@ -345,6 +346,19 @@ def system(session, test_type): test_map[test_type](session) +def _has_xdist_arg(posargs): + return any( + arg == "-n" + or ( + arg.startswith("-n") + and (arg[2:].isdigit() or arg[2:] in ("auto", "logical")) + ) + or arg == "--numprocesses" + or arg.startswith("--numprocesses=") + for arg in posargs + ) + + def system_default(session): """Run the system test suite.""" constraints_path = str( @@ -378,9 +392,18 @@ def system_default(session): *session.posargs, ) if system_test_folder_exists: + in_emulator = bool( + session.env.get("BIGTABLE_EMULATOR_HOST") + or os.environ.get("BIGTABLE_EMULATOR_HOST") + ) + has_xdist_arg = _has_xdist_arg(session.posargs) + xdist_args = ( + [] if (in_emulator or has_xdist_arg) else ["-n", "8", "--dist=loadfile"] + ) session.run( "py.test", "--quiet", + *xdist_args, f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, *session.posargs, @@ -401,6 +424,7 @@ def samples(session): "mock", "pytest", "pytest-asyncio", + "pytest-xdist", *req_args, ) # Install the library from this checkout last so the samples exercise the @@ -410,9 +434,12 @@ def samples(session): has_target = any(not arg.startswith("-") for arg in session.posargs) target_paths = [] if has_target else [os.path.join("samples")] + has_xdist_arg = _has_xdist_arg(session.posargs) + xdist_args = [] if has_xdist_arg else ["-n", "8", "--dist=loadfile"] session.run( "py.test", "--quiet", + *xdist_args, f"--junitxml=samples_{session.python}_sponge_log.xml", "--ignore=samples/legacy_client/beam", *target_paths, diff --git a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py index 04a164a0f9ce..70c3c98e28d9 100644 --- a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py +++ b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py @@ -23,7 +23,9 @@ import argparse -def run_instance_operations(project_id, instance_id, cluster_id): +def run_instance_operations( + project_id, instance_id, cluster_id, location_id="us-central1-f" +): # [START bigtable_check_instance_exists] from google.api_core.exceptions import NotFound @@ -49,7 +51,7 @@ def run_instance_operations(project_id, instance_id, cluster_id): instance_path = client.instance_path(project_id, instance_id) cluster = bigtable_admin_v2.Cluster( - location=client.common_location_path(project_id, "us-central1-f"), + location=client.common_location_path(project_id, location_id), serve_nodes=1, default_storage_type=bigtable_admin_v2.StorageType.SSD, ) @@ -128,7 +130,7 @@ def delete_instance(project_id, instance_id): # [END bigtable_delete_instance] -def add_cluster(project_id, instance_id, cluster_id): +def add_cluster(project_id, instance_id, cluster_id, location_id="us-central1-b"): # [START bigtable_create_cluster] from google.api_core.exceptions import NotFound @@ -144,7 +146,7 @@ def add_cluster(project_id, instance_id, cluster_id): print(cluster.name.split("/")[-1]) new_cluster = bigtable_admin_v2.Cluster( - location=client.common_location_path(project_id, "us-central1-a"), + location=client.common_location_path(project_id, location_id), serve_nodes=1, default_storage_type=bigtable_admin_v2.StorageType.SSD, ) diff --git a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py index 1c5203b1a680..eb6158b66429 100644 --- a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py +++ b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py @@ -28,18 +28,59 @@ INSTANCE_ID = f"instance-admin-{str(uuid.uuid4())[:8]}" CLUSTER_ID = f"cluster-admin-{str(uuid.uuid4())[:8]}" NEW_CLUSTER_ID = f"cluster-add-{str(uuid.uuid4())[:8]}" +FALLBACK_ZONES = ( + "us-central1-f", + "us-central1-b", + "us-central1-c", + "us-east1-b", + "us-west1-b", +) + + +def _is_zone_stockout(exc): + message = str(exc).lower() + return "sold out" in message or "not enough servers" in message + + +def _run_with_zone_fallback(fn, zones, capsys, *args): + last_exc = None + for zone in zones: + capsys.readouterr() + try: + fn(*args, location_id=zone) + return zone + except exceptions.GoogleAPICallError as exc: + if not _is_zone_stockout(exc): + raise + last_exc = exc + raise last_exc def test_instance_operations(capsys): try: - run_instance_operations(PROJECT, INSTANCE_ID, CLUSTER_ID) + primary_zone = _run_with_zone_fallback( + run_instance_operations, + FALLBACK_ZONES, + capsys, + PROJECT, + INSTANCE_ID, + CLUSTER_ID, + ) out, _ = capsys.readouterr() assert f"Created instance: {INSTANCE_ID}" in out assert "Listing instances:" in out assert "Listing clusters..." in out backoff_503 = backoff.on_exception(backoff.expo, exceptions.ServiceUnavailable) - backoff_503(add_cluster)(PROJECT, INSTANCE_ID, NEW_CLUSTER_ID) + secondary_zones = tuple(z for z in FALLBACK_ZONES if z != primary_zone) + _run_with_zone_fallback( + backoff_503(add_cluster), + secondary_zones, + capsys, + PROJECT, + INSTANCE_ID, + NEW_CLUSTER_ID, + ) out, _ = capsys.readouterr() assert f"Cluster created: {NEW_CLUSTER_ID}" in out diff --git a/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/instanceadmin_async.py b/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/instanceadmin_async.py index daf96c8c5bd9..0712ab3f7a1d 100644 --- a/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/instanceadmin_async.py +++ b/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/instanceadmin_async.py @@ -24,7 +24,9 @@ import asyncio -async def run_instance_operations(project_id, instance_id, cluster_id): +async def run_instance_operations( + project_id, instance_id, cluster_id, location_id="us-central1-f" +): # [START bigtable_async_check_instance_exists] from google.api_core.exceptions import NotFound @@ -50,7 +52,7 @@ async def run_instance_operations(project_id, instance_id, cluster_id): instance_path = client.instance_path(project_id, instance_id) cluster = bigtable_admin_v2.Cluster( - location=client.common_location_path(project_id, "us-central1-f"), + location=client.common_location_path(project_id, location_id), serve_nodes=1, default_storage_type=bigtable_admin_v2.StorageType.SSD, ) @@ -129,7 +131,7 @@ async def delete_instance(project_id, instance_id): # [END bigtable_async_delete_instance] -async def add_cluster(project_id, instance_id, cluster_id): +async def add_cluster(project_id, instance_id, cluster_id, location_id="us-central1-b"): # [START bigtable_async_create_cluster] from google.api_core.exceptions import NotFound @@ -145,7 +147,7 @@ async def add_cluster(project_id, instance_id, cluster_id): print(cluster.name.split("/")[-1]) new_cluster = bigtable_admin_v2.Cluster( - location=client.common_location_path(project_id, "us-central1-a"), + location=client.common_location_path(project_id, location_id), serve_nodes=1, default_storage_type=bigtable_admin_v2.StorageType.SSD, ) diff --git a/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/test_instanceadmin_async.py b/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/test_instanceadmin_async.py index 3ec71138d30d..d4965cf08390 100644 --- a/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/test_instanceadmin_async.py +++ b/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/test_instanceadmin_async.py @@ -29,19 +29,60 @@ INSTANCE_ID = f"instance-admin-async-{str(uuid.uuid4())[:8]}" CLUSTER_ID = f"cluster-admin-async-{str(uuid.uuid4())[:8]}" NEW_CLUSTER_ID = f"cluster-add-async-{str(uuid.uuid4())[:8]}" +FALLBACK_ZONES = ( + "us-central1-f", + "us-central1-b", + "us-central1-c", + "us-east1-b", + "us-west1-b", +) + + +def _is_zone_stockout(exc): + message = str(exc).lower() + return "sold out" in message or "not enough servers" in message + + +async def _run_with_zone_fallback(fn, zones, capsys, *args): + last_exc = None + for zone in zones: + capsys.readouterr() + try: + await fn(*args, location_id=zone) + return zone + except exceptions.GoogleAPICallError as exc: + if not _is_zone_stockout(exc): + raise + last_exc = exc + raise last_exc @pytest.mark.asyncio async def test_instance_operations(capsys): try: - await run_instance_operations(PROJECT, INSTANCE_ID, CLUSTER_ID) + primary_zone = await _run_with_zone_fallback( + run_instance_operations, + FALLBACK_ZONES, + capsys, + PROJECT, + INSTANCE_ID, + CLUSTER_ID, + ) out, _ = capsys.readouterr() assert f"Created instance: {INSTANCE_ID}" in out assert "Listing instances:" in out assert "Listing clusters..." in out backoff_503 = backoff.on_exception(backoff.expo, exceptions.ServiceUnavailable) - await backoff_503(add_cluster)(PROJECT, INSTANCE_ID, NEW_CLUSTER_ID) + secondary_zones = tuple(z for z in FALLBACK_ZONES if z != primary_zone) + await _run_with_zone_fallback( + backoff_503(add_cluster), + secondary_zones, + capsys, + PROJECT, + INSTANCE_ID, + NEW_CLUSTER_ID, + ) out, _ = capsys.readouterr() assert f"Cluster created: {NEW_CLUSTER_ID}" in out diff --git a/packages/google-cloud-bigtable/samples/conftest.py b/packages/google-cloud-bigtable/samples/conftest.py index 4c23415ab7b4..d43dff2e3b71 100644 --- a/packages/google-cloud-bigtable/samples/conftest.py +++ b/packages/google-cloud-bigtable/samples/conftest.py @@ -76,6 +76,11 @@ def pytest_configure(config) -> None: ): return + # Under pytest-xdist, only the controller provisions and tears down the + # shared sample test instance; workers inherit BIGTABLE_INSTANCE via os.environ. + if hasattr(config, "workerinput"): + return + project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") if not project_id: raise pytest.UsageError( @@ -106,23 +111,45 @@ def pytest_configure(config) -> None: signal.signal(signal.SIGTERM, _sigterm_handler) print(f"\nCreating temporary sample test instance: {instance_name}") + fallback_zones = ( + "us-central1-b", + "us-central1-c", + "us-central1-f", + "us-east1-b", + "us-west1-b", + ) try: - operation = client.create_instance( - parent=project_path, - instance_id=instance_id, - instance=bigtable_admin.Instance( - display_name=instance_id, - type_=bigtable_admin.Instance.Type.PRODUCTION, - ), - clusters={ - f"{instance_id}-c1": bigtable_admin.Cluster( - location=client.common_location_path(project_id, "us-central1-b"), - serve_nodes=1, - default_storage_type=bigtable_admin.StorageType.SSD, + last_exc = None + for zone in fallback_zones: + try: + operation = client.create_instance( + parent=project_path, + instance_id=instance_id, + instance=bigtable_admin.Instance( + display_name=instance_id, + type_=bigtable_admin.Instance.Type.PRODUCTION, + ), + clusters={ + f"{instance_id}-c1": bigtable_admin.Cluster( + location=client.common_location_path(project_id, zone), + serve_nodes=1, + default_storage_type=bigtable_admin.StorageType.SSD, + ) + }, ) - }, - ) - operation.result(timeout=240) + operation.result(timeout=240) + last_exc = None + break + except exceptions.GoogleAPICallError as exc: + message = str(exc).lower() + if "sold out" not in message and "not enough servers" not in message: + raise + print( + f"Zone {zone} temporarily sold out for {instance_id}, trying next zone..." + ) + last_exc = exc + if last_exc is not None: + raise last_exc os.environ["BIGTABLE_INSTANCE"] = instance_id print(f"Created temporary sample test instance: {instance_id}") except Exception: diff --git a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py index 53f1bfbb1906..e64ff46200ff 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py @@ -36,7 +36,9 @@ from google.cloud.bigtable import enums -def run_instance_operations(project_id, instance_id, cluster_id): +def run_instance_operations( + project_id, instance_id, cluster_id, location_id="us-central1-f" +): """Check Instance exists. Creates a Production instance with default Cluster. List instances in a project. @@ -49,7 +51,6 @@ def run_instance_operations(project_id, instance_id, cluster_id): :param instance_id: Instance of the client. """ client = bigtable.Client(project=project_id, admin=True) - location_id = "us-central1-f" serve_nodes = 1 storage_type = enums.StorageType.SSD labels = {"prod-label": "prod-label"} @@ -121,7 +122,7 @@ def delete_instance(project_id, instance_id): # [END bigtable_delete_instance_legacy] -def add_cluster(project_id, instance_id, cluster_id): +def add_cluster(project_id, instance_id, cluster_id, location_id="us-central1-b"): """Add Cluster :type project_id: str @@ -136,7 +137,6 @@ def add_cluster(project_id, instance_id, cluster_id): client = bigtable.Client(project=project_id, admin=True) instance = client.instance(instance_id) - location_id = "us-central1-a" serve_nodes = 1 storage_type = enums.StorageType.SSD diff --git a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/test_instanceadmin.py b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/test_instanceadmin.py index 669280967733..b46c862f6cd7 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/test_instanceadmin.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/test_instanceadmin.py @@ -33,6 +33,32 @@ INSTANCE = INSTANCE_ID_FORMAT.format(random.randrange(ID_RANGE), int(time.time())) CLUSTER1 = CLUSTER_ID_FORMAT.format(random.randrange(ID_RANGE)) CLUSTER2 = CLUSTER_ID_FORMAT.format(random.randrange(ID_RANGE)) +FALLBACK_ZONES = ( + "us-central1-f", + "us-central1-b", + "us-central1-c", + "us-east1-b", + "us-west1-b", +) + + +def _is_zone_stockout(exc): + message = str(exc).lower() + return "sold out" in message or "not enough servers" in message + + +def _run_with_zone_fallback(fn, zones, capsys, *args): + last_exc = None + for zone in zones: + capsys.readouterr() + try: + fn(*args, location_id=zone) + return zone + except exceptions.GoogleAPICallError as exc: + if not _is_zone_stockout(exc): + raise + last_exc = exc + raise last_exc @pytest.fixture(scope="module", autouse=True) @@ -73,7 +99,14 @@ def disposal(instance): def test_run_instance_operations(capsys, dispose_of): dispose_of(INSTANCE) - instanceadmin.run_instance_operations(PROJECT, INSTANCE, CLUSTER1) + primary_zone = _run_with_zone_fallback( + instanceadmin.run_instance_operations, + FALLBACK_ZONES, + capsys, + PROJECT, + INSTANCE, + CLUSTER1, + ) out = capsys.readouterr().out assert f"Instance {INSTANCE} does not exist." in out assert "Creating an instance" in out @@ -85,7 +118,9 @@ def test_run_instance_operations(capsys, dispose_of): assert "Listing clusters..." in out assert f"\n{CLUSTER1}\n" in out - instanceadmin.run_instance_operations(PROJECT, INSTANCE, CLUSTER1) + instanceadmin.run_instance_operations( + PROJECT, INSTANCE, CLUSTER1, location_id=primary_zone + ) out = capsys.readouterr().out assert f"Instance {INSTANCE} already exists." in out assert "Listing instances" in out @@ -110,7 +145,14 @@ def _set_up_instance(): assert f"Instance {INSTANCE} does not exist" in out # Ok, create it then - instanceadmin.run_instance_operations(PROJECT, INSTANCE, CLUSTER1) + _run_with_zone_fallback( + instanceadmin.run_instance_operations, + FALLBACK_ZONES, + capsys, + PROJECT, + INSTANCE, + CLUSTER1, + ) capsys.readouterr() # throw away output _set_up_instance() @@ -135,17 +177,33 @@ def _set_up_instance(): assert f"Instance {INSTANCE} does not exist" in out # Get the instance created - instanceadmin.run_instance_operations(PROJECT, INSTANCE, CLUSTER1) + primary_zone = _run_with_zone_fallback( + instanceadmin.run_instance_operations, + FALLBACK_ZONES, + capsys, + PROJECT, + INSTANCE, + CLUSTER1, + ) capsys.readouterr() # throw away output + return primary_zone - _set_up_instance() + primary_zone = _set_up_instance() # Add a cluster to that instance # Avoid failing for "instance is currently being changed" by # applying an exponential backoff backoff_503 = backoff.on_exception(backoff.expo, exceptions.ServiceUnavailable) - backoff_503(instanceadmin.add_cluster)(PROJECT, INSTANCE, CLUSTER2) + secondary_zones = tuple(z for z in FALLBACK_ZONES if z != primary_zone) + secondary_zone = _run_with_zone_fallback( + backoff_503(instanceadmin.add_cluster), + secondary_zones, + capsys, + PROJECT, + INSTANCE, + CLUSTER2, + ) out = capsys.readouterr().out assert f"Adding cluster to instance {INSTANCE}" in out assert "Listing clusters..." in out @@ -153,7 +211,7 @@ def _set_up_instance(): assert f"Cluster created: {CLUSTER2}" in out # Try to add the same cluster again, won't work - instanceadmin.add_cluster(PROJECT, INSTANCE, CLUSTER2) + instanceadmin.add_cluster(PROJECT, INSTANCE, CLUSTER2, location_id=secondary_zone) out = capsys.readouterr().out assert "Listing clusters..." in out assert f"\n{CLUSTER1}\n" in out diff --git a/packages/google-cloud-bigtable/samples/utils.py b/packages/google-cloud-bigtable/samples/utils.py index a28eb58d80ed..23b0c3f2feec 100644 --- a/packages/google-cloud-bigtable/samples/utils.py +++ b/packages/google-cloud-bigtable/samples/utils.py @@ -14,11 +14,14 @@ Provides helper logic used across samples """ +import inspect + from google.api_core import exceptions from google.api_core.retry import Retry, if_exception_type from google.cloud import bigtable, bigtable_admin_v2 from google.cloud.bigtable.column_family import ColumnFamily +from google.cloud.bigtable.data import BigtableDataClient, ReadRowsQuery from google.cloud.bigtable_admin_v2.types import ColumnFamily as ColumnFamily_pb delete_retry = Retry( @@ -93,7 +96,7 @@ def create_table(project, instance_id, table_id, column_families={}): @Retry( - on_error=if_exception_type( + predicate=if_exception_type( exceptions.PreconditionFailed, exceptions.FailedPrecondition, exceptions.NotFound, @@ -109,13 +112,28 @@ def wait_for_table(table): """ if hasattr(table, "exists"): if not table.exists(): - raise exceptions.NotFound - else: - table_path = table.table_name if hasattr(table, "table_name") else str(table) - admin_client = bigtable_admin_v2.BigtableTableAdminClient() - admin_client.get_table( - request={ - "name": table_path, - "view": bigtable_admin_v2.Table.View.NAME_ONLY, - } - ) + raise exceptions.NotFound("Table does not exist yet") + return + + # Data-client Table / TableAsync instances and raw table_path strings (e.g. + # from create_table) do not expose .exists(), so verify via admin + data plane. + table_path = table.table_name if hasattr(table, "table_name") else str(table) + admin_client = bigtable_admin_v2.BigtableTableAdminClient() + admin_client.get_table( + request={ + "name": table_path, + "view": bigtable_admin_v2.Table.View.NAME_ONLY, + } + ) + + # Probe the data plane as well: get_table() can succeed while mutations + # still return FailedPrecondition ("table ... is still being created"). + if hasattr(table, "read_rows") and not inspect.iscoroutinefunction(table.read_rows): + table.read_rows(ReadRowsQuery(limit=1)) + return + + parts = table_path.split("/") + if len(parts) == 6 and parts[0] == "projects": + with BigtableDataClient(project=parts[1]) as data_client: + with data_client.get_table(parts[3], parts[5]) as data_table: + data_table.read_rows(ReadRowsQuery(limit=1)) diff --git a/packages/google-cloud-bigtable/tests/system/conftest.py b/packages/google-cloud-bigtable/tests/system/conftest.py index 7a7b315ec542..543f603404a0 100644 --- a/packages/google-cloud-bigtable/tests/system/conftest.py +++ b/packages/google-cloud-bigtable/tests/system/conftest.py @@ -20,3 +20,31 @@ script_path = os.path.dirname(os.path.realpath(__file__)) sys.path.append(script_path) + + +def pytest_configure(config) -> None: + """Clean up leaked test instances in the controller before workers start.""" + from google.cloud.environment_vars import BIGTABLE_EMULATOR + + if ( + os.getenv(BIGTABLE_EMULATOR) + or getattr(config.option, "help", False) + or getattr(config.option, "collectonly", False) + or hasattr(config, "workerinput") + ): + return + + project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") + if not project_id: + try: + import google.auth + + _, project_id = google.auth.default() + except Exception: + return + + if project_id: + from utils import clear_stale_instances + + # Purge leaked test instances older than 1 hour before allocating nodes. + clear_stale_instances(project_id, older_than_days=1 / 24, max_deletions=100) diff --git a/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py b/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py index 34f6ccf3c2a9..f4e3be1d5820 100644 --- a/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py +++ b/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py @@ -91,16 +91,52 @@ def clear(self): self._exc_list.clear() self.fail_mid_stream = False + def _make_deadline_exceeded(self): + if CrossSync.is_async: + return AioRpcError(StatusCode.DEADLINE_EXCEEDED, Metadata(), Metadata()) + else: + exc = RpcError(StatusCode.DEADLINE_EXCEEDED) + exc.trailing_metadata = lambda: [] + exc.initial_metadata = lambda: [] + exc.code = lambda: StatusCode.DEADLINE_EXCEEDED + exc.details = lambda: "Deadline Exceeded" + + def _result(): + raise exc + + exc.result = _result + return exc + @CrossSync.convert async def intercept_unary_unary(self, continuation, client_call_details, request): if self._exc_list: raise self._exc_list.pop(0) + # Deterministic client-side timeout for *_failure_timeout tests: + # When the CI worker and Bigtable cluster sit in the same GCP region + # (e.g. Cloud Build in us-central1 with <1ms RTT), tiny 1–5ms timeouts + # race against Bigtable returning server headers (setting cluster_id) on + # one hand, and expiring inside Python's local shard-dispatch loop before + # all shards start on the other. Using a 50ms timeout gives Python ample + # headroom to start all sub-requests under xdist CPU load, while sleeping + # past the deadline here guarantees DEADLINE_EXCEEDED with no server headers. + if ( + client_call_details.timeout is not None + and client_call_details.timeout <= 0.1 + ): + await CrossSync.sleep(client_call_details.timeout + 0.01) + raise self._make_deadline_exceeded() return await continuation(client_call_details, request) @CrossSync.convert async def intercept_unary_stream(self, continuation, client_call_details, request): if not self.fail_mid_stream and self._exc_list: raise self._exc_list.pop(0) + if ( + client_call_details.timeout is not None + and client_call_details.timeout <= 0.1 + ): + await CrossSync.sleep(client_call_details.timeout + 0.01) + raise self._make_deadline_exceeded() response = await continuation(client_call_details, request) @@ -124,6 +160,10 @@ async def __anext__(self): if not self._raised: self._raised = True if self._exc: + if callable(getattr(self._call, "cancel", None)): + self._call.cancel() + elif callable(getattr(self._call, "close", None)): + self._call.close() raise self._exc return await self._call.__anext__() @@ -329,7 +369,7 @@ async def test_read_rows_failure_timeout(self, table, temp_rows, handler): await temp_rows.add_row(b"row_key_1") handler.clear() with pytest.raises(GoogleAPICallError): - await table.read_rows(ReadRowsQuery(), operation_timeout=0.001) + await table.read_rows(ReadRowsQuery(), operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -521,7 +561,7 @@ async def test_read_rows_stream_failure_timeout(self, table, temp_rows, handler) await temp_rows.add_row(b"row_key_1") handler.clear() generator = await table.read_rows_stream( - ReadRowsQuery(), operation_timeout=0.001 + ReadRowsQuery(), operation_timeout=0.05 ) with pytest.raises(GoogleAPICallError): [_ async for _ in generator] @@ -748,7 +788,7 @@ async def test_read_row_failure_timeout(self, table, temp_rows, handler): await temp_rows.add_row(b"row_key_1") handler.clear() with pytest.raises(GoogleAPICallError): - await table.read_row(b"row_key_1", operation_timeout=0.001) + await table.read_row(b"row_key_1", operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -917,7 +957,7 @@ async def test_read_rows_sharded_failure_timeout(self, table, temp_rows, handler query2 = ReadRowsQuery(row_keys=[b"b"]) handler.clear() with pytest.raises(ShardedReadRowsExceptionGroup) as e: - await table.read_rows_sharded([query1, query2], operation_timeout=0.005) + await table.read_rows_sharded([query1, query2], operation_timeout=0.05) assert len(e.value.exceptions) == 2 for sub_exc in e.value.exceptions: assert isinstance(sub_exc.__cause__, DeadlineExceeded) @@ -1159,7 +1199,7 @@ async def test_bulk_mutate_rows_failure_timeout(self, table, temp_rows, handler) handler.clear() with pytest.raises(MutationsExceptionGroup): - await table.bulk_mutate_rows([entry], operation_timeout=0.001) + await table.bulk_mutate_rows([entry], operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -1381,9 +1421,7 @@ async def test_mutate_rows_batcher_failure_timeout(self, table, temp_rows, handl entry = RowMutationEntry(row_key, [mutation]) with pytest.raises(MutationsExceptionGroup): - async with table.mutations_batcher( - batch_operation_timeout=0.001 - ) as batcher: + async with table.mutations_batcher(batch_operation_timeout=0.05) as batcher: await batcher.append(entry) # validate counts assert len(handler.completed_operations) == 1 @@ -1559,7 +1597,7 @@ async def test_mutate_row_failure_timeout(self, table, temp_rows, handler): mutation = SetCell(TEST_FAMILY, b"q", b"v") with pytest.raises(GoogleAPICallError): - await table.mutate_row(row_key, [mutation], operation_timeout=0.001) + await table.mutate_row(row_key, [mutation], operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -1801,7 +1839,7 @@ async def test_sample_row_keys_failure_timeout(self, table, handler): No grpc headers expected """ with pytest.raises(GoogleAPICallError): - await table.sample_row_keys(operation_timeout=0.001) + await table.sample_row_keys(operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -1967,7 +2005,7 @@ async def test_read_modify_write_failure_timeout(self, table, temp_rows, handler await temp_rows.add_row(row_key, value=0, family=family, qualifier=qualifier) rule = IncrementRule(family, qualifier, 1) with pytest.raises(GoogleAPICallError): - await table.read_modify_write_row(row_key, rule, operation_timeout=0.001) + await table.read_modify_write_row(row_key, rule, operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -2158,7 +2196,7 @@ async def test_check_and_mutate_row_failure_timeout( row_key, predicate=ValueRangeFilter(0, 2), true_case_mutations=true_mutation, - operation_timeout=0.001, + operation_timeout=0.05, ) # validate counts assert len(handler.completed_operations) == 1 diff --git a/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py b/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py index 7daf8ba2d03c..d1d2cfe531fe 100644 --- a/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py +++ b/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py @@ -81,14 +81,39 @@ def clear(self): self._exc_list.clear() self.fail_mid_stream = False + def _make_deadline_exceeded(self): + exc = RpcError(StatusCode.DEADLINE_EXCEEDED) + exc.trailing_metadata = lambda: [] + exc.initial_metadata = lambda: [] + exc.code = lambda: StatusCode.DEADLINE_EXCEEDED + exc.details = lambda: "Deadline Exceeded" + + def _result(): + raise exc + + exc.result = _result + return exc + def intercept_unary_unary(self, continuation, client_call_details, request): if self._exc_list: raise self._exc_list.pop(0) + if ( + client_call_details.timeout is not None + and client_call_details.timeout <= 0.1 + ): + CrossSync._Sync_Impl.sleep(client_call_details.timeout + 0.01) + raise self._make_deadline_exceeded() return continuation(client_call_details, request) def intercept_unary_stream(self, continuation, client_call_details, request): if not self.fail_mid_stream and self._exc_list: raise self._exc_list.pop(0) + if ( + client_call_details.timeout is not None + and client_call_details.timeout <= 0.1 + ): + CrossSync._Sync_Impl.sleep(client_call_details.timeout + 0.01) + raise self._make_deadline_exceeded() response = continuation(client_call_details, request) if self.fail_mid_stream and self._exc_list: exc = self._exc_list.pop(0) @@ -106,6 +131,10 @@ def __next__(self): if not self._raised: self._raised = True if self._exc: + if callable(getattr(self._call, "cancel", None)): + self._call.cancel() + elif callable(getattr(self._call, "close", None)): + self._call.close() raise self._exc return self._call.__next__() @@ -276,7 +305,7 @@ def test_read_rows_failure_timeout(self, table, temp_rows, handler): temp_rows.add_row(b"row_key_1") handler.clear() with pytest.raises(GoogleAPICallError): - table.read_rows(ReadRowsQuery(), operation_timeout=0.001) + table.read_rows(ReadRowsQuery(), operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -435,7 +464,7 @@ def test_read_rows_stream_failure_timeout(self, table, temp_rows, handler): No grpc headers expected""" temp_rows.add_row(b"row_key_1") handler.clear() - generator = table.read_rows_stream(ReadRowsQuery(), operation_timeout=0.001) + generator = table.read_rows_stream(ReadRowsQuery(), operation_timeout=0.05) with pytest.raises(GoogleAPICallError): [_ for _ in generator] assert len(handler.completed_operations) == 1 @@ -624,7 +653,7 @@ def test_read_row_failure_timeout(self, table, temp_rows, handler): temp_rows.add_row(b"row_key_1") handler.clear() with pytest.raises(GoogleAPICallError): - table.read_row(b"row_key_1", operation_timeout=0.001) + table.read_row(b"row_key_1", operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -770,7 +799,7 @@ def test_read_rows_sharded_failure_timeout(self, table, temp_rows, handler): query2 = ReadRowsQuery(row_keys=[b"b"]) handler.clear() with pytest.raises(ShardedReadRowsExceptionGroup) as e: - table.read_rows_sharded([query1, query2], operation_timeout=0.005) + table.read_rows_sharded([query1, query2], operation_timeout=0.05) assert len(e.value.exceptions) == 2 for sub_exc in e.value.exceptions: assert isinstance(sub_exc.__cause__, DeadlineExceeded) @@ -971,7 +1000,7 @@ def test_bulk_mutate_rows_failure_timeout(self, table, temp_rows, handler): entry = RowMutationEntry(row_key, [mutation]) handler.clear() with pytest.raises(MutationsExceptionGroup): - table.bulk_mutate_rows([entry], operation_timeout=0.001) + table.bulk_mutate_rows([entry], operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -1155,7 +1184,7 @@ def test_mutate_rows_batcher_failure_timeout(self, table, temp_rows, handler): mutation = SetCell(TEST_FAMILY, b"q", b"v") entry = RowMutationEntry(row_key, [mutation]) with pytest.raises(MutationsExceptionGroup): - with table.mutations_batcher(batch_operation_timeout=0.001) as batcher: + with table.mutations_batcher(batch_operation_timeout=0.05) as batcher: batcher.append(entry) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -1302,7 +1331,7 @@ def test_mutate_row_failure_timeout(self, table, temp_rows, handler): row_key = b"row_key_1" mutation = SetCell(TEST_FAMILY, b"q", b"v") with pytest.raises(GoogleAPICallError): - table.mutate_row(row_key, [mutation], operation_timeout=0.001) + table.mutate_row(row_key, [mutation], operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -1471,7 +1500,7 @@ def test_sample_row_keys_failure_timeout(self, table, handler): No grpc headers expected""" with pytest.raises(GoogleAPICallError): - table.sample_row_keys(operation_timeout=0.001) + table.sample_row_keys(operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -1567,7 +1596,7 @@ def test_read_modify_write_failure_timeout(self, table, temp_rows, handler): temp_rows.add_row(row_key, value=0, family=family, qualifier=qualifier) rule = IncrementRule(family, qualifier, 1) with pytest.raises(GoogleAPICallError): - table.read_modify_write_row(row_key, rule, operation_timeout=0.001) + table.read_modify_write_row(row_key, rule, operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -1680,7 +1709,7 @@ def test_check_and_mutate_row_failure_timeout(self, table, temp_rows, handler): row_key, predicate=ValueRangeFilter(0, 2), true_case_mutations=true_mutation, - operation_timeout=0.001, + operation_timeout=0.05, ) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 diff --git a/packages/google-cloud-bigtable/tests/system/utils.py b/packages/google-cloud-bigtable/tests/system/utils.py index fe3f3a2b865f..2824d4b247df 100644 --- a/packages/google-cloud-bigtable/tests/system/utils.py +++ b/packages/google-cloud-bigtable/tests/system/utils.py @@ -27,15 +27,23 @@ def clear_stale_instances( prefix: Union[str, Tuple[str, ...]] = ( "python-bigtable-tests", "g-c-p", + "ndef-", + "new-", + "dif-", + "dif-cmek-", + "ndif-", "admin-overlay-instance", "bt-samples-", + "metric-scale-test-", + "instanceadmin-", + "instance-admin-", ), - older_than_days: int = 1, - max_deletions: int = 5, + older_than_days: float = 1, + max_deletions: int = 50, ): """ Synchronously deletes any instances in the given project that are older - than older_than_days and whose name or display name matches the given prefix. + than older_than_days and whose name, display name, or test label matches. """ if os.getenv(BIGTABLE_EMULATOR): return @@ -57,11 +65,12 @@ def clear_stale_instances( break for instance in response.instances: - # Check if instance matches the prefix + # Check if instance matches the prefix or python-system test label display_name_matches = instance.display_name.startswith(prefix) name_matches = instance.name.split("/")[-1].startswith(prefix) + label_matches = "python-system" in getattr(instance, "labels", {}) - if display_name_matches or name_matches: + if display_name_matches or name_matches or label_matches: if instance.create_time: try: create_time = instance.create_time diff --git a/packages/google-cloud-bigtable/tests/system/v2_client/conftest.py b/packages/google-cloud-bigtable/tests/system/v2_client/conftest.py index 59e016f52af8..82ae486b2ad8 100644 --- a/packages/google-cloud-bigtable/tests/system/v2_client/conftest.py +++ b/packages/google-cloud-bigtable/tests/system/v2_client/conftest.py @@ -58,7 +58,7 @@ def location_id(): @pytest.fixture(scope="session") def serve_nodes(): - return 3 + return 1 @pytest.fixture(scope="session") diff --git a/packages/google-cloud-bigtable/tests/system/v2_client/test_data_api.py b/packages/google-cloud-bigtable/tests/system/v2_client/test_data_api.py index 3a45224d390c..59bfe1915d25 100644 --- a/packages/google-cloud-bigtable/tests/system/v2_client/test_data_api.py +++ b/packages/google-cloud-bigtable/tests/system/v2_client/test_data_api.py @@ -356,7 +356,9 @@ def test_table_mutate_rows_retries_timeout(data_table, rows_to_delete): rows_to_delete.append(row_2) row_2.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL1, timestamp=timestamp) - statuses = data_table.mutate_rows([row, row_2]) + statuses = data_table.mutate_rows( + [row, row_2], retry=DEFAULT_RETRY.with_deadline(2.0) + ) assert statuses[0].code == Code.OK assert statuses[1].code == Code.DEADLINE_EXCEEDED @@ -940,10 +942,10 @@ def test_table_read_rows_retry_timeout_mid_stream( # Shorten the deadline so the timeout test is shorter. data_table_read_rows_retry_tests._table_impl.default_read_rows_operation_timeout = ( - 10.0 + 2.0 ) rows_data = data_table_read_rows_retry_tests.read_rows( - retry=DEFAULT_RETRY_READ_ROWS.with_deadline(10.0) + retry=DEFAULT_RETRY_READ_ROWS.with_deadline(2.0) ) with pytest.raises(exceptions.RetryError): rows_data.consume_all() @@ -971,10 +973,10 @@ def test_table_read_rows_retry_timeout_establishing_stream( # Shorten the deadline so the timeout test is shorter. data_table_read_rows_retry_tests._table_impl.default_read_rows_operation_timeout = ( - 10.0 + 2.0 ) rows_data = data_table_read_rows_retry_tests.read_rows( - retry=DEFAULT_RETRY_READ_ROWS.with_deadline(10.0) + retry=DEFAULT_RETRY_READ_ROWS.with_deadline(2.0) ) with pytest.raises(exceptions.RetryError): rows_data.consume_all() diff --git a/packages/google-cloud-logging/tests/system/gapic/v2/test_system_logging_service_v2_v2.py b/packages/google-cloud-logging/tests/system/gapic/v2/test_system_logging_service_v2_v2.py index d08703d48548..6fe019ae02ef 100644 --- a/packages/google-cloud-logging/tests/system/gapic/v2/test_system_logging_service_v2_v2.py +++ b/packages/google-cloud-logging/tests/system/gapic/v2/test_system_logging_service_v2_v2.py @@ -31,10 +31,13 @@ def test_write_log_entries(self): # Guard: Skip if project or credentials are missing # This prevents the "prerelease_deps" job from failing - if not project_id or "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ: - pytest.skip( - "System tests require GOOGLE_CLOUD_PROJECT and GOOGLE_APPLICATION_CREDENTIALS" - ) + if not project_id: + pytest.skip("System tests require GOOGLE_CLOUD_PROJECT or PROJECT_ID") + + try: + google.auth.default() + except google.auth.exceptions.DefaultCredentialsError: + pytest.skip("System tests require Application Default Credentials") # Instantiate the client # Since we've confirmed the env var exists, this call is now safe diff --git a/packages/google-cloud-logging/tests/system/test_system.py b/packages/google-cloud-logging/tests/system/test_system.py index 52280aa35ee9..2b29bcbb255c 100644 --- a/packages/google-cloud-logging/tests/system/test_system.py +++ b/packages/google-cloud-logging/tests/system/test_system.py @@ -105,18 +105,23 @@ class Config(object): def setUpModule(): + import google.auth + import google.auth.exceptions + # Use GOOGLE_CLOUD_PROJECT or PROJECT_ID project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") or os.environ.get("PROJECT_ID") - # Check if we have credentials (either via file or environment) - # google.auth.default() will check GOOGLE_APPLICATION_CREDENTIALS - has_creds = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") is not None + if not project_id: + raise unittest.SkipTest( + "System tests skipped: GOOGLE_CLOUD_PROJECT or PROJECT_ID must be set." + ) - # Guard: Skip the entire module if the environment is not ready - if not project_id or not has_creds: + # Check if we have credentials (either via GOOGLE_APPLICATION_CREDENTIALS or ADC) + try: + google.auth.default() + except google.auth.exceptions.DefaultCredentialsError: raise unittest.SkipTest( - "System tests skipped: GOOGLE_CLOUD_PROJECT and " - "GOOGLE_APPLICATION_CREDENTIALS must be set." + "System tests skipped: Application Default Credentials must be available." ) # Explicitly pass the project to the client diff --git a/packages/google-cloud-storage/tests/conformance/_utils.py b/packages/google-cloud-storage/tests/conformance/_utils.py index 1857a46f34ef..96479f5776bf 100644 --- a/packages/google-cloud-storage/tests/conformance/_utils.py +++ b/packages/google-cloud-storage/tests/conformance/_utils.py @@ -1,8 +1,143 @@ +import os +import socket +import subprocess import time import traceback +from typing import Optional, Tuple import requests +DEFAULT_TESTBENCH_IMAGE = ( + "gcr.io/cloud-devrel-public-resources/storage-testbench:latest" +) + + +def get_container_docker_network() -> Optional[str]: + """Return a shared user-defined Docker network (e.g. 'cloudbuild') when + running inside a container attached to one, or None on a regular host. + """ + if not os.path.exists("/.dockerenv"): + return None + hostname = socket.gethostname() + try: + result = subprocess.run( + [ + "docker", + "inspect", + "-f", + "{{range $k, $v := .NetworkSettings.Networks}}{{$k}} {{end}}", + hostname, + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + if result.returncode == 0: + networks = [ + net + for net in result.stdout.strip().split() + if net and net not in ("bridge", "host", "none") + ] + if "cloudbuild" in networks: + return "cloudbuild" + if networks: + return networks[0] + # Fallback if container hostname was overridden from container ID. + cb_check = subprocess.run( + ["docker", "network", "inspect", "cloudbuild"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + check=False, + ) + if cb_check.returncode == 0: + return "cloudbuild" + except (OSError, subprocess.SubprocessError): + return None + return None + + +def stop_testbench(container_name: str) -> None: + """Force-remove a running storage-testbench container, ignoring errors.""" + subprocess.run( + ["docker", "rm", "-f", container_name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + + +def start_testbench( + container_name: str, + host_http_port: int, + host_grpc_port: Optional[int] = None, + image: str = DEFAULT_TESTBENCH_IMAGE, + ready_timeout: float = 30.0, +) -> Tuple[str, Optional[str]]: + """Pull and start storage-testbench, returning (http_endpoint, grpc_endpoint). + + When running inside a container on a Docker network (such as Cloud Build's + 'cloudbuild' network), attaches the testbench to that same network and + returns container-DNS endpoints; otherwise publishes host ports on localhost. + """ + subprocess.run(["docker", "pull", image], check=True) + stop_testbench(container_name) + + network = get_container_docker_network() + if network: + cmd = [ + "docker", + "run", + "--rm", + "-d", + "--name", + container_name, + "--network", + network, + image, + ] + http_endpoint = f"http://{container_name}:9000" + grpc_endpoint = ( + f"{container_name}:{host_grpc_port}" if host_grpc_port is not None else None + ) + else: + cmd = [ + "docker", + "run", + "--rm", + "-d", + "--name", + container_name, + "-p", + f"{host_http_port}:9000", + ] + if host_grpc_port is not None: + cmd.extend(["-p", f"{host_grpc_port}:{host_grpc_port}"]) + cmd.append(image) + http_endpoint = f"http://localhost:{host_http_port}" + grpc_endpoint = ( + f"localhost:{host_grpc_port}" if host_grpc_port is not None else None + ) + + subprocess.run(cmd, check=True) + + deadline = time.time() + ready_timeout + last_err = None + while time.time() < deadline: + try: + requests.get(f"{http_endpoint}/", timeout=2) + return http_endpoint, grpc_endpoint + except requests.exceptions.RequestException as exc: + last_err = exc + time.sleep(0.5) + + stop_testbench(container_name) + raise RuntimeError( + f"storage-testbench ({container_name}) did not become ready at " + f"{http_endpoint} within {ready_timeout}s: {last_err}" + ) + def start_grpc_server(grpc_endpoint, http_endpoint): """Starts the testbench gRPC server if it's not already running. diff --git a/packages/google-cloud-storage/tests/conformance/conftest.py b/packages/google-cloud-storage/tests/conformance/conftest.py index fb553ff1cbc3..a9bd31023323 100644 --- a/packages/google-cloud-storage/tests/conformance/conftest.py +++ b/packages/google-cloud-storage/tests/conformance/conftest.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import atexit import os import random import uuid +from typing import Optional import pytest from google.auth.credentials import AnonymousCredentials @@ -22,8 +24,55 @@ from google.cloud import storage +from ._utils import start_testbench, stop_testbench + """Environment variable or default host for Storage testbench emulator.""" _HOST = os.environ.get("STORAGE_EMULATOR_HOST", "http://localhost:9000") +_TESTBENCH_CONTAINER: Optional[str] = None + + +def _cleanup_testbench() -> None: + global _TESTBENCH_CONTAINER + container = _TESTBENCH_CONTAINER + if container: + _TESTBENCH_CONTAINER = None + stop_testbench(container) + + +def _running_retry_conformance(config) -> bool: + args = [str(arg) for arg in (config.args or [])] + if not args: + return True + return any("test_conformance" in arg or os.path.isdir(arg) for arg in args) + + +def pytest_configure(config) -> None: + """Start a single shared storage-testbench container for retry conformance tests. + + Under pytest-xdist, only the controller starts and stops the container; + workers inherit STORAGE_EMULATOR_HOST via os.environ. + """ + global _TESTBENCH_CONTAINER + if getattr(config.option, "help", False) or getattr( + config.option, "collectonly", False + ): + return + if hasattr(config, "workerinput"): + return + if os.environ.get("STORAGE_EMULATOR_HOST"): + return + if not _running_retry_conformance(config): + return + + container_name = f"storage-conformance-{uuid.uuid4().hex[:8]}" + http_endpoint, _ = start_testbench(container_name, host_http_port=9000) + _TESTBENCH_CONTAINER = container_name + atexit.register(_cleanup_testbench) + os.environ["STORAGE_EMULATOR_HOST"] = http_endpoint + + +def pytest_unconfigure(config) -> None: + _cleanup_testbench() """Emulated project information for the storage testbench.""" @@ -57,10 +106,11 @@ def _create_block(desired_kib): @pytest.fixture def client(): + host = os.environ.get("STORAGE_EMULATOR_HOST", _HOST) client = storage.Client( project=_CONF_TEST_PROJECT_ID, credentials=AnonymousCredentials(), - client_options={"api_endpoint": _HOST}, + client_options={"api_endpoint": host}, ) return client diff --git a/packages/google-cloud-storage/tests/conformance/test_bidi_reads.py b/packages/google-cloud-storage/tests/conformance/test_bidi_reads.py index 272da47bc034..3dace4d79c96 100644 --- a/packages/google-cloud-storage/tests/conformance/test_bidi_reads.py +++ b/packages/google-cloud-storage/tests/conformance/test_bidi_reads.py @@ -1,8 +1,5 @@ import io -import subprocess -import time import traceback -import urllib import uuid import pytest @@ -17,7 +14,7 @@ from google.cloud.storage.asyncio.async_multi_range_downloader import ( AsyncMultiRangeDownloader, ) -from tests.conformance._utils import start_grpc_server +from tests.conformance._utils import start_grpc_server, start_testbench, stop_testbench # --- Configuration --- @@ -25,45 +22,29 @@ TEST_BENCH_ENDPOINT = ( "http://localhost:9001" # 9000 in VM is taken by test_conformance.py ) -_PORT = urllib.parse.urlsplit(TEST_BENCH_ENDPOINT).port +_HTTP_PORT = 9001 _GRPC_PORT = 8888 PROJECT_NUMBER = "12345" # A dummy project number is fine for the testbench. GRPC_ENDPOINT = f"localhost:{_GRPC_PORT}" CONTENT_LENGTH = 1024 * 10 # 10 KB -_DEFAULT_IMAGE_NAME = "gcr.io/cloud-devrel-public-resources/storage-testbench" -_DEFAULT_IMAGE_TAG = "latest" -_DOCKER_IMAGE = f"{_DEFAULT_IMAGE_NAME}:{_DEFAULT_IMAGE_TAG}" -_PULL_CMD = ["docker", "pull", _DOCKER_IMAGE] -_RUN_CMD = [ - "docker", - "run", - "--name", - "bidi_reads_container", - "--rm", - "-d", - "-p", - f"{_PORT}:9000", - "-p", - f"{_GRPC_PORT}:{_GRPC_PORT}", - _DOCKER_IMAGE, -] -_DOCKER_STOP_CMD = [ - "docker", - "stop", - "bidi_reads_container", -] - @pytest.fixture(scope="module") def testbench(): - subprocess.run(_PULL_CMD) - proc = subprocess.Popen(_RUN_CMD) - time.sleep(10) - yield GRPC_ENDPOINT, TEST_BENCH_ENDPOINT - subprocess.run(_DOCKER_STOP_CMD) - proc.kill() + global TEST_BENCH_ENDPOINT, GRPC_ENDPOINT + container_name = f"bidi-reads-{uuid.uuid4().hex[:8]}" + http_endpoint, grpc_endpoint = start_testbench( + container_name, + host_http_port=_HTTP_PORT, + host_grpc_port=_GRPC_PORT, + ) + TEST_BENCH_ENDPOINT = http_endpoint + GRPC_ENDPOINT = grpc_endpoint + try: + yield GRPC_ENDPOINT, TEST_BENCH_ENDPOINT + finally: + stop_testbench(container_name) def _is_retriable(exc): diff --git a/packages/google-cloud-storage/tests/conformance/test_bidi_writes.py b/packages/google-cloud-storage/tests/conformance/test_bidi_writes.py index 5490a419ec84..aa423a97632c 100644 --- a/packages/google-cloud-storage/tests/conformance/test_bidi_writes.py +++ b/packages/google-cloud-storage/tests/conformance/test_bidi_writes.py @@ -1,6 +1,3 @@ -import subprocess -import time -import urllib import uuid import grpc @@ -15,51 +12,36 @@ AsyncAppendableObjectWriter, ) from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient -from tests.conformance._utils import start_grpc_server +from tests.conformance._utils import start_grpc_server, start_testbench, stop_testbench # --- Configuration --- TEST_BENCH_ENDPOINT = ( "http://localhost:9002" # 9000 in VM is taken by test_conformance.py, 9001 by reads ) -_PORT = urllib.parse.urlsplit(TEST_BENCH_ENDPOINT).port +_HTTP_PORT = 9002 _GRPC_PORT = 8888 PROJECT_NUMBER = "12345" # A dummy project number is fine for the testbench. GRPC_ENDPOINT = f"localhost:{_GRPC_PORT}" HTTP_ENDPOINT = TEST_BENCH_ENDPOINT -_DEFAULT_IMAGE_NAME = "gcr.io/cloud-devrel-public-resources/storage-testbench" -_DEFAULT_IMAGE_TAG = "latest" -_DOCKER_IMAGE = f"{_DEFAULT_IMAGE_NAME}:{_DEFAULT_IMAGE_TAG}" -_PULL_CMD = ["docker", "pull", _DOCKER_IMAGE] -_RUN_CMD = [ - "docker", - "run", - "--name", - "bidi_writes_container", - "--rm", - "-d", - "-p", - f"{_PORT}:9000", - "-p", - f"{_GRPC_PORT}:8888", - _DOCKER_IMAGE, -] -_DOCKER_STOP_CMD = [ - "docker", - "stop", - "bidi_writes_container", -] - @pytest.fixture(scope="module") def testbench(): - subprocess.run(_PULL_CMD) - proc = subprocess.Popen(_RUN_CMD) - time.sleep(10) - yield GRPC_ENDPOINT, HTTP_ENDPOINT - subprocess.run(_DOCKER_STOP_CMD) - proc.kill() + global TEST_BENCH_ENDPOINT, HTTP_ENDPOINT, GRPC_ENDPOINT + container_name = f"bidi-writes-{uuid.uuid4().hex[:8]}" + http_endpoint, grpc_endpoint = start_testbench( + container_name, + host_http_port=_HTTP_PORT, + host_grpc_port=_GRPC_PORT, + ) + TEST_BENCH_ENDPOINT = http_endpoint + HTTP_ENDPOINT = http_endpoint + GRPC_ENDPOINT = grpc_endpoint + try: + yield GRPC_ENDPOINT, HTTP_ENDPOINT + finally: + stop_testbench(container_name) CONTENT = b"A" * 1024 * 1024 * 10 # 10 MiB diff --git a/packages/google-cloud-storage/tests/conformance/test_conformance.py b/packages/google-cloud-storage/tests/conformance/test_conformance.py index 2263429acac4..fc1d662b28ed 100644 --- a/packages/google-cloud-storage/tests/conformance/test_conformance.py +++ b/packages/google-cloud-storage/tests/conformance/test_conformance.py @@ -17,10 +17,7 @@ import functools import logging import os -import subprocess import tempfile -import time -import urllib import uuid import pytest @@ -36,14 +33,6 @@ """Environment variable or default host for Storage testbench emulator.""" _HOST = os.environ.get("STORAGE_EMULATOR_HOST", "http://localhost:9000") -_PORT = urllib.parse.urlsplit(_HOST).port - -"""The storage testbench docker image info and commands.""" -_DEFAULT_IMAGE_NAME = "gcr.io/cloud-devrel-public-resources/storage-testbench" -_DEFAULT_IMAGE_TAG = "latest" -_DOCKER_IMAGE = f"{_DEFAULT_IMAGE_NAME}:{_DEFAULT_IMAGE_TAG}" -_PULL_CMD = ["docker", "pull", _DOCKER_IMAGE] -_RUN_CMD = ["docker", "run", "--rm", "-d", "-p", f"{_PORT}:9000", _DOCKER_IMAGE] _CONF_TEST_PROJECT_ID = "my-project-id" _CONF_TEST_SERVICE_ACCOUNT_EMAIL = ( @@ -922,6 +911,7 @@ def run_test_case( hmac_key, file_data, ): + host = host or os.environ.get("STORAGE_EMULATOR_HOST", _HOST) scenario = _CONFORMANCE_TESTS[scenario_id - 1] expect_success = scenario["expectSuccess"] precondition_provided = scenario["preconditionProvided"] @@ -979,29 +969,22 @@ def run_test_case( ### Run Conformance Tests for Retry Strategy ########################################################################################### ######################################################################################################################################## -# Pull storage-testbench docker image -subprocess.run(_PULL_CMD) -time.sleep(5) - -# Run docker image to start storage-testbench -with subprocess.Popen(_RUN_CMD) as proc: - # Run retry conformance tests - for scenario in _CONFORMANCE_TESTS: - id = scenario["id"] - methods = scenario["methods"] - cases = scenario["cases"] - for i, c in enumerate(cases): - for m in methods: - method_name = m["name"] - method_group = m["group"] if m.get("group", None) else m["name"] - if method_group not in method_mapping: - logging.info(f"No tests for operation {method_name}") - continue - - for lib_func in method_mapping[method_group]: - test_name = f"test-S{id}-{method_name}-{lib_func.__name__}-{i}" - globals()[test_name] = functools.partial( - run_test_case, id, m, c, lib_func, _HOST - ) - time.sleep(5) - proc.kill() +# Register retry conformance tests (storage-testbench lifecycle is managed +# once by conftest.pytest_configure / pytest_unconfigure). +for scenario in _CONFORMANCE_TESTS: + id = scenario["id"] + methods = scenario["methods"] + cases = scenario["cases"] + for i, c in enumerate(cases): + for m in methods: + method_name = m["name"] + method_group = m["group"] if m.get("group", None) else m["name"] + if method_group not in method_mapping: + logging.info(f"No tests for operation {method_name}") + continue + + for lib_func in method_mapping[method_group]: + test_name = f"test-S{id}-{method_name}-{lib_func.__name__}-{i}" + globals()[test_name] = functools.partial( + run_test_case, id, m, c, lib_func, None + ) diff --git a/packages/google-cloud-storage/tests/resumable_media/system/requests/conftest.py b/packages/google-cloud-storage/tests/resumable_media/system/requests/conftest.py index 67908795bded..8e3751db0833 100644 --- a/packages/google-cloud-storage/tests/resumable_media/system/requests/conftest.py +++ b/packages/google-cloud-storage/tests/resumable_media/system/requests/conftest.py @@ -13,6 +13,8 @@ # limitations under the License. """py.test fixtures to be shared across multiple system test modules.""" +import os + import google.auth # type: ignore import google.auth.transport.requests as tr_requests # type: ignore import pytest # type: ignore @@ -24,7 +26,12 @@ def ensure_bucket(transport): get_response = transport.get(utils.BUCKET_URL) if get_response.status_code == 404: credentials = transport.credentials - query_params = {"project": credentials.project_id} + project_id = ( + getattr(credentials, "project_id", None) + or os.environ.get("GOOGLE_CLOUD_PROJECT") + or os.environ.get("PROJECT_ID") + ) + query_params = {"project": project_id} payload = {"name": utils.BUCKET_NAME} post_response = transport.post( utils.BUCKET_POST_URL, params=query_params, json=payload diff --git a/packages/google-cloud-storage/tests/system/conftest.py b/packages/google-cloud-storage/tests/system/conftest.py index a74fd74f9652..827b1157367f 100644 --- a/packages/google-cloud-storage/tests/system/conftest.py +++ b/packages/google-cloud-storage/tests/system/conftest.py @@ -245,7 +245,7 @@ def signing_bucket_name(): @pytest.fixture(scope="session") -def signing_bucket(storage_client, signing_bucket_name): +def signing_bucket(storage_client, signing_bucket_name, service_account): bucket = storage_client.bucket(signing_bucket_name) # Create the bucket only if it doesn't yet exist. try: diff --git a/packages/google-cloud-storage/tests/system/test_bucket.py b/packages/google-cloud-storage/tests/system/test_bucket.py index ae194fe46793..39c65ae95237 100644 --- a/packages/google-cloud-storage/tests/system/test_bucket.py +++ b/packages/google-cloud-storage/tests/system/test_bucket.py @@ -456,7 +456,6 @@ def test_bucket_copy_blob_w_metageneration_match( payload = b"DEADBEEF" bucket_name = _helpers.unique_name("generation-match") bucket = storage_client.bucket(bucket_name) - bucket.requester_pays = True created = _helpers.retry_429_503(storage_client.create_bucket)(bucket) buckets_to_delete.append(created) assert created.name == bucket_name diff --git a/packages/google-resumable-media/tests/system/requests/conftest.py b/packages/google-resumable-media/tests/system/requests/conftest.py index 54ae10a2b5ca..7ea27e473755 100644 --- a/packages/google-resumable-media/tests/system/requests/conftest.py +++ b/packages/google-resumable-media/tests/system/requests/conftest.py @@ -13,6 +13,8 @@ # limitations under the License. """py.test fixtures to be shared across multiple system test modules.""" +import os + import google.auth # type: ignore import google.auth.transport.requests as tr_requests # type: ignore import pytest # type: ignore @@ -24,7 +26,12 @@ def ensure_bucket(transport): get_response = transport.get(utils.BUCKET_URL) if get_response.status_code == 404: credentials = transport.credentials - query_params = {"project": credentials.project_id} + project_id = ( + getattr(credentials, "project_id", None) + or os.environ.get("GOOGLE_CLOUD_PROJECT") + or os.environ.get("PROJECT_ID") + ) + query_params = {"project": project_id} payload = {"name": utils.BUCKET_NAME} post_response = transport.post( utils.BUCKET_POST_URL, params=query_params, json=payload diff --git a/packages/google-resumable-media/tests_async/system/requests/conftest.py b/packages/google-resumable-media/tests_async/system/requests/conftest.py index f9acc483223e..ba2395707374 100644 --- a/packages/google-resumable-media/tests_async/system/requests/conftest.py +++ b/packages/google-resumable-media/tests_async/system/requests/conftest.py @@ -13,6 +13,9 @@ # limitations under the License. """py.test fixtures to be shared across multiple system test modules.""" +import inspect +import os + import google.auth.transport._aiohttp_requests as tr_requests # type: ignore import pytest # type: ignore from google.auth._default_async import default_async # type: ignore @@ -24,7 +27,12 @@ async def ensure_bucket(transport): get_response = await transport.request("GET", utils.BUCKET_URL) if get_response.status == 404: credentials = transport.credentials - query_params = {"project": credentials.project_id} + project_id = ( + getattr(credentials, "project_id", None) + or os.environ.get("GOOGLE_CLOUD_PROJECT") + or os.environ.get("PROJECT_ID") + ) + query_params = {"project": project_id} payload = {"name": utils.BUCKET_NAME} post_response = await transport.request( "POST", utils.BUCKET_POST_URL, params=query_params, json=payload @@ -44,12 +52,22 @@ async def cleanup_bucket(transport): def _get_authorized_transport(): credentials, project_id = default_async(scopes=(utils.GCS_RW_SCOPE,)) + if not inspect.iscoroutinefunction(credentials.before_request): + import google.auth + import google.auth.transport.requests + from google.oauth2 import _credentials_async + + sync_creds, sync_project = google.auth.default(scopes=(utils.GCS_RW_SCOPE,)) + sync_creds.refresh(google.auth.transport.requests.Request()) + credentials = _credentials_async.Credentials(token=sync_creds.token) + credentials.project_id = ( + getattr(sync_creds, "project_id", None) or sync_project or project_id + ) return tr_requests.AuthorizedSession(credentials) @pytest.fixture(scope="module") async def authorized_transport(): - credentials, project_id = default_async(scopes=(utils.GCS_RW_SCOPE,)) yield _get_authorized_transport() diff --git a/packages/google-resumable-media/tests_async/system/requests/test_download.py b/packages/google-resumable-media/tests_async/system/requests/test_download.py index 5c5ac7d8582e..2602cfa6056f 100644 --- a/packages/google-resumable-media/tests_async/system/requests/test_download.py +++ b/packages/google-resumable-media/tests_async/system/requests/test_download.py @@ -23,7 +23,6 @@ import google.auth.transport._aiohttp_requests as tr_requests # type: ignore import multidict # type: ignore import pytest # type: ignore -from google.auth._default_async import default_async # type: ignore import google._async_resumable_media.requests as resumable_requests import google._async_resumable_media.requests.download as download_mod @@ -181,9 +180,8 @@ async def secret_file(authorized_transport, bucket): # Transport that returns corrupt data, so we can exercise checksum handling. @pytest.fixture(scope="module") -async def corrupting_transport(): - credentials, _ = default_async(scopes=(utils.GCS_RW_SCOPE,)) - yield CorruptingAuthorizedSession(credentials) +async def corrupting_transport(authorized_transport): + yield CorruptingAuthorizedSession(authorized_transport.credentials) @pytest.fixture(scope="module") diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py index 564625a92d73..264a63039864 100644 --- a/packages/pandas-gbq/tests/system/test_auth.py +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -11,7 +11,7 @@ from pandas_gbq import auth -IS_RUNNING_ON_CI = "KOKORO_BUILD_ID" in os.environ +IS_RUNNING_ON_CI = bool(os.environ.get("KOKORO_BUILD_ID") or os.environ.get("CI")) def mock_default_credentials(scopes=None, request=None):