diff --git a/.librarian/generator-input/client-post-processing/bigtable-integration.yaml b/.librarian/generator-input/client-post-processing/bigtable-integration.yaml index 7bb45a26eaea..af3fcfcf9270 100644 --- a/.librarian/generator-input/client-post-processing/bigtable-integration.yaml +++ b/.librarian/generator-input/client-post-processing/bigtable-integration.yaml @@ -405,6 +405,7 @@ replacements: SYSTEM_TEST_STANDARD_DEPENDENCIES: List[str] = [ "mock", "pytest", + "pytest-xdist", "google-cloud-testutils", ] SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [ @@ -696,6 +697,19 @@ replacements: test_map[test_type](session) + def _has_xdist_arg(posargs): + return any( + arg == "-n" + or ( + arg.startswith("-n") + and (arg[2:].isdigit() or arg[2:] in ("auto", "logical")) + ) + or arg == "--numprocesses" + or arg.startswith("--numprocesses=") + for arg in posargs + ) + + def system_default(session): """Run the system test suite.""" constraints_path = str( @@ -729,9 +743,18 @@ replacements: *session.posargs, ) if system_test_folder_exists: + in_emulator = bool( + session.env.get("BIGTABLE_EMULATOR_HOST") + or os.environ.get("BIGTABLE_EMULATOR_HOST") + ) + has_xdist_arg = _has_xdist_arg(session.posargs) + xdist_args = ( + [] if (in_emulator or has_xdist_arg) else ["-n", "8", "--dist=loadfile"] + ) session.run( "py.test", "--quiet", + *xdist_args, f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, *session.posargs, @@ -752,6 +775,7 @@ replacements: "mock", "pytest", "pytest-asyncio", + "pytest-xdist", *req_args, ) # Install the library from this checkout last so the samples exercise the @@ -761,9 +785,12 @@ replacements: has_target = any(not arg.startswith("-") for arg in session.posargs) target_paths = [] if has_target else [os.path.join("samples")] + has_xdist_arg = _has_xdist_arg(session.posargs) + xdist_args = [] if has_xdist_arg else ["-n", "8", "--dist=loadfile"] session.run( "py.test", "--quiet", + *xdist_args, f"--junitxml=samples_{session.python}_sponge_log.xml", "--ignore=samples/legacy_client/beam", *target_paths, diff --git a/ci/cancel_superseded_builds.py b/ci/cancel_superseded_builds.py new file mode 100644 index 000000000000..ad28a0aba426 --- /dev/null +++ b/ci/cancel_superseded_builds.py @@ -0,0 +1,345 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Cancel superseded Cloud Build PR runs when a newer commit is pushed. + +Cloud Build triggers do not provide a native ``cancel-in-progress`` setting +(unlike GitHub Actions ``concurrency`` or Kokoro's +``stop_outdated_presubmit_builds``). When Cloud Build concurrency slots are +saturated, a newly pushed commit sits in ``QUEUED`` and cannot even start its +first build step until an older ``WORKING`` build exits. + +This script solves both cases using only the Python standard library and the +Cloud Build REST API (authenticated via the GCE metadata server): + +1. ``--cancel-older`` (run once at build startup): + Lists all ``QUEUED`` / ``WORKING`` builds for the same trigger and PR number + whose ``createTime`` is earlier than this build and cancels them. +2. ``--watch`` (run in the background while tests execute): + Polls every ``--interval`` seconds (default 30s) for any ``QUEUED`` / + ``WORKING`` build for the same trigger and PR number whose ``createTime`` is + newer than this build (whether from a new commit or a duplicate re-run). If + found, this build cancels itself via ``builds:cancel`` so its concurrency + slot is freed immediately for the newer run. + +Requires ``PROJECT_ID``, ``LOCATION``, and ``BUILD_ID`` environment variables +(populated from Cloud Build built-in substitutions) and +``roles/cloudbuild.builds.editor`` on the build service account. + +Non-PR builds (e.g. postsubmit pushes on ``main`` where ``_PR_NUMBER`` is +absent) exit cleanly as a no-op. Any transient API/network errors fail open +with a warning so cancellation checks never break CI test execution. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + + +METADATA_TOKEN_URL = ( + "http://metadata.google.internal/computeMetadata/v1/" + "instance/service-accounts/default/token" +) + + +def _get_access_token() -> str: + """Fetch a short-lived OAuth2 access token from the GCE metadata server.""" + req = urllib.request.Request( + METADATA_TOKEN_URL, + headers={"Metadata-Flavor": "Google"}, + ) + with urllib.request.urlopen(req, timeout=10) as resp: + return json.loads(resp.read().decode("utf-8"))["access_token"] + + +def _api_request( + method: str, + url: str, + token: str, + body: bytes | None = None, +) -> dict: + """Execute an authenticated JSON HTTP request against the Cloud Build API.""" + headers = {"Authorization": f"Bearer {token}"} + if body is not None: + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=body, headers=headers, method=method) + with urllib.request.urlopen(req, timeout=15) as resp: + raw = resp.read().decode("utf-8") + return json.loads(raw) if raw else {} + + +def _parse_timestamp(raw_ts: str) -> datetime: + """Parse an RFC 3339 Cloud Build timestamp (e.g. ``2026-11-19T17:00:00.123Z``).""" + # Python 3.11+ fromisoformat handles trailing "Z", normalize for 3.10 compatibility. + normalized = raw_ts.replace("Z", "+00:00") + # Truncate sub-microsecond nanoseconds (9 digits) to 6 digits if present. + if "." in normalized: + date_part, rest = normalized.split(".", 1) + sign_idx = max(rest.find("+"), rest.find("-")) + frac = rest[:sign_idx][:6].ljust(6, "0") + tz = rest[sign_idx:] + normalized = f"{date_part}.{frac}{tz}" + return datetime.fromisoformat(normalized) + + +def _base_url(project_id: str, location: str) -> str: + """Return the regional Cloud Build v1 builds endpoint.""" + return ( + f"https://cloudbuild.googleapis.com/v1/" + f"projects/{project_id}/locations/{location}/builds" + ) + + +def _get_build(project_id: str, location: str, build_id: str, token: str) -> dict: + """Fetch metadata (trigger ID, substitutions, createTime) for a single build.""" + url = f"{_base_url(project_id, location)}/{build_id}" + return _api_request("GET", url, token) + + +def _list_active_trigger_builds( + project_id: str, + location: str, + trigger_id: str, + token: str, +) -> list[dict]: + """List all QUEUED or WORKING builds belonging to the same Cloud Build trigger.""" + filter_expr = f'trigger_id="{trigger_id}" AND (status="QUEUED" OR status="WORKING")' + builds: list[dict] = [] + page_token = "" + + while True: + params = {"filter": filter_expr, "pageSize": "100"} + if page_token: + params["pageToken"] = page_token + url = f"{_base_url(project_id, location)}?{urllib.parse.urlencode(params)}" + data = _api_request("GET", url, token) + builds.extend(data.get("builds", [])) + page_token = data.get("nextPageToken", "") + if not page_token: + break + + return builds + + +def _cancel_build(project_id: str, location: str, build_id: str, token: str) -> None: + """Issue a Cloud Build CancelBuild request for ``build_id``.""" + url = f"{_base_url(project_id, location)}/{build_id}:cancel" + _api_request("POST", url, token, body=b"{}") + + +def _extract_pr_context(build: dict) -> tuple[str, str, str, datetime] | None: + """Extract ``(trigger_id, pr_number, commit_sha, create_time)`` for a PR build. + + Returns ``None`` if the build is not associated with a trigger or pull request + (e.g. postsubmit continuous builds on ``main``). + """ + trigger_id = build.get("buildTriggerId", "") + substitutions = build.get("substitutions") or {} + pr_number = substitutions.get("_PR_NUMBER", "") + commit_sha = substitutions.get("COMMIT_SHA", "") + create_time_raw = build.get("createTime", "") + + if not trigger_id or not pr_number or not create_time_raw: + return None + + return trigger_id, pr_number, commit_sha, _parse_timestamp(create_time_raw) + + +def _cancel_older_builds( + project_id: str, + location: str, + my_build_id: str, + trigger_id: str, + pr_number: str, + my_commit_sha: str, + my_create_time: datetime, + token: str, +) -> None: + """Cancel any older QUEUED/WORKING builds for the same PR and trigger.""" + active_builds = _list_active_trigger_builds(project_id, location, trigger_id, token) + for other in active_builds: + other_id = other.get("id", "") + if not other_id or other_id == my_build_id: + continue + other_subs = other.get("substitutions") or {} + if other_subs.get("_PR_NUMBER") != pr_number: + continue + other_sha = other_subs.get("COMMIT_SHA", "") + other_create_raw = other.get("createTime", "") + if not other_create_raw: + continue + # Cancel any earlier build for the same PR and trigger — including + # duplicate runs / manual re-runs of the same commit SHA — using + # build ID as a deterministic tie-breaker if timestamps match. + if (_parse_timestamp(other_create_raw), other_id) < ( + my_create_time, + my_build_id, + ): + print( + f"[cancel-superseded] Cancelling older {other.get('status')} build " + f"{other_id} (PR #{pr_number}, commit {other_sha[:8]}) superseded by " + f"{my_build_id} (commit {my_commit_sha[:8]}).", + flush=True, + ) + try: + _cancel_build(project_id, location, other_id, token) + except urllib.error.URLError as exc: + print( + f"[cancel-superseded] Warning: failed to cancel {other_id}: {exc}", + file=sys.stderr, + flush=True, + ) + + +def _watch_for_newer_build( + project_id: str, + location: str, + my_build_id: str, + trigger_id: str, + pr_number: str, + my_commit_sha: str, + my_create_time: datetime, + interval: int, +) -> None: + """Poll until a newer commit build for the same PR + trigger appears, then self-cancel.""" + while True: + time.sleep(interval) + try: + # Refresh metadata token each iteration so long-running builds never hit expiry. + token = _get_access_token() + active_builds = _list_active_trigger_builds( + project_id, location, trigger_id, token + ) + for other in active_builds: + other_id = other.get("id", "") + if not other_id or other_id == my_build_id: + continue + other_subs = other.get("substitutions") or {} + if other_subs.get("_PR_NUMBER") != pr_number: + continue + other_sha = other_subs.get("COMMIT_SHA", "") + other_create_raw = other.get("createTime", "") + if not other_create_raw: + continue + if (_parse_timestamp(other_create_raw), other_id) > ( + my_create_time, + my_build_id, + ): + print( + f"[cancel-superseded] Newer {other.get('status')} build {other_id} " + f"(PR #{pr_number}, commit {other_sha[:8]}) detected; " + f"self-cancelling {my_build_id} to free worker slot.", + flush=True, + ) + _cancel_build(project_id, location, my_build_id, token) + return + except Exception as exc: + # Fail open on transient metadata/API errors and keep watching. + print( + f"[cancel-superseded] Watchdog warning (will retry): {exc}", + file=sys.stderr, + flush=True, + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Cancel superseded Cloud Build PR runs for the same trigger." + ) + parser.add_argument( + "--cancel-older", + action="store_true", + help="Cancel older QUEUED/WORKING builds for the same PR and trigger.", + ) + parser.add_argument( + "--watch", + action="store_true", + help="Continuously watch for a newer commit on the same PR and self-cancel.", + ) + parser.add_argument( + "--interval", + type=int, + default=30, + help="Polling interval in seconds for --watch mode (default: 30).", + ) + args = parser.parse_args() + + project_id = os.environ.get("PROJECT_ID", "") + location = os.environ.get("LOCATION", "") + build_id = os.environ.get("BUILD_ID", "") + if not project_id or not location or not build_id: + return 0 + + try: + token = _get_access_token() + current_build = _get_build(project_id, location, build_id, token) + except Exception as exc: + print( + f"[cancel-superseded] Warning: unable to query build metadata ({exc}); skipping.", + file=sys.stderr, + flush=True, + ) + return 0 + + ctx = _extract_pr_context(current_build) + if ctx is None: + # Postsubmit push or manual build without _PR_NUMBER; nothing to cancel. + return 0 + + trigger_id, pr_number, my_commit_sha, my_create_time = ctx + + if args.cancel_older: + try: + _cancel_older_builds( + project_id, + location, + build_id, + trigger_id, + pr_number, + my_commit_sha, + my_create_time, + token, + ) + except Exception as exc: + print( + f"[cancel-superseded] Warning: failed to cancel older builds ({exc}).", + file=sys.stderr, + flush=True, + ) + + if args.watch: + _watch_for_newer_build( + project_id, + location, + build_id, + trigger_id, + pr_number, + my_commit_sha, + my_create_time, + args.interval, + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml new file mode 100644 index 000000000000..7c0d65939370 --- /dev/null +++ b/ci/cloudbuild/system-test.yaml @@ -0,0 +1,382 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - name: 'us-central1-docker.pkg.dev/client-library-releases/release-images-prod/python-multi@sha256:d529954c373542171f9fc411e0102cebe542b511e320a965edf8820285f4f122' + id: 'run-system-tests-shard' + secretEnv: + - 'GAPIC_GENERATOR_ADS_ENV' + - 'GAPIC_GENERATOR_SERVICE_ACCOUNT_JSON' + - 'GOOGLE_AUTH_SERVICE_ACCOUNT_JSON' + - 'GOOGLE_AUTH_AUTHORIZED_USER_JSON' + env: + # Populate standard GCP project and CI environment variables expected by system test suites. + - 'CI=true' + - 'PIP_BREAK_SYSTEM_PACKAGES=1' + - 'UV_VENV_SEED=1' + - 'RUN_COMPLIANCE_TESTS=false' + - 'GOOGLE_CLOUD_PROJECT=${PROJECT_ID}' + - 'GCLOUD_PROJECT=${PROJECT_ID}' + - 'PROJECT_ID=${PROJECT_ID}' + # Cloud Build built-ins used by ci/cancel_superseded_builds.py to cancel outdated PR runs. + - 'LOCATION=${LOCATION}' + - 'BUILD_ID=${BUILD_ID}' + - 'FIRESTORE_PROJECT_ID=${_FIRESTORE_PROJECT_ID}' + # Pass sharding and diff parameters consumed by ci/get_package_shards.py. + - 'MAX_SHARDS=${_MAX_SHARDS}' + - 'SHARD_INDEX=${_SHARD_INDEX}' + - 'MIN_SHARD_WEIGHT=${_MIN_SHARD_WEIGHT}' + - 'BUILD_TYPE=${_BUILD_TYPE}' + - 'TARGET_BRANCH=${_BASE_BRANCH}' + - 'TEST_ALL_PACKAGES=${_TEST_ALL_PACKAGES}' + - 'DISABLED_PACKAGES=${_DISABLED_PACKAGES}' + - 'PACKAGE_WEIGHTS=${_PACKAGE_WEIGHTS}' + # Specify the nox session consumed by ci/run_conditional_tests.sh and ci/run_single_test.sh. + - 'NOX_SESSION=${_NOX_SESSION}' + - 'TEST_TYPE=${_NOX_SESSION}' + # Configure parallel package execution, time budget/threshold, and unbuffered Python output. + - 'PARALLEL_WORKERS=${_PARALLEL_WORKERS}' + - 'CONTINUE_ON_ERROR=true' + - 'TIME_BUDGET_MINUTES=${_TIME_BUDGET_MINUTES}' + - 'HARD_TIMEOUT_MINUTES=${_HARD_TIMEOUT_MINUTES}' + - 'PYTHONUNBUFFERED=1' + script: | + #!/usr/bin/env bash + set -eo pipefail + + # Cancel any older QUEUED/WORKING builds for this PR and shard so newer pushes + # immediately take precedence instead of piling up in the Cloud Build queue. + python3 ci/cancel_superseded_builds.py --cancel-older || true + + # Derive PY_VERSION (e.g. "3.12" from "system-3.12") required by ci/run_single_test.sh. + export PY_VERSION="${NOX_SESSION#system-}" + + # Deepen both the checked-out PR commit and target branch history so git diff in + # ci/get_package_shards.py finds the merge base in Cloud Build's shallow checkout, + # and populate local refs/heads/${TARGET_BRANCH} so ci/run_conditional_tests.sh's + # local insteadOf fetch succeeds silently. + git fetch --no-tags --quiet origin \ + "$(git rev-parse HEAD)" \ + "+refs/heads/${TARGET_BRANCH}:refs/remotes/origin/${TARGET_BRANCH}" \ + "+refs/heads/${TARGET_BRANCH}:refs/heads/${TARGET_BRANCH}" \ + --depth=200 || true + + # Import ci/get_package_shards.py without modifying the script on disk. + # Filter get_packages() in memory to exclude _DISABLED_PACKAGES. + PACKAGE_LIST=$(python3 -c ' + import os + import sys + + sys.path.insert(0, "ci") + import get_package_shards + + # Normalize disabled package paths by stripping trailing slashes. + disabled_packages = { + package_path.strip().rstrip("/") + for package_path in os.environ.get("DISABLED_PACKAGES", "").replace(",", " ").split() + if package_path.strip() + } + + def has_system_tests(package_path): + normalized = package_path.strip().rstrip("/") + return normalized.startswith("packages/") and any( + os.path.exists(os.path.join(normalized, rel_path)) + for rel_path in ( + "tests/system", + "tests/system_live", + "system_tests", + "tests/system.py", + ) + ) + + original_get_packages = get_package_shards.get_packages + def filtered_get_packages(*args, **kwargs): + filtered_packages = {} + for package_name, package_paths in original_get_packages( + handwritten_only=False + ).items(): + # Keep only packages/ paths with system tests that are NOT in disabled_packages + kept_paths = [ + path + for path in package_paths + if path.strip().rstrip("/") not in disabled_packages + and has_system_tests(path) + ] + if kept_paths: + filtered_packages[package_name] = kept_paths + return filtered_packages + + get_package_shards.get_packages = filtered_get_packages + + # Compute the packages that need testing and distribute them across balanced shards. + packages_to_test = get_package_shards.get_packages_to_test() + shards = get_package_shards.group_packages(packages_to_test) + + shard_index = int(os.environ.get("SHARD_INDEX", "0")) + print(shards[shard_index].get("packages", "") if shard_index < len(shards) else "") + ') + + # Report immediate green (< 10s) if no packages in this PR map to this shard index. + if [[ -z "${PACKAGE_LIST}" ]]; then + echo "No packages assigned to shard ${SHARD_INDEX}. Exiting 0." + exit 0 + fi + + # Install uv so nox sessions use fast cached wheel installation instead of slow virtualenv + pip. + python3 -m pip install --quiet uv || true + + # Create runtime shims in /tmp/cloudbuild_shims so tracked repository files remain unmodified. + mkdir -p /tmp/cloudbuild_shims + + # 1. Python startup hook (sitecustomize.py) & Pytest plugin (cloudbuild_pytest_plugin.py): + # - Package noxfile.py files skip the system session if GOOGLE_APPLICATION_CREDENTIALS + # is unset. We export a placeholder value ("CLOUD_BUILD_ADC") to satisfy noxfile.py, + # then remove it in sitecustomize.py for every non-nox Python subprocess (such as + # sqlalchemy-spanner's create_test_database.py and pytest) so google.auth.default() + # uses the Cloud Build worker VM's Application Default Credentials (ADC). + # - Package noxfile.py files invoke `py.test --quiet`, which only outputs dots (".") + # without newlines and causes Cloud Build logs to appear frozen for several minutes. + # Overriding `quiet = 0` and `verbose = 1` makes pytest print one newline-terminated + # status line as each individual test finishes. + cat << 'EOF' > /tmp/cloudbuild_shims/sitecustomize.py + import os + import sys + + if ( + os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") == "CLOUD_BUILD_ADC" + and os.path.basename(sys.argv[0]) != "nox" + ): + os.environ.pop("GOOGLE_APPLICATION_CREDENTIALS", None) + EOF + + cat << 'EOF' > /tmp/cloudbuild_shims/cloudbuild_pytest_plugin.py + import os + + if os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") == "CLOUD_BUILD_ADC": + os.environ.pop("GOOGLE_APPLICATION_CREDENTIALS", None) + + + def pytest_configure(config): + config.option.quiet = 0 + config.option.verbose = max(config.option.verbose, 1) + EOF + + # 2. Nox wrapper (/tmp/cloudbuild_shims/nox): + # - When PARALLEL_WORKERS > 1, `ci/run_conditional_tests.sh` redirects each worker's + # stdout/stderr (file descriptors 1 and 2) to a temporary log file, which otherwise + # produces no console output until the entire package finishes. + # - Before invoking `ci/run_conditional_tests.sh`, we duplicate the container's live + # stdout to file descriptor 3 (`exec 3>&1`). This wrapper streams every output line + # prefixed with `[]` directly to file descriptor 3 in real time while + # also saving the raw output to a temporary file (`TEMP_LOG_FILE`). + # - At completion, it calculates how much of the per-package time budget was used + # (e.g., 18 minutes out of a 20-minute budget = 90%) and logs the summary. + # - If the nox session fails, it outputs the `=== FAILURES ===` section to stdout so + # `ci/run_conditional_tests.sh` prints a clean failure summary at the end of the run. + cat << 'EOF' > /tmp/cloudbuild_shims/nox + #!/usr/bin/env bash + PACKAGE_NAME=$(basename "$PWD") + REAL_NOX=$(which -a nox | grep -v '/tmp/cloudbuild_shims' | head -n 1) + TEMP_LOG_FILE=$(mktemp) + START_TIME=$(date +%s) + NOX_ARGS=("$@") + + # Per-package setup for google-cloud-firestore, google-auth, and gapic-generator. + if [[ "${PACKAGE_NAME}" == "google-cloud-firestore" && -n "${FIRESTORE_PROJECT_ID:-}" ]]; then + export GOOGLE_CLOUD_PROJECT="${FIRESTORE_PROJECT_ID}" + export GCLOUD_PROJECT="${FIRESTORE_PROJECT_ID}" + elif [[ "${PACKAGE_NAME}" == "google-auth" ]]; then + mkdir -p system_tests/data + printf '%s' "${GOOGLE_AUTH_SERVICE_ACCOUNT_JSON}" > system_tests/data/service_account.json + printf '%s' "${GOOGLE_AUTH_AUTHORIZED_USER_JSON}" > system_tests/data/authorized_user.json + export GOOGLE_APPLICATION_CREDENTIALS="${PWD}/system_tests/data/service_account.json" + NOX_ARGS=(--stop-on-first-error -f system_tests/noxfile.py) + elif [[ "${PACKAGE_NAME}" == "gapic-generator" ]]; then + ADS_ENV_FILE=$(mktemp) + ADS_SA_FILE=$(mktemp --suffix=.json) + trap 'rm -f "${ADS_ENV_FILE}" "${ADS_SA_FILE}" "${TEMP_LOG_FILE}"' EXIT + printf '%s\n' "${GAPIC_GENERATOR_ADS_ENV}" > "${ADS_ENV_FILE}" + printf '%s' "${GAPIC_GENERATOR_SERVICE_ACCOUNT_JSON}" > "${ADS_SA_FILE}" + set -a + source "${ADS_ENV_FILE}" + set +a + rm -f "${ADS_ENV_FILE}" + export GOOGLE_APPLICATION_CREDENTIALS="${ADS_SA_FILE}" + export GOOGLE_ADS_VIDEO_PATH="/tmp/gapic_generator_test_video.mp4" + if [[ ! -f "${GOOGLE_ADS_VIDEO_PATH}" ]]; then + if command -v ffmpeg >/dev/null 2>&1; then + ffmpeg -y -loglevel error -f lavfi -i testsrc=duration=8:size=1280x720:rate=30 -c:v libx264 -b:v 1M -pix_fmt yuv420p "${GOOGLE_ADS_VIDEO_PATH}" + else + python3 -c ' + import os, struct + ftyp = struct.pack(">I4s4sI4s", 20, b"ftyp", b"isom", 512, b"isom") + pad = b"\x00" * (600 * 1024) + free = struct.pack(">I4s", 8 + len(pad), b"free") + pad + open(os.environ["GOOGLE_ADS_VIDEO_PATH"], "wb").write(ftyp + free) + ' + fi + fi + fi + + echo "[${PACKAGE_NAME}] Starting nox ${NOX_ARGS[*]}" >&3 + "$REAL_NOX" "${NOX_ARGS[@]}" 2>&1 | python3 -u -c ' + import os, sys + prefix = f"[{sys.argv[1]}] " + with open(sys.argv[2], "w", buffering=1) as log_f, os.fdopen(3, "w", buffering=1, closefd=False) as live_f: + for line in sys.stdin: + log_f.write(line) + live_f.write(prefix + line) + ' "${PACKAGE_NAME}" "${TEMP_LOG_FILE}" + EXIT_CODE=${PIPESTATUS[0]} + + # Calculate elapsed time and percentage of the target time budget spent on this package. + ELAPSED_SECONDS=$(( $(date +%s) - START_TIME )) + BUDGET_MINUTES="${TIME_BUDGET_MINUTES:-20}" + HARD_LIMIT_MINUTES="${HARD_TIMEOUT_MINUTES:-30}" + BUDGET_SECONDS=$(( BUDGET_MINUTES * 60 )) + BUDGET_PERCENTAGE=$(( (ELAPSED_SECONDS * 100 + BUDGET_SECONDS / 2) / BUDGET_SECONDS )) + BUDGET_MESSAGE=$(printf "Time budget usage: %d%% (%dm %02ds / %dm budget, %dm hard threshold)" \ + "${BUDGET_PERCENTAGE}" "$(( ELAPSED_SECONDS / 60 ))" "$(( ELAPSED_SECONDS % 60 ))" \ + "${BUDGET_MINUTES}" "${HARD_LIMIT_MINUTES}") + echo "[${PACKAGE_NAME}] ${BUDGET_MESSAGE}" >&3 + echo "${BUDGET_MESSAGE}" + + # Compare actual runtime in minutes against the configured weight in PACKAGE_WEIGHTS + # (defaulting to 1). Only propose an adjustment when the difference is at least 3 minutes + # (>= 15% of the 20-minute budget) to avoid noise from normal test runtime variance. + PROPOSED_WEIGHT=$(( (ELAPSED_SECONDS + 30) / 60 )) + if [[ ${PROPOSED_WEIGHT} -lt 1 ]]; then + PROPOSED_WEIGHT=1 + fi + CONFIGURED_WEIGHT=$(printf '%s\n' "${PACKAGE_WEIGHTS:-}" | awk -F: -v package_name="${PACKAGE_NAME}" ' + { gsub(/^[[:space:]]+|[[:space:]]+$/, "", $1); gsub(/[[:space:]]/, "", $2) } + $1 == package_name && $2 != "" { print $2 } + ' | tail -n 1) + CONFIGURED_WEIGHT="${CONFIGURED_WEIGHT:-1}" + WEIGHT_DIFFERENCE=$(( PROPOSED_WEIGHT - CONFIGURED_WEIGHT )) + if [[ ${WEIGHT_DIFFERENCE} -lt 0 ]]; then + WEIGHT_DIFFERENCE=$(( -WEIGHT_DIFFERENCE )) + fi + if [[ ${WEIGHT_DIFFERENCE} -ge 3 ]]; then + WEIGHT_MESSAGE="Weight adjustment needed for ${PACKAGE_NAME}: configured=${CONFIGURED_WEIGHT}, proposed=${PROPOSED_WEIGHT} (${PACKAGE_NAME}:${PROPOSED_WEIGHT})" + echo "[${PACKAGE_NAME}] ${WEIGHT_MESSAGE}" >&3 + echo "${WEIGHT_MESSAGE}" + echo " ${PACKAGE_NAME}: ${PROPOSED_WEIGHT} # was ${CONFIGURED_WEIGHT} (actual: $(( ELAPSED_SECONDS / 60 ))m $(printf '%02d' $(( ELAPSED_SECONDS % 60 )))s)" >> /tmp/cloudbuild_weight_adjustments.txt + fi + + # Emit failure details to stdout (captured by run_conditional_tests.sh) only when tests fail + # so passing runs do not print their logs a second time. + if [[ ${EXIT_CODE} -ne 0 ]]; then + sed -n '/=== FAILURES ===/,$p' "${TEMP_LOG_FILE}" + fi + rm -f "${TEMP_LOG_FILE}" + exit ${EXIT_CODE} + EOF + chmod +x /tmp/cloudbuild_shims/nox + + # Set placeholder credentials for noxfile.py if not already set. + if [[ -z "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then + export GOOGLE_APPLICATION_CREDENTIALS="CLOUD_BUILD_ADC" + fi + + # Prepend /tmp/cloudbuild_shims to PATH and PYTHONPATH so our nox wrapper and pytest + # plugin are picked up automatically by ci/run_single_test.sh and pytest. + export PATH="/tmp/cloudbuild_shims:${PATH}" + export PYTHONPATH="/tmp/cloudbuild_shims${PYTHONPATH:+:${PYTHONPATH}}" + export PYTEST_ADDOPTS="-p cloudbuild_pytest_plugin ${PYTEST_ADDOPTS:-}" + + # Open file descriptor 3 pointing to the container's live stdout so child workers + # spawned by xargs in ci/run_conditional_tests.sh can stream real-time logs. + exec 3>&1 + + # Export PACKAGE_LIST and delegate execution to the existing monorepo test runner script. + rm -f /tmp/cloudbuild_weight_adjustments.txt + CANCEL_WATCHDOG_PID="" + cleanup_on_exit() { + # Stop the background cancellation watchdog so it never keeps the build container open. + if [[ -n "${CANCEL_WATCHDOG_PID}" ]]; then + kill "${CANCEL_WATCHDOG_PID}" 2>/dev/null || true + fi + echo "" + echo "============================================================" + if [[ -s /tmp/cloudbuild_weight_adjustments.txt ]]; then + echo "⚠️ WEIGHT ADJUSTMENTS NEEDED (drifted >= 3m from configured weight):" + sort -u /tmp/cloudbuild_weight_adjustments.txt + else + echo "✅ WEIGHT SUMMARY: No weight adjustments needed (all packages within 2m of configured weight)." + fi + echo "============================================================" + } + trap cleanup_on_exit EXIT + + # Run a background watchdog that self-cancels this build if a newer commit is pushed + # to the same PR while all Cloud Build worker slots are occupied. + python3 ci/cancel_superseded_builds.py --watch & + CANCEL_WATCHDOG_PID=$! + + export PACKAGE_LIST + echo "=== System Shard ${SHARD_INDEX}/${MAX_SHARDS} running ${NOX_SESSION} for: ${PACKAGE_LIST} ===" + ./ci/run_conditional_tests.sh + +# Hard 30-minute (1800s) CI timeout ceiling. +timeout: 1800s + +substitutions: + _MAX_SHARDS: '5' + _SHARD_INDEX: '0' + _MIN_SHARD_WEIGHT: '10' + _PARALLEL_WORKERS: '4' + _TIME_BUDGET_MINUTES: '20' + _HARD_TIMEOUT_MINUTES: '30' + _BUILD_TYPE: 'presubmit' + _BASE_BRANCH: 'main' + _TEST_ALL_PACKAGES: 'false' + _DISABLED_PACKAGES: '' + _FIRESTORE_PROJECT_ID: '' + _NOX_SESSION: 'system-3.12' + _PACKAGE_WEIGHTS: | + bigquery-magics: 2 + gapic-generator: 5 + google-auth: 5 + google-cloud-bigquery: 8 + google-cloud-bigquery-storage: 8 + google-cloud-bigtable: 8 + google-cloud-compute: 8 + google-cloud-datastore: 8 + google-cloud-dns: 1 + google-cloud-error-reporting: 1 + google-cloud-firestore: 8 + google-cloud-logging: 10 + google-cloud-ndb: 12 + google-cloud-pubsub: 15 + google-cloud-spanner: 17 + google-cloud-storage: 7 + google-resumable-media: 1 + pandas-gbq: 12 + sqlalchemy-bigquery: 11 + +options: + machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY + +availableSecrets: + secretManager: + - versionName: projects/$PROJECT_ID/secrets/gapic-generator-ads-env/versions/latest + env: 'GAPIC_GENERATOR_ADS_ENV' + - versionName: projects/$PROJECT_ID/secrets/gapic-generator-service-account/versions/latest + env: 'GAPIC_GENERATOR_SERVICE_ACCOUNT_JSON' + - versionName: projects/$PROJECT_ID/secrets/google-auth-service-account/versions/latest + env: 'GOOGLE_AUTH_SERVICE_ACCOUNT_JSON' + - versionName: projects/$PROJECT_ID/secrets/google-auth-authorized-user/versions/latest + env: 'GOOGLE_AUTH_AUTHORIZED_USER_JSON' diff --git a/packages/google-cloud-bigquery-storage/samples/snippets/query_and_wait_arrow_test.py b/packages/google-cloud-bigquery-storage/samples/snippets/query_and_wait_arrow_test.py index ad7af73392b4..91ddd405d20e 100644 --- a/packages/google-cloud-bigquery-storage/samples/snippets/query_and_wait_arrow_test.py +++ b/packages/google-cloud-bigquery-storage/samples/snippets/query_and_wait_arrow_test.py @@ -12,12 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools + import pyarrow +from google.cloud import bigquery + from . import query_and_wait_arrow -def test_query_and_wait_arrow(): +def test_query_and_wait_arrow(monkeypatch): + # TODO: Remove use_query_cache=False workaround once google-cloud-bigquery > 3.47.0 + # (containing #18958) is released and pinned in requirements.txt. + monkeypatch.setattr( + query_and_wait_arrow.bigquery, + "Client", + functools.partial( + bigquery.Client, + default_query_job_config=bigquery.QueryJobConfig(use_query_cache=False), + ), + ) batches = query_and_wait_arrow.query_and_wait_arrow() total_rows = 0 diff --git a/packages/google-cloud-bigquery-storage/samples/snippets/requirements.txt b/packages/google-cloud-bigquery-storage/samples/snippets/requirements.txt index 5b613fdf002f..4f6a4e7b35ca 100644 --- a/packages/google-cloud-bigquery-storage/samples/snippets/requirements.txt +++ b/packages/google-cloud-bigquery-storage/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ google-cloud-bigquery-storage==2.43.0 -google-cloud-bigquery==3.46.2 +google-cloud-bigquery==3.47.0 pyarrow==26.0.0 pytest==9.1.1 diff --git a/packages/google-cloud-bigquery/samples/snippets/authenticate_service_account_test.py b/packages/google-cloud-bigquery/samples/snippets/authenticate_service_account_test.py index fbdd2d0649ff..9c3ef374d304 100644 --- a/packages/google-cloud-bigquery/samples/snippets/authenticate_service_account_test.py +++ b/packages/google-cloud-bigquery/samples/snippets/authenticate_service_account_test.py @@ -24,9 +24,11 @@ def mock_credentials(*args: Any, **kwargs: Any) -> google.auth.credentials.Credentials: - credentials, _ = google.auth.default( + credentials, project_id = google.auth.default( ["https://www.googleapis.com/auth/cloud-platform"] ) + if not hasattr(credentials, "project_id"): + credentials.project_id = project_id # type: ignore[attr-defined] return credentials diff --git a/packages/google-cloud-bigtable/google/cloud/bigtable/data/_sync_autogen/client.py b/packages/google-cloud-bigtable/google/cloud/bigtable/data/_sync_autogen/client.py index a485b53f78c4..e485894419cf 100644 --- a/packages/google-cloud-bigtable/google/cloud/bigtable/data/_sync_autogen/client.py +++ b/packages/google-cloud-bigtable/google/cloud/bigtable/data/_sync_autogen/client.py @@ -1424,9 +1424,7 @@ def mutate_row( def execute_rpc(): return self.client._gapic_client.mutate_row( - request=request, - timeout=next(attempt_timeout_gen), - retry=None, + request=request, timeout=next(attempt_timeout_gen), retry=None ) return tracked_retry( diff --git a/packages/google-cloud-bigtable/noxfile.py b/packages/google-cloud-bigtable/noxfile.py index 045dcc166a38..dff330ad3691 100644 --- a/packages/google-cloud-bigtable/noxfile.py +++ b/packages/google-cloud-bigtable/noxfile.py @@ -54,6 +54,7 @@ SYSTEM_TEST_STANDARD_DEPENDENCIES: List[str] = [ "mock", "pytest", + "pytest-xdist", "google-cloud-testutils", ] SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [ @@ -345,6 +346,19 @@ def system(session, test_type): test_map[test_type](session) +def _has_xdist_arg(posargs): + return any( + arg == "-n" + or ( + arg.startswith("-n") + and (arg[2:].isdigit() or arg[2:] in ("auto", "logical")) + ) + or arg == "--numprocesses" + or arg.startswith("--numprocesses=") + for arg in posargs + ) + + def system_default(session): """Run the system test suite.""" constraints_path = str( @@ -378,9 +392,18 @@ def system_default(session): *session.posargs, ) if system_test_folder_exists: + in_emulator = bool( + session.env.get("BIGTABLE_EMULATOR_HOST") + or os.environ.get("BIGTABLE_EMULATOR_HOST") + ) + has_xdist_arg = _has_xdist_arg(session.posargs) + xdist_args = ( + [] if (in_emulator or has_xdist_arg) else ["-n", "8", "--dist=loadfile"] + ) session.run( "py.test", "--quiet", + *xdist_args, f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, *session.posargs, @@ -401,6 +424,7 @@ def samples(session): "mock", "pytest", "pytest-asyncio", + "pytest-xdist", *req_args, ) # Install the library from this checkout last so the samples exercise the @@ -410,9 +434,12 @@ def samples(session): has_target = any(not arg.startswith("-") for arg in session.posargs) target_paths = [] if has_target else [os.path.join("samples")] + has_xdist_arg = _has_xdist_arg(session.posargs) + xdist_args = [] if has_xdist_arg else ["-n", "8", "--dist=loadfile"] session.run( "py.test", "--quiet", + *xdist_args, f"--junitxml=samples_{session.python}_sponge_log.xml", "--ignore=samples/legacy_client/beam", *target_paths, diff --git a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py index 04a164a0f9ce..70c3c98e28d9 100644 --- a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py +++ b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py @@ -23,7 +23,9 @@ import argparse -def run_instance_operations(project_id, instance_id, cluster_id): +def run_instance_operations( + project_id, instance_id, cluster_id, location_id="us-central1-f" +): # [START bigtable_check_instance_exists] from google.api_core.exceptions import NotFound @@ -49,7 +51,7 @@ def run_instance_operations(project_id, instance_id, cluster_id): instance_path = client.instance_path(project_id, instance_id) cluster = bigtable_admin_v2.Cluster( - location=client.common_location_path(project_id, "us-central1-f"), + location=client.common_location_path(project_id, location_id), serve_nodes=1, default_storage_type=bigtable_admin_v2.StorageType.SSD, ) @@ -128,7 +130,7 @@ def delete_instance(project_id, instance_id): # [END bigtable_delete_instance] -def add_cluster(project_id, instance_id, cluster_id): +def add_cluster(project_id, instance_id, cluster_id, location_id="us-central1-b"): # [START bigtable_create_cluster] from google.api_core.exceptions import NotFound @@ -144,7 +146,7 @@ def add_cluster(project_id, instance_id, cluster_id): print(cluster.name.split("/")[-1]) new_cluster = bigtable_admin_v2.Cluster( - location=client.common_location_path(project_id, "us-central1-a"), + location=client.common_location_path(project_id, location_id), serve_nodes=1, default_storage_type=bigtable_admin_v2.StorageType.SSD, ) diff --git a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py index 1c5203b1a680..eb6158b66429 100644 --- a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py +++ b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py @@ -28,18 +28,59 @@ INSTANCE_ID = f"instance-admin-{str(uuid.uuid4())[:8]}" CLUSTER_ID = f"cluster-admin-{str(uuid.uuid4())[:8]}" NEW_CLUSTER_ID = f"cluster-add-{str(uuid.uuid4())[:8]}" +FALLBACK_ZONES = ( + "us-central1-f", + "us-central1-b", + "us-central1-c", + "us-east1-b", + "us-west1-b", +) + + +def _is_zone_stockout(exc): + message = str(exc).lower() + return "sold out" in message or "not enough servers" in message + + +def _run_with_zone_fallback(fn, zones, capsys, *args): + last_exc = None + for zone in zones: + capsys.readouterr() + try: + fn(*args, location_id=zone) + return zone + except exceptions.GoogleAPICallError as exc: + if not _is_zone_stockout(exc): + raise + last_exc = exc + raise last_exc def test_instance_operations(capsys): try: - run_instance_operations(PROJECT, INSTANCE_ID, CLUSTER_ID) + primary_zone = _run_with_zone_fallback( + run_instance_operations, + FALLBACK_ZONES, + capsys, + PROJECT, + INSTANCE_ID, + CLUSTER_ID, + ) out, _ = capsys.readouterr() assert f"Created instance: {INSTANCE_ID}" in out assert "Listing instances:" in out assert "Listing clusters..." in out backoff_503 = backoff.on_exception(backoff.expo, exceptions.ServiceUnavailable) - backoff_503(add_cluster)(PROJECT, INSTANCE_ID, NEW_CLUSTER_ID) + secondary_zones = tuple(z for z in FALLBACK_ZONES if z != primary_zone) + _run_with_zone_fallback( + backoff_503(add_cluster), + secondary_zones, + capsys, + PROJECT, + INSTANCE_ID, + NEW_CLUSTER_ID, + ) out, _ = capsys.readouterr() assert f"Cluster created: {NEW_CLUSTER_ID}" in out diff --git a/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/instanceadmin_async.py b/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/instanceadmin_async.py index daf96c8c5bd9..0712ab3f7a1d 100644 --- a/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/instanceadmin_async.py +++ b/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/instanceadmin_async.py @@ -24,7 +24,9 @@ import asyncio -async def run_instance_operations(project_id, instance_id, cluster_id): +async def run_instance_operations( + project_id, instance_id, cluster_id, location_id="us-central1-f" +): # [START bigtable_async_check_instance_exists] from google.api_core.exceptions import NotFound @@ -50,7 +52,7 @@ async def run_instance_operations(project_id, instance_id, cluster_id): instance_path = client.instance_path(project_id, instance_id) cluster = bigtable_admin_v2.Cluster( - location=client.common_location_path(project_id, "us-central1-f"), + location=client.common_location_path(project_id, location_id), serve_nodes=1, default_storage_type=bigtable_admin_v2.StorageType.SSD, ) @@ -129,7 +131,7 @@ async def delete_instance(project_id, instance_id): # [END bigtable_async_delete_instance] -async def add_cluster(project_id, instance_id, cluster_id): +async def add_cluster(project_id, instance_id, cluster_id, location_id="us-central1-b"): # [START bigtable_async_create_cluster] from google.api_core.exceptions import NotFound @@ -145,7 +147,7 @@ async def add_cluster(project_id, instance_id, cluster_id): print(cluster.name.split("/")[-1]) new_cluster = bigtable_admin_v2.Cluster( - location=client.common_location_path(project_id, "us-central1-a"), + location=client.common_location_path(project_id, location_id), serve_nodes=1, default_storage_type=bigtable_admin_v2.StorageType.SSD, ) diff --git a/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/test_instanceadmin_async.py b/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/test_instanceadmin_async.py index 3ec71138d30d..d4965cf08390 100644 --- a/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/test_instanceadmin_async.py +++ b/packages/google-cloud-bigtable/samples/admin_client_async/instanceadmin/test_instanceadmin_async.py @@ -29,19 +29,60 @@ INSTANCE_ID = f"instance-admin-async-{str(uuid.uuid4())[:8]}" CLUSTER_ID = f"cluster-admin-async-{str(uuid.uuid4())[:8]}" NEW_CLUSTER_ID = f"cluster-add-async-{str(uuid.uuid4())[:8]}" +FALLBACK_ZONES = ( + "us-central1-f", + "us-central1-b", + "us-central1-c", + "us-east1-b", + "us-west1-b", +) + + +def _is_zone_stockout(exc): + message = str(exc).lower() + return "sold out" in message or "not enough servers" in message + + +async def _run_with_zone_fallback(fn, zones, capsys, *args): + last_exc = None + for zone in zones: + capsys.readouterr() + try: + await fn(*args, location_id=zone) + return zone + except exceptions.GoogleAPICallError as exc: + if not _is_zone_stockout(exc): + raise + last_exc = exc + raise last_exc @pytest.mark.asyncio async def test_instance_operations(capsys): try: - await run_instance_operations(PROJECT, INSTANCE_ID, CLUSTER_ID) + primary_zone = await _run_with_zone_fallback( + run_instance_operations, + FALLBACK_ZONES, + capsys, + PROJECT, + INSTANCE_ID, + CLUSTER_ID, + ) out, _ = capsys.readouterr() assert f"Created instance: {INSTANCE_ID}" in out assert "Listing instances:" in out assert "Listing clusters..." in out backoff_503 = backoff.on_exception(backoff.expo, exceptions.ServiceUnavailable) - await backoff_503(add_cluster)(PROJECT, INSTANCE_ID, NEW_CLUSTER_ID) + secondary_zones = tuple(z for z in FALLBACK_ZONES if z != primary_zone) + await _run_with_zone_fallback( + backoff_503(add_cluster), + secondary_zones, + capsys, + PROJECT, + INSTANCE_ID, + NEW_CLUSTER_ID, + ) out, _ = capsys.readouterr() assert f"Cluster created: {NEW_CLUSTER_ID}" in out diff --git a/packages/google-cloud-bigtable/samples/conftest.py b/packages/google-cloud-bigtable/samples/conftest.py index 4c23415ab7b4..d43dff2e3b71 100644 --- a/packages/google-cloud-bigtable/samples/conftest.py +++ b/packages/google-cloud-bigtable/samples/conftest.py @@ -76,6 +76,11 @@ def pytest_configure(config) -> None: ): return + # Under pytest-xdist, only the controller provisions and tears down the + # shared sample test instance; workers inherit BIGTABLE_INSTANCE via os.environ. + if hasattr(config, "workerinput"): + return + project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") if not project_id: raise pytest.UsageError( @@ -106,23 +111,45 @@ def pytest_configure(config) -> None: signal.signal(signal.SIGTERM, _sigterm_handler) print(f"\nCreating temporary sample test instance: {instance_name}") + fallback_zones = ( + "us-central1-b", + "us-central1-c", + "us-central1-f", + "us-east1-b", + "us-west1-b", + ) try: - operation = client.create_instance( - parent=project_path, - instance_id=instance_id, - instance=bigtable_admin.Instance( - display_name=instance_id, - type_=bigtable_admin.Instance.Type.PRODUCTION, - ), - clusters={ - f"{instance_id}-c1": bigtable_admin.Cluster( - location=client.common_location_path(project_id, "us-central1-b"), - serve_nodes=1, - default_storage_type=bigtable_admin.StorageType.SSD, + last_exc = None + for zone in fallback_zones: + try: + operation = client.create_instance( + parent=project_path, + instance_id=instance_id, + instance=bigtable_admin.Instance( + display_name=instance_id, + type_=bigtable_admin.Instance.Type.PRODUCTION, + ), + clusters={ + f"{instance_id}-c1": bigtable_admin.Cluster( + location=client.common_location_path(project_id, zone), + serve_nodes=1, + default_storage_type=bigtable_admin.StorageType.SSD, + ) + }, ) - }, - ) - operation.result(timeout=240) + operation.result(timeout=240) + last_exc = None + break + except exceptions.GoogleAPICallError as exc: + message = str(exc).lower() + if "sold out" not in message and "not enough servers" not in message: + raise + print( + f"Zone {zone} temporarily sold out for {instance_id}, trying next zone..." + ) + last_exc = exc + if last_exc is not None: + raise last_exc os.environ["BIGTABLE_INSTANCE"] = instance_id print(f"Created temporary sample test instance: {instance_id}") except Exception: diff --git a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py index 53f1bfbb1906..e64ff46200ff 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py @@ -36,7 +36,9 @@ from google.cloud.bigtable import enums -def run_instance_operations(project_id, instance_id, cluster_id): +def run_instance_operations( + project_id, instance_id, cluster_id, location_id="us-central1-f" +): """Check Instance exists. Creates a Production instance with default Cluster. List instances in a project. @@ -49,7 +51,6 @@ def run_instance_operations(project_id, instance_id, cluster_id): :param instance_id: Instance of the client. """ client = bigtable.Client(project=project_id, admin=True) - location_id = "us-central1-f" serve_nodes = 1 storage_type = enums.StorageType.SSD labels = {"prod-label": "prod-label"} @@ -121,7 +122,7 @@ def delete_instance(project_id, instance_id): # [END bigtable_delete_instance_legacy] -def add_cluster(project_id, instance_id, cluster_id): +def add_cluster(project_id, instance_id, cluster_id, location_id="us-central1-b"): """Add Cluster :type project_id: str @@ -136,7 +137,6 @@ def add_cluster(project_id, instance_id, cluster_id): client = bigtable.Client(project=project_id, admin=True) instance = client.instance(instance_id) - location_id = "us-central1-a" serve_nodes = 1 storage_type = enums.StorageType.SSD diff --git a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/test_instanceadmin.py b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/test_instanceadmin.py index 669280967733..b46c862f6cd7 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/test_instanceadmin.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/test_instanceadmin.py @@ -33,6 +33,32 @@ INSTANCE = INSTANCE_ID_FORMAT.format(random.randrange(ID_RANGE), int(time.time())) CLUSTER1 = CLUSTER_ID_FORMAT.format(random.randrange(ID_RANGE)) CLUSTER2 = CLUSTER_ID_FORMAT.format(random.randrange(ID_RANGE)) +FALLBACK_ZONES = ( + "us-central1-f", + "us-central1-b", + "us-central1-c", + "us-east1-b", + "us-west1-b", +) + + +def _is_zone_stockout(exc): + message = str(exc).lower() + return "sold out" in message or "not enough servers" in message + + +def _run_with_zone_fallback(fn, zones, capsys, *args): + last_exc = None + for zone in zones: + capsys.readouterr() + try: + fn(*args, location_id=zone) + return zone + except exceptions.GoogleAPICallError as exc: + if not _is_zone_stockout(exc): + raise + last_exc = exc + raise last_exc @pytest.fixture(scope="module", autouse=True) @@ -73,7 +99,14 @@ def disposal(instance): def test_run_instance_operations(capsys, dispose_of): dispose_of(INSTANCE) - instanceadmin.run_instance_operations(PROJECT, INSTANCE, CLUSTER1) + primary_zone = _run_with_zone_fallback( + instanceadmin.run_instance_operations, + FALLBACK_ZONES, + capsys, + PROJECT, + INSTANCE, + CLUSTER1, + ) out = capsys.readouterr().out assert f"Instance {INSTANCE} does not exist." in out assert "Creating an instance" in out @@ -85,7 +118,9 @@ def test_run_instance_operations(capsys, dispose_of): assert "Listing clusters..." in out assert f"\n{CLUSTER1}\n" in out - instanceadmin.run_instance_operations(PROJECT, INSTANCE, CLUSTER1) + instanceadmin.run_instance_operations( + PROJECT, INSTANCE, CLUSTER1, location_id=primary_zone + ) out = capsys.readouterr().out assert f"Instance {INSTANCE} already exists." in out assert "Listing instances" in out @@ -110,7 +145,14 @@ def _set_up_instance(): assert f"Instance {INSTANCE} does not exist" in out # Ok, create it then - instanceadmin.run_instance_operations(PROJECT, INSTANCE, CLUSTER1) + _run_with_zone_fallback( + instanceadmin.run_instance_operations, + FALLBACK_ZONES, + capsys, + PROJECT, + INSTANCE, + CLUSTER1, + ) capsys.readouterr() # throw away output _set_up_instance() @@ -135,17 +177,33 @@ def _set_up_instance(): assert f"Instance {INSTANCE} does not exist" in out # Get the instance created - instanceadmin.run_instance_operations(PROJECT, INSTANCE, CLUSTER1) + primary_zone = _run_with_zone_fallback( + instanceadmin.run_instance_operations, + FALLBACK_ZONES, + capsys, + PROJECT, + INSTANCE, + CLUSTER1, + ) capsys.readouterr() # throw away output + return primary_zone - _set_up_instance() + primary_zone = _set_up_instance() # Add a cluster to that instance # Avoid failing for "instance is currently being changed" by # applying an exponential backoff backoff_503 = backoff.on_exception(backoff.expo, exceptions.ServiceUnavailable) - backoff_503(instanceadmin.add_cluster)(PROJECT, INSTANCE, CLUSTER2) + secondary_zones = tuple(z for z in FALLBACK_ZONES if z != primary_zone) + secondary_zone = _run_with_zone_fallback( + backoff_503(instanceadmin.add_cluster), + secondary_zones, + capsys, + PROJECT, + INSTANCE, + CLUSTER2, + ) out = capsys.readouterr().out assert f"Adding cluster to instance {INSTANCE}" in out assert "Listing clusters..." in out @@ -153,7 +211,7 @@ def _set_up_instance(): assert f"Cluster created: {CLUSTER2}" in out # Try to add the same cluster again, won't work - instanceadmin.add_cluster(PROJECT, INSTANCE, CLUSTER2) + instanceadmin.add_cluster(PROJECT, INSTANCE, CLUSTER2, location_id=secondary_zone) out = capsys.readouterr().out assert "Listing clusters..." in out assert f"\n{CLUSTER1}\n" in out diff --git a/packages/google-cloud-bigtable/samples/utils.py b/packages/google-cloud-bigtable/samples/utils.py index a28eb58d80ed..23b0c3f2feec 100644 --- a/packages/google-cloud-bigtable/samples/utils.py +++ b/packages/google-cloud-bigtable/samples/utils.py @@ -14,11 +14,14 @@ Provides helper logic used across samples """ +import inspect + from google.api_core import exceptions from google.api_core.retry import Retry, if_exception_type from google.cloud import bigtable, bigtable_admin_v2 from google.cloud.bigtable.column_family import ColumnFamily +from google.cloud.bigtable.data import BigtableDataClient, ReadRowsQuery from google.cloud.bigtable_admin_v2.types import ColumnFamily as ColumnFamily_pb delete_retry = Retry( @@ -93,7 +96,7 @@ def create_table(project, instance_id, table_id, column_families={}): @Retry( - on_error=if_exception_type( + predicate=if_exception_type( exceptions.PreconditionFailed, exceptions.FailedPrecondition, exceptions.NotFound, @@ -109,13 +112,28 @@ def wait_for_table(table): """ if hasattr(table, "exists"): if not table.exists(): - raise exceptions.NotFound - else: - table_path = table.table_name if hasattr(table, "table_name") else str(table) - admin_client = bigtable_admin_v2.BigtableTableAdminClient() - admin_client.get_table( - request={ - "name": table_path, - "view": bigtable_admin_v2.Table.View.NAME_ONLY, - } - ) + raise exceptions.NotFound("Table does not exist yet") + return + + # Data-client Table / TableAsync instances and raw table_path strings (e.g. + # from create_table) do not expose .exists(), so verify via admin + data plane. + table_path = table.table_name if hasattr(table, "table_name") else str(table) + admin_client = bigtable_admin_v2.BigtableTableAdminClient() + admin_client.get_table( + request={ + "name": table_path, + "view": bigtable_admin_v2.Table.View.NAME_ONLY, + } + ) + + # Probe the data plane as well: get_table() can succeed while mutations + # still return FailedPrecondition ("table ... is still being created"). + if hasattr(table, "read_rows") and not inspect.iscoroutinefunction(table.read_rows): + table.read_rows(ReadRowsQuery(limit=1)) + return + + parts = table_path.split("/") + if len(parts) == 6 and parts[0] == "projects": + with BigtableDataClient(project=parts[1]) as data_client: + with data_client.get_table(parts[3], parts[5]) as data_table: + data_table.read_rows(ReadRowsQuery(limit=1)) diff --git a/packages/google-cloud-bigtable/tests/system/conftest.py b/packages/google-cloud-bigtable/tests/system/conftest.py index 7a7b315ec542..543f603404a0 100644 --- a/packages/google-cloud-bigtable/tests/system/conftest.py +++ b/packages/google-cloud-bigtable/tests/system/conftest.py @@ -20,3 +20,31 @@ script_path = os.path.dirname(os.path.realpath(__file__)) sys.path.append(script_path) + + +def pytest_configure(config) -> None: + """Clean up leaked test instances in the controller before workers start.""" + from google.cloud.environment_vars import BIGTABLE_EMULATOR + + if ( + os.getenv(BIGTABLE_EMULATOR) + or getattr(config.option, "help", False) + or getattr(config.option, "collectonly", False) + or hasattr(config, "workerinput") + ): + return + + project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") + if not project_id: + try: + import google.auth + + _, project_id = google.auth.default() + except Exception: + return + + if project_id: + from utils import clear_stale_instances + + # Purge leaked test instances older than 1 hour before allocating nodes. + clear_stale_instances(project_id, older_than_days=1 / 24, max_deletions=100) diff --git a/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py b/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py index 34f6ccf3c2a9..f4e3be1d5820 100644 --- a/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py +++ b/packages/google-cloud-bigtable/tests/system/data/test_metrics_async.py @@ -91,16 +91,52 @@ def clear(self): self._exc_list.clear() self.fail_mid_stream = False + def _make_deadline_exceeded(self): + if CrossSync.is_async: + return AioRpcError(StatusCode.DEADLINE_EXCEEDED, Metadata(), Metadata()) + else: + exc = RpcError(StatusCode.DEADLINE_EXCEEDED) + exc.trailing_metadata = lambda: [] + exc.initial_metadata = lambda: [] + exc.code = lambda: StatusCode.DEADLINE_EXCEEDED + exc.details = lambda: "Deadline Exceeded" + + def _result(): + raise exc + + exc.result = _result + return exc + @CrossSync.convert async def intercept_unary_unary(self, continuation, client_call_details, request): if self._exc_list: raise self._exc_list.pop(0) + # Deterministic client-side timeout for *_failure_timeout tests: + # When the CI worker and Bigtable cluster sit in the same GCP region + # (e.g. Cloud Build in us-central1 with <1ms RTT), tiny 1–5ms timeouts + # race against Bigtable returning server headers (setting cluster_id) on + # one hand, and expiring inside Python's local shard-dispatch loop before + # all shards start on the other. Using a 50ms timeout gives Python ample + # headroom to start all sub-requests under xdist CPU load, while sleeping + # past the deadline here guarantees DEADLINE_EXCEEDED with no server headers. + if ( + client_call_details.timeout is not None + and client_call_details.timeout <= 0.1 + ): + await CrossSync.sleep(client_call_details.timeout + 0.01) + raise self._make_deadline_exceeded() return await continuation(client_call_details, request) @CrossSync.convert async def intercept_unary_stream(self, continuation, client_call_details, request): if not self.fail_mid_stream and self._exc_list: raise self._exc_list.pop(0) + if ( + client_call_details.timeout is not None + and client_call_details.timeout <= 0.1 + ): + await CrossSync.sleep(client_call_details.timeout + 0.01) + raise self._make_deadline_exceeded() response = await continuation(client_call_details, request) @@ -124,6 +160,10 @@ async def __anext__(self): if not self._raised: self._raised = True if self._exc: + if callable(getattr(self._call, "cancel", None)): + self._call.cancel() + elif callable(getattr(self._call, "close", None)): + self._call.close() raise self._exc return await self._call.__anext__() @@ -329,7 +369,7 @@ async def test_read_rows_failure_timeout(self, table, temp_rows, handler): await temp_rows.add_row(b"row_key_1") handler.clear() with pytest.raises(GoogleAPICallError): - await table.read_rows(ReadRowsQuery(), operation_timeout=0.001) + await table.read_rows(ReadRowsQuery(), operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -521,7 +561,7 @@ async def test_read_rows_stream_failure_timeout(self, table, temp_rows, handler) await temp_rows.add_row(b"row_key_1") handler.clear() generator = await table.read_rows_stream( - ReadRowsQuery(), operation_timeout=0.001 + ReadRowsQuery(), operation_timeout=0.05 ) with pytest.raises(GoogleAPICallError): [_ async for _ in generator] @@ -748,7 +788,7 @@ async def test_read_row_failure_timeout(self, table, temp_rows, handler): await temp_rows.add_row(b"row_key_1") handler.clear() with pytest.raises(GoogleAPICallError): - await table.read_row(b"row_key_1", operation_timeout=0.001) + await table.read_row(b"row_key_1", operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -917,7 +957,7 @@ async def test_read_rows_sharded_failure_timeout(self, table, temp_rows, handler query2 = ReadRowsQuery(row_keys=[b"b"]) handler.clear() with pytest.raises(ShardedReadRowsExceptionGroup) as e: - await table.read_rows_sharded([query1, query2], operation_timeout=0.005) + await table.read_rows_sharded([query1, query2], operation_timeout=0.05) assert len(e.value.exceptions) == 2 for sub_exc in e.value.exceptions: assert isinstance(sub_exc.__cause__, DeadlineExceeded) @@ -1159,7 +1199,7 @@ async def test_bulk_mutate_rows_failure_timeout(self, table, temp_rows, handler) handler.clear() with pytest.raises(MutationsExceptionGroup): - await table.bulk_mutate_rows([entry], operation_timeout=0.001) + await table.bulk_mutate_rows([entry], operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -1381,9 +1421,7 @@ async def test_mutate_rows_batcher_failure_timeout(self, table, temp_rows, handl entry = RowMutationEntry(row_key, [mutation]) with pytest.raises(MutationsExceptionGroup): - async with table.mutations_batcher( - batch_operation_timeout=0.001 - ) as batcher: + async with table.mutations_batcher(batch_operation_timeout=0.05) as batcher: await batcher.append(entry) # validate counts assert len(handler.completed_operations) == 1 @@ -1559,7 +1597,7 @@ async def test_mutate_row_failure_timeout(self, table, temp_rows, handler): mutation = SetCell(TEST_FAMILY, b"q", b"v") with pytest.raises(GoogleAPICallError): - await table.mutate_row(row_key, [mutation], operation_timeout=0.001) + await table.mutate_row(row_key, [mutation], operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -1801,7 +1839,7 @@ async def test_sample_row_keys_failure_timeout(self, table, handler): No grpc headers expected """ with pytest.raises(GoogleAPICallError): - await table.sample_row_keys(operation_timeout=0.001) + await table.sample_row_keys(operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -1967,7 +2005,7 @@ async def test_read_modify_write_failure_timeout(self, table, temp_rows, handler await temp_rows.add_row(row_key, value=0, family=family, qualifier=qualifier) rule = IncrementRule(family, qualifier, 1) with pytest.raises(GoogleAPICallError): - await table.read_modify_write_row(row_key, rule, operation_timeout=0.001) + await table.read_modify_write_row(row_key, rule, operation_timeout=0.05) # validate counts assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -2158,7 +2196,7 @@ async def test_check_and_mutate_row_failure_timeout( row_key, predicate=ValueRangeFilter(0, 2), true_case_mutations=true_mutation, - operation_timeout=0.001, + operation_timeout=0.05, ) # validate counts assert len(handler.completed_operations) == 1 diff --git a/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py b/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py index 7daf8ba2d03c..d1d2cfe531fe 100644 --- a/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py +++ b/packages/google-cloud-bigtable/tests/system/data/test_metrics_autogen.py @@ -81,14 +81,39 @@ def clear(self): self._exc_list.clear() self.fail_mid_stream = False + def _make_deadline_exceeded(self): + exc = RpcError(StatusCode.DEADLINE_EXCEEDED) + exc.trailing_metadata = lambda: [] + exc.initial_metadata = lambda: [] + exc.code = lambda: StatusCode.DEADLINE_EXCEEDED + exc.details = lambda: "Deadline Exceeded" + + def _result(): + raise exc + + exc.result = _result + return exc + def intercept_unary_unary(self, continuation, client_call_details, request): if self._exc_list: raise self._exc_list.pop(0) + if ( + client_call_details.timeout is not None + and client_call_details.timeout <= 0.1 + ): + CrossSync._Sync_Impl.sleep(client_call_details.timeout + 0.01) + raise self._make_deadline_exceeded() return continuation(client_call_details, request) def intercept_unary_stream(self, continuation, client_call_details, request): if not self.fail_mid_stream and self._exc_list: raise self._exc_list.pop(0) + if ( + client_call_details.timeout is not None + and client_call_details.timeout <= 0.1 + ): + CrossSync._Sync_Impl.sleep(client_call_details.timeout + 0.01) + raise self._make_deadline_exceeded() response = continuation(client_call_details, request) if self.fail_mid_stream and self._exc_list: exc = self._exc_list.pop(0) @@ -106,6 +131,10 @@ def __next__(self): if not self._raised: self._raised = True if self._exc: + if callable(getattr(self._call, "cancel", None)): + self._call.cancel() + elif callable(getattr(self._call, "close", None)): + self._call.close() raise self._exc return self._call.__next__() @@ -276,7 +305,7 @@ def test_read_rows_failure_timeout(self, table, temp_rows, handler): temp_rows.add_row(b"row_key_1") handler.clear() with pytest.raises(GoogleAPICallError): - table.read_rows(ReadRowsQuery(), operation_timeout=0.001) + table.read_rows(ReadRowsQuery(), operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -435,7 +464,7 @@ def test_read_rows_stream_failure_timeout(self, table, temp_rows, handler): No grpc headers expected""" temp_rows.add_row(b"row_key_1") handler.clear() - generator = table.read_rows_stream(ReadRowsQuery(), operation_timeout=0.001) + generator = table.read_rows_stream(ReadRowsQuery(), operation_timeout=0.05) with pytest.raises(GoogleAPICallError): [_ for _ in generator] assert len(handler.completed_operations) == 1 @@ -624,7 +653,7 @@ def test_read_row_failure_timeout(self, table, temp_rows, handler): temp_rows.add_row(b"row_key_1") handler.clear() with pytest.raises(GoogleAPICallError): - table.read_row(b"row_key_1", operation_timeout=0.001) + table.read_row(b"row_key_1", operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -770,7 +799,7 @@ def test_read_rows_sharded_failure_timeout(self, table, temp_rows, handler): query2 = ReadRowsQuery(row_keys=[b"b"]) handler.clear() with pytest.raises(ShardedReadRowsExceptionGroup) as e: - table.read_rows_sharded([query1, query2], operation_timeout=0.005) + table.read_rows_sharded([query1, query2], operation_timeout=0.05) assert len(e.value.exceptions) == 2 for sub_exc in e.value.exceptions: assert isinstance(sub_exc.__cause__, DeadlineExceeded) @@ -971,7 +1000,7 @@ def test_bulk_mutate_rows_failure_timeout(self, table, temp_rows, handler): entry = RowMutationEntry(row_key, [mutation]) handler.clear() with pytest.raises(MutationsExceptionGroup): - table.bulk_mutate_rows([entry], operation_timeout=0.001) + table.bulk_mutate_rows([entry], operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -1155,7 +1184,7 @@ def test_mutate_rows_batcher_failure_timeout(self, table, temp_rows, handler): mutation = SetCell(TEST_FAMILY, b"q", b"v") entry = RowMutationEntry(row_key, [mutation]) with pytest.raises(MutationsExceptionGroup): - with table.mutations_batcher(batch_operation_timeout=0.001) as batcher: + with table.mutations_batcher(batch_operation_timeout=0.05) as batcher: batcher.append(entry) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 @@ -1302,7 +1331,7 @@ def test_mutate_row_failure_timeout(self, table, temp_rows, handler): row_key = b"row_key_1" mutation = SetCell(TEST_FAMILY, b"q", b"v") with pytest.raises(GoogleAPICallError): - table.mutate_row(row_key, [mutation], operation_timeout=0.001) + table.mutate_row(row_key, [mutation], operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -1471,7 +1500,7 @@ def test_sample_row_keys_failure_timeout(self, table, handler): No grpc headers expected""" with pytest.raises(GoogleAPICallError): - table.sample_row_keys(operation_timeout=0.001) + table.sample_row_keys(operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -1567,7 +1596,7 @@ def test_read_modify_write_failure_timeout(self, table, temp_rows, handler): temp_rows.add_row(row_key, value=0, family=family, qualifier=qualifier) rule = IncrementRule(family, qualifier, 1) with pytest.raises(GoogleAPICallError): - table.read_modify_write_row(row_key, rule, operation_timeout=0.001) + table.read_modify_write_row(row_key, rule, operation_timeout=0.05) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 operation = handler.completed_operations[0] @@ -1680,7 +1709,7 @@ def test_check_and_mutate_row_failure_timeout(self, table, temp_rows, handler): row_key, predicate=ValueRangeFilter(0, 2), true_case_mutations=true_mutation, - operation_timeout=0.001, + operation_timeout=0.05, ) assert len(handler.completed_operations) == 1 assert len(handler.completed_attempts) == 1 diff --git a/packages/google-cloud-bigtable/tests/system/utils.py b/packages/google-cloud-bigtable/tests/system/utils.py index fe3f3a2b865f..2824d4b247df 100644 --- a/packages/google-cloud-bigtable/tests/system/utils.py +++ b/packages/google-cloud-bigtable/tests/system/utils.py @@ -27,15 +27,23 @@ def clear_stale_instances( prefix: Union[str, Tuple[str, ...]] = ( "python-bigtable-tests", "g-c-p", + "ndef-", + "new-", + "dif-", + "dif-cmek-", + "ndif-", "admin-overlay-instance", "bt-samples-", + "metric-scale-test-", + "instanceadmin-", + "instance-admin-", ), - older_than_days: int = 1, - max_deletions: int = 5, + older_than_days: float = 1, + max_deletions: int = 50, ): """ Synchronously deletes any instances in the given project that are older - than older_than_days and whose name or display name matches the given prefix. + than older_than_days and whose name, display name, or test label matches. """ if os.getenv(BIGTABLE_EMULATOR): return @@ -57,11 +65,12 @@ def clear_stale_instances( break for instance in response.instances: - # Check if instance matches the prefix + # Check if instance matches the prefix or python-system test label display_name_matches = instance.display_name.startswith(prefix) name_matches = instance.name.split("/")[-1].startswith(prefix) + label_matches = "python-system" in getattr(instance, "labels", {}) - if display_name_matches or name_matches: + if display_name_matches or name_matches or label_matches: if instance.create_time: try: create_time = instance.create_time diff --git a/packages/google-cloud-bigtable/tests/system/v2_client/conftest.py b/packages/google-cloud-bigtable/tests/system/v2_client/conftest.py index 59e016f52af8..82ae486b2ad8 100644 --- a/packages/google-cloud-bigtable/tests/system/v2_client/conftest.py +++ b/packages/google-cloud-bigtable/tests/system/v2_client/conftest.py @@ -58,7 +58,7 @@ def location_id(): @pytest.fixture(scope="session") def serve_nodes(): - return 3 + return 1 @pytest.fixture(scope="session") diff --git a/packages/google-cloud-bigtable/tests/system/v2_client/test_data_api.py b/packages/google-cloud-bigtable/tests/system/v2_client/test_data_api.py index 3a45224d390c..59bfe1915d25 100644 --- a/packages/google-cloud-bigtable/tests/system/v2_client/test_data_api.py +++ b/packages/google-cloud-bigtable/tests/system/v2_client/test_data_api.py @@ -356,7 +356,9 @@ def test_table_mutate_rows_retries_timeout(data_table, rows_to_delete): rows_to_delete.append(row_2) row_2.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL1, timestamp=timestamp) - statuses = data_table.mutate_rows([row, row_2]) + statuses = data_table.mutate_rows( + [row, row_2], retry=DEFAULT_RETRY.with_deadline(2.0) + ) assert statuses[0].code == Code.OK assert statuses[1].code == Code.DEADLINE_EXCEEDED @@ -940,10 +942,10 @@ def test_table_read_rows_retry_timeout_mid_stream( # Shorten the deadline so the timeout test is shorter. data_table_read_rows_retry_tests._table_impl.default_read_rows_operation_timeout = ( - 10.0 + 2.0 ) rows_data = data_table_read_rows_retry_tests.read_rows( - retry=DEFAULT_RETRY_READ_ROWS.with_deadline(10.0) + retry=DEFAULT_RETRY_READ_ROWS.with_deadline(2.0) ) with pytest.raises(exceptions.RetryError): rows_data.consume_all() @@ -971,10 +973,10 @@ def test_table_read_rows_retry_timeout_establishing_stream( # Shorten the deadline so the timeout test is shorter. data_table_read_rows_retry_tests._table_impl.default_read_rows_operation_timeout = ( - 10.0 + 2.0 ) rows_data = data_table_read_rows_retry_tests.read_rows( - retry=DEFAULT_RETRY_READ_ROWS.with_deadline(10.0) + retry=DEFAULT_RETRY_READ_ROWS.with_deadline(2.0) ) with pytest.raises(exceptions.RetryError): rows_data.consume_all() diff --git a/packages/google-cloud-bigtable/tests/unit/data/_async/test_client.py b/packages/google-cloud-bigtable/tests/unit/data/_async/test_client.py index 3eaffac1e639..ff6126fc65bf 100644 --- a/packages/google-cloud-bigtable/tests/unit/data/_async/test_client.py +++ b/packages/google-cloud-bigtable/tests/unit/data/_async/test_client.py @@ -908,6 +908,12 @@ async def test__multiple_table_registration(self): async with self._make_client(project="project-id") as client: async with client.get_table("instance_1", "table_1") as table_1: + assert table_1._register_instance_future is not None + if not CrossSync.is_async: + # TODO(https://github.com/googleapis/google-cloud-python/issues/19158): + # Wait for background registration until client._register_instance + # records ownership synchronously. + table_1._register_instance_future.result() instance_1_path = client._gapic_client.instance_path( client.project, "instance_1" ) @@ -1052,7 +1058,12 @@ async def test_get_api_surface(self, method): ) else: raise TypeError(f"unexpected method: {method}") - await CrossSync.yield_to_event_loop() + # TODO(https://github.com/googleapis/google-cloud-python/issues/19158): + # Wait for the background registration task instead of yield_to_event_loop() + # (a no-op in sync mode) until client._register_instance records ownership + # synchronously. + assert surface._register_instance_future is not None + await CrossSync.wait([surface._register_instance_future]) if method != "get_materialized_view": assert surface.table_id == expected_table_id assert ( @@ -1167,7 +1178,12 @@ async def test_api_surface_context_manager(self, method): else: raise TypeError(f"unexpected method: {method}") async with fn() as table: - await CrossSync.yield_to_event_loop() + # TODO(https://github.com/googleapis/google-cloud-python/issues/19158): + # Wait for the background registration task instead of + # yield_to_event_loop() (a no-op in sync mode) until + # client._register_instance records ownership synchronously. + assert table._register_instance_future is not None + await CrossSync.wait([table._register_instance_future]) assert isinstance(table, surface_type) if method != "get_materialized_view": assert table.table_id == expected_table_id @@ -1407,6 +1423,10 @@ async def test_ctor(self, use_emulator): expected_mutate_rows_attempt_timeout = 0.75 client = self._make_client(use_emulator=use_emulator) assert not client._active_instances + if not use_emulator: + # Avoid opening a real PingAndWarm RPC so background registration + # completes deterministically in sync and async modes. + client._ping_and_warm_instances = CrossSync.Mock() table = self._get_target_class()( client, @@ -1420,7 +1440,12 @@ async def test_ctor(self, use_emulator): default_mutate_rows_operation_timeout=expected_mutate_rows_operation_timeout, default_mutate_rows_attempt_timeout=expected_mutate_rows_attempt_timeout, ) - await CrossSync.yield_to_event_loop() + # TODO(https://github.com/googleapis/google-cloud-python/issues/19158): + # Wait for the background registration task instead of yield_to_event_loop() + # (a no-op in sync mode) until client._register_instance records ownership + # synchronously. + assert table._register_instance_future is not None + await CrossSync.wait([table._register_instance_future]) assert table.table_id == expected_table_id assert table.instance_id == expected_instance_id assert ( @@ -1464,7 +1489,6 @@ async def test_ctor(self, use_emulator): == expected_mutate_rows_attempt_timeout ) if use_emulator: - await table._register_instance_future assert table._register_instance_future.done() assert not table._register_instance_future.cancelled() assert table._register_instance_future.exception() is None @@ -1853,6 +1877,10 @@ async def test_ctor(self, use_emulator): expected_mutate_rows_attempt_timeout = 0.75 client = self._make_client(use_emulator=use_emulator) assert not client._active_instances + if not use_emulator: + # Avoid opening a real PingAndWarm RPC so background registration + # completes deterministically in sync and async modes. + client._ping_and_warm_instances = CrossSync.Mock() view = self._get_target_class()( client, @@ -1867,7 +1895,12 @@ async def test_ctor(self, use_emulator): default_mutate_rows_operation_timeout=expected_mutate_rows_operation_timeout, default_mutate_rows_attempt_timeout=expected_mutate_rows_attempt_timeout, ) - await CrossSync.yield_to_event_loop() + # TODO(https://github.com/googleapis/google-cloud-python/issues/19158): + # Wait for the background registration task instead of yield_to_event_loop() + # (a no-op in sync mode) until client._register_instance records ownership + # synchronously. + assert view._register_instance_future is not None + await CrossSync.wait([view._register_instance_future]) assert view.table_id == expected_table_id assert ( view.table_name @@ -1915,7 +1948,6 @@ async def test_ctor(self, use_emulator): == expected_mutate_rows_attempt_timeout ) if use_emulator: - await view._register_instance_future assert view._register_instance_future.done() assert not view._register_instance_future.cancelled() assert view._register_instance_future.exception() is None @@ -1982,6 +2014,10 @@ async def test_ctor(self, use_emulator): expected_mutate_rows_attempt_timeout = 0.75 client = self._make_client(use_emulator=use_emulator) assert not client._active_instances + if not use_emulator: + # Avoid opening a real PingAndWarm RPC so background registration + # completes deterministically in sync and async modes. + client._ping_and_warm_instances = CrossSync.Mock() view = self._get_target_class()( client, @@ -1995,7 +2031,12 @@ async def test_ctor(self, use_emulator): default_mutate_rows_operation_timeout=expected_mutate_rows_operation_timeout, default_mutate_rows_attempt_timeout=expected_mutate_rows_attempt_timeout, ) - await CrossSync.yield_to_event_loop() + # TODO(https://github.com/googleapis/google-cloud-python/issues/19158): + # Wait for the background registration task instead of yield_to_event_loop() + # (a no-op in sync mode) until client._register_instance records ownership + # synchronously. + assert view._register_instance_future is not None + await CrossSync.wait([view._register_instance_future]) assert view.instance_id == expected_instance_id assert ( view.instance_name @@ -2038,7 +2079,6 @@ async def test_ctor(self, use_emulator): == expected_mutate_rows_attempt_timeout ) if use_emulator: - await view._register_instance_future assert view._register_instance_future.done() assert not view._register_instance_future.cancelled() assert view._register_instance_future.exception() is None diff --git a/packages/google-cloud-bigtable/tests/unit/data/_sync_autogen/test_client.py b/packages/google-cloud-bigtable/tests/unit/data/_sync_autogen/test_client.py index 76910fea8852..2654f8445a4f 100644 --- a/packages/google-cloud-bigtable/tests/unit/data/_sync_autogen/test_client.py +++ b/packages/google-cloud-bigtable/tests/unit/data/_sync_autogen/test_client.py @@ -759,6 +759,8 @@ def test__multiple_table_registration(self): with self._make_client(project="project-id") as client: with client.get_table("instance_1", "table_1") as table_1: + assert table_1._register_instance_future is not None + table_1._register_instance_future.result() instance_1_path = client._gapic_client.instance_path( client.project, "instance_1" ) @@ -883,7 +885,8 @@ def test_get_api_surface(self, method): ) else: raise TypeError(f"unexpected method: {method}") - CrossSync._Sync_Impl.yield_to_event_loop() + assert surface._register_instance_future is not None + CrossSync._Sync_Impl.wait([surface._register_instance_future]) if method != "get_materialized_view": assert surface.table_id == expected_table_id assert ( @@ -983,7 +986,8 @@ def test_api_surface_context_manager(self, method): else: raise TypeError(f"unexpected method: {method}") with fn() as table: - CrossSync._Sync_Impl.yield_to_event_loop() + assert table._register_instance_future is not None + CrossSync._Sync_Impl.wait([table._register_instance_future]) assert isinstance(table, surface_type) if method != "get_materialized_view": assert table.table_id == expected_table_id @@ -1176,6 +1180,8 @@ def test_ctor(self, use_emulator): expected_mutate_rows_attempt_timeout = 0.75 client = self._make_client(use_emulator=use_emulator) assert not client._active_instances + if not use_emulator: + client._ping_and_warm_instances = CrossSync._Sync_Impl.Mock() table = self._get_target_class()( client, expected_instance_id, @@ -1188,7 +1194,8 @@ def test_ctor(self, use_emulator): default_mutate_rows_operation_timeout=expected_mutate_rows_operation_timeout, default_mutate_rows_attempt_timeout=expected_mutate_rows_attempt_timeout, ) - CrossSync._Sync_Impl.yield_to_event_loop() + assert table._register_instance_future is not None + CrossSync._Sync_Impl.wait([table._register_instance_future]) assert table.table_id == expected_table_id assert table.instance_id == expected_instance_id assert ( @@ -1232,7 +1239,6 @@ def test_ctor(self, use_emulator): == expected_mutate_rows_attempt_timeout ) if use_emulator: - table._register_instance_future assert table._register_instance_future.done() assert not table._register_instance_future.cancelled() assert table._register_instance_future.exception() is None @@ -1531,6 +1537,8 @@ def test_ctor(self, use_emulator): expected_mutate_rows_attempt_timeout = 0.75 client = self._make_client(use_emulator=use_emulator) assert not client._active_instances + if not use_emulator: + client._ping_and_warm_instances = CrossSync._Sync_Impl.Mock() view = self._get_target_class()( client, expected_instance_id, @@ -1544,7 +1552,8 @@ def test_ctor(self, use_emulator): default_mutate_rows_operation_timeout=expected_mutate_rows_operation_timeout, default_mutate_rows_attempt_timeout=expected_mutate_rows_attempt_timeout, ) - CrossSync._Sync_Impl.yield_to_event_loop() + assert view._register_instance_future is not None + CrossSync._Sync_Impl.wait([view._register_instance_future]) assert view.table_id == expected_table_id assert ( view.table_name @@ -1592,7 +1601,6 @@ def test_ctor(self, use_emulator): == expected_mutate_rows_attempt_timeout ) if use_emulator: - view._register_instance_future assert view._register_instance_future.done() assert not view._register_instance_future.cancelled() assert view._register_instance_future.exception() is None @@ -1654,6 +1662,8 @@ def test_ctor(self, use_emulator): expected_mutate_rows_attempt_timeout = 0.75 client = self._make_client(use_emulator=use_emulator) assert not client._active_instances + if not use_emulator: + client._ping_and_warm_instances = CrossSync._Sync_Impl.Mock() view = self._get_target_class()( client, expected_instance_id, @@ -1666,7 +1676,8 @@ def test_ctor(self, use_emulator): default_mutate_rows_operation_timeout=expected_mutate_rows_operation_timeout, default_mutate_rows_attempt_timeout=expected_mutate_rows_attempt_timeout, ) - CrossSync._Sync_Impl.yield_to_event_loop() + assert view._register_instance_future is not None + CrossSync._Sync_Impl.wait([view._register_instance_future]) assert view.instance_id == expected_instance_id assert ( view.instance_name @@ -1709,7 +1720,6 @@ def test_ctor(self, use_emulator): == expected_mutate_rows_attempt_timeout ) if use_emulator: - view._register_instance_future assert view._register_instance_future.done() assert not view._register_instance_future.cancelled() assert view._register_instance_future.exception() is None @@ -2788,10 +2798,7 @@ def test_mutate_row_attempt_timeout_per_retry(self): ] mutation = mutations.DeleteAllFromRow() table.mutate_row( - "row_key", - mutation, - operation_timeout=10.0, - attempt_timeout=5.0, + "row_key", mutation, operation_timeout=10.0, attempt_timeout=5.0 ) mock_timeout_gen.assert_called_once_with(5.0, 10.0) assert mock_gapic.call_count == 2 diff --git a/packages/google-cloud-logging/tests/system/gapic/v2/test_system_logging_service_v2_v2.py b/packages/google-cloud-logging/tests/system/gapic/v2/test_system_logging_service_v2_v2.py index d08703d48548..6fe019ae02ef 100644 --- a/packages/google-cloud-logging/tests/system/gapic/v2/test_system_logging_service_v2_v2.py +++ b/packages/google-cloud-logging/tests/system/gapic/v2/test_system_logging_service_v2_v2.py @@ -31,10 +31,13 @@ def test_write_log_entries(self): # Guard: Skip if project or credentials are missing # This prevents the "prerelease_deps" job from failing - if not project_id or "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ: - pytest.skip( - "System tests require GOOGLE_CLOUD_PROJECT and GOOGLE_APPLICATION_CREDENTIALS" - ) + if not project_id: + pytest.skip("System tests require GOOGLE_CLOUD_PROJECT or PROJECT_ID") + + try: + google.auth.default() + except google.auth.exceptions.DefaultCredentialsError: + pytest.skip("System tests require Application Default Credentials") # Instantiate the client # Since we've confirmed the env var exists, this call is now safe diff --git a/packages/google-cloud-logging/tests/system/test_system.py b/packages/google-cloud-logging/tests/system/test_system.py index 52280aa35ee9..2b29bcbb255c 100644 --- a/packages/google-cloud-logging/tests/system/test_system.py +++ b/packages/google-cloud-logging/tests/system/test_system.py @@ -105,18 +105,23 @@ class Config(object): def setUpModule(): + import google.auth + import google.auth.exceptions + # Use GOOGLE_CLOUD_PROJECT or PROJECT_ID project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") or os.environ.get("PROJECT_ID") - # Check if we have credentials (either via file or environment) - # google.auth.default() will check GOOGLE_APPLICATION_CREDENTIALS - has_creds = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") is not None + if not project_id: + raise unittest.SkipTest( + "System tests skipped: GOOGLE_CLOUD_PROJECT or PROJECT_ID must be set." + ) - # Guard: Skip the entire module if the environment is not ready - if not project_id or not has_creds: + # Check if we have credentials (either via GOOGLE_APPLICATION_CREDENTIALS or ADC) + try: + google.auth.default() + except google.auth.exceptions.DefaultCredentialsError: raise unittest.SkipTest( - "System tests skipped: GOOGLE_CLOUD_PROJECT and " - "GOOGLE_APPLICATION_CREDENTIALS must be set." + "System tests skipped: Application Default Credentials must be available." ) # Explicitly pass the project to the client diff --git a/packages/google-cloud-storage/tests/conformance/_utils.py b/packages/google-cloud-storage/tests/conformance/_utils.py index 1857a46f34ef..96479f5776bf 100644 --- a/packages/google-cloud-storage/tests/conformance/_utils.py +++ b/packages/google-cloud-storage/tests/conformance/_utils.py @@ -1,8 +1,143 @@ +import os +import socket +import subprocess import time import traceback +from typing import Optional, Tuple import requests +DEFAULT_TESTBENCH_IMAGE = ( + "gcr.io/cloud-devrel-public-resources/storage-testbench:latest" +) + + +def get_container_docker_network() -> Optional[str]: + """Return a shared user-defined Docker network (e.g. 'cloudbuild') when + running inside a container attached to one, or None on a regular host. + """ + if not os.path.exists("/.dockerenv"): + return None + hostname = socket.gethostname() + try: + result = subprocess.run( + [ + "docker", + "inspect", + "-f", + "{{range $k, $v := .NetworkSettings.Networks}}{{$k}} {{end}}", + hostname, + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + if result.returncode == 0: + networks = [ + net + for net in result.stdout.strip().split() + if net and net not in ("bridge", "host", "none") + ] + if "cloudbuild" in networks: + return "cloudbuild" + if networks: + return networks[0] + # Fallback if container hostname was overridden from container ID. + cb_check = subprocess.run( + ["docker", "network", "inspect", "cloudbuild"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + check=False, + ) + if cb_check.returncode == 0: + return "cloudbuild" + except (OSError, subprocess.SubprocessError): + return None + return None + + +def stop_testbench(container_name: str) -> None: + """Force-remove a running storage-testbench container, ignoring errors.""" + subprocess.run( + ["docker", "rm", "-f", container_name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + + +def start_testbench( + container_name: str, + host_http_port: int, + host_grpc_port: Optional[int] = None, + image: str = DEFAULT_TESTBENCH_IMAGE, + ready_timeout: float = 30.0, +) -> Tuple[str, Optional[str]]: + """Pull and start storage-testbench, returning (http_endpoint, grpc_endpoint). + + When running inside a container on a Docker network (such as Cloud Build's + 'cloudbuild' network), attaches the testbench to that same network and + returns container-DNS endpoints; otherwise publishes host ports on localhost. + """ + subprocess.run(["docker", "pull", image], check=True) + stop_testbench(container_name) + + network = get_container_docker_network() + if network: + cmd = [ + "docker", + "run", + "--rm", + "-d", + "--name", + container_name, + "--network", + network, + image, + ] + http_endpoint = f"http://{container_name}:9000" + grpc_endpoint = ( + f"{container_name}:{host_grpc_port}" if host_grpc_port is not None else None + ) + else: + cmd = [ + "docker", + "run", + "--rm", + "-d", + "--name", + container_name, + "-p", + f"{host_http_port}:9000", + ] + if host_grpc_port is not None: + cmd.extend(["-p", f"{host_grpc_port}:{host_grpc_port}"]) + cmd.append(image) + http_endpoint = f"http://localhost:{host_http_port}" + grpc_endpoint = ( + f"localhost:{host_grpc_port}" if host_grpc_port is not None else None + ) + + subprocess.run(cmd, check=True) + + deadline = time.time() + ready_timeout + last_err = None + while time.time() < deadline: + try: + requests.get(f"{http_endpoint}/", timeout=2) + return http_endpoint, grpc_endpoint + except requests.exceptions.RequestException as exc: + last_err = exc + time.sleep(0.5) + + stop_testbench(container_name) + raise RuntimeError( + f"storage-testbench ({container_name}) did not become ready at " + f"{http_endpoint} within {ready_timeout}s: {last_err}" + ) + def start_grpc_server(grpc_endpoint, http_endpoint): """Starts the testbench gRPC server if it's not already running. diff --git a/packages/google-cloud-storage/tests/conformance/conftest.py b/packages/google-cloud-storage/tests/conformance/conftest.py index fb553ff1cbc3..a9bd31023323 100644 --- a/packages/google-cloud-storage/tests/conformance/conftest.py +++ b/packages/google-cloud-storage/tests/conformance/conftest.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import atexit import os import random import uuid +from typing import Optional import pytest from google.auth.credentials import AnonymousCredentials @@ -22,8 +24,55 @@ from google.cloud import storage +from ._utils import start_testbench, stop_testbench + """Environment variable or default host for Storage testbench emulator.""" _HOST = os.environ.get("STORAGE_EMULATOR_HOST", "http://localhost:9000") +_TESTBENCH_CONTAINER: Optional[str] = None + + +def _cleanup_testbench() -> None: + global _TESTBENCH_CONTAINER + container = _TESTBENCH_CONTAINER + if container: + _TESTBENCH_CONTAINER = None + stop_testbench(container) + + +def _running_retry_conformance(config) -> bool: + args = [str(arg) for arg in (config.args or [])] + if not args: + return True + return any("test_conformance" in arg or os.path.isdir(arg) for arg in args) + + +def pytest_configure(config) -> None: + """Start a single shared storage-testbench container for retry conformance tests. + + Under pytest-xdist, only the controller starts and stops the container; + workers inherit STORAGE_EMULATOR_HOST via os.environ. + """ + global _TESTBENCH_CONTAINER + if getattr(config.option, "help", False) or getattr( + config.option, "collectonly", False + ): + return + if hasattr(config, "workerinput"): + return + if os.environ.get("STORAGE_EMULATOR_HOST"): + return + if not _running_retry_conformance(config): + return + + container_name = f"storage-conformance-{uuid.uuid4().hex[:8]}" + http_endpoint, _ = start_testbench(container_name, host_http_port=9000) + _TESTBENCH_CONTAINER = container_name + atexit.register(_cleanup_testbench) + os.environ["STORAGE_EMULATOR_HOST"] = http_endpoint + + +def pytest_unconfigure(config) -> None: + _cleanup_testbench() """Emulated project information for the storage testbench.""" @@ -57,10 +106,11 @@ def _create_block(desired_kib): @pytest.fixture def client(): + host = os.environ.get("STORAGE_EMULATOR_HOST", _HOST) client = storage.Client( project=_CONF_TEST_PROJECT_ID, credentials=AnonymousCredentials(), - client_options={"api_endpoint": _HOST}, + client_options={"api_endpoint": host}, ) return client diff --git a/packages/google-cloud-storage/tests/conformance/test_bidi_reads.py b/packages/google-cloud-storage/tests/conformance/test_bidi_reads.py index 272da47bc034..3dace4d79c96 100644 --- a/packages/google-cloud-storage/tests/conformance/test_bidi_reads.py +++ b/packages/google-cloud-storage/tests/conformance/test_bidi_reads.py @@ -1,8 +1,5 @@ import io -import subprocess -import time import traceback -import urllib import uuid import pytest @@ -17,7 +14,7 @@ from google.cloud.storage.asyncio.async_multi_range_downloader import ( AsyncMultiRangeDownloader, ) -from tests.conformance._utils import start_grpc_server +from tests.conformance._utils import start_grpc_server, start_testbench, stop_testbench # --- Configuration --- @@ -25,45 +22,29 @@ TEST_BENCH_ENDPOINT = ( "http://localhost:9001" # 9000 in VM is taken by test_conformance.py ) -_PORT = urllib.parse.urlsplit(TEST_BENCH_ENDPOINT).port +_HTTP_PORT = 9001 _GRPC_PORT = 8888 PROJECT_NUMBER = "12345" # A dummy project number is fine for the testbench. GRPC_ENDPOINT = f"localhost:{_GRPC_PORT}" CONTENT_LENGTH = 1024 * 10 # 10 KB -_DEFAULT_IMAGE_NAME = "gcr.io/cloud-devrel-public-resources/storage-testbench" -_DEFAULT_IMAGE_TAG = "latest" -_DOCKER_IMAGE = f"{_DEFAULT_IMAGE_NAME}:{_DEFAULT_IMAGE_TAG}" -_PULL_CMD = ["docker", "pull", _DOCKER_IMAGE] -_RUN_CMD = [ - "docker", - "run", - "--name", - "bidi_reads_container", - "--rm", - "-d", - "-p", - f"{_PORT}:9000", - "-p", - f"{_GRPC_PORT}:{_GRPC_PORT}", - _DOCKER_IMAGE, -] -_DOCKER_STOP_CMD = [ - "docker", - "stop", - "bidi_reads_container", -] - @pytest.fixture(scope="module") def testbench(): - subprocess.run(_PULL_CMD) - proc = subprocess.Popen(_RUN_CMD) - time.sleep(10) - yield GRPC_ENDPOINT, TEST_BENCH_ENDPOINT - subprocess.run(_DOCKER_STOP_CMD) - proc.kill() + global TEST_BENCH_ENDPOINT, GRPC_ENDPOINT + container_name = f"bidi-reads-{uuid.uuid4().hex[:8]}" + http_endpoint, grpc_endpoint = start_testbench( + container_name, + host_http_port=_HTTP_PORT, + host_grpc_port=_GRPC_PORT, + ) + TEST_BENCH_ENDPOINT = http_endpoint + GRPC_ENDPOINT = grpc_endpoint + try: + yield GRPC_ENDPOINT, TEST_BENCH_ENDPOINT + finally: + stop_testbench(container_name) def _is_retriable(exc): diff --git a/packages/google-cloud-storage/tests/conformance/test_bidi_writes.py b/packages/google-cloud-storage/tests/conformance/test_bidi_writes.py index 5490a419ec84..aa423a97632c 100644 --- a/packages/google-cloud-storage/tests/conformance/test_bidi_writes.py +++ b/packages/google-cloud-storage/tests/conformance/test_bidi_writes.py @@ -1,6 +1,3 @@ -import subprocess -import time -import urllib import uuid import grpc @@ -15,51 +12,36 @@ AsyncAppendableObjectWriter, ) from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient -from tests.conformance._utils import start_grpc_server +from tests.conformance._utils import start_grpc_server, start_testbench, stop_testbench # --- Configuration --- TEST_BENCH_ENDPOINT = ( "http://localhost:9002" # 9000 in VM is taken by test_conformance.py, 9001 by reads ) -_PORT = urllib.parse.urlsplit(TEST_BENCH_ENDPOINT).port +_HTTP_PORT = 9002 _GRPC_PORT = 8888 PROJECT_NUMBER = "12345" # A dummy project number is fine for the testbench. GRPC_ENDPOINT = f"localhost:{_GRPC_PORT}" HTTP_ENDPOINT = TEST_BENCH_ENDPOINT -_DEFAULT_IMAGE_NAME = "gcr.io/cloud-devrel-public-resources/storage-testbench" -_DEFAULT_IMAGE_TAG = "latest" -_DOCKER_IMAGE = f"{_DEFAULT_IMAGE_NAME}:{_DEFAULT_IMAGE_TAG}" -_PULL_CMD = ["docker", "pull", _DOCKER_IMAGE] -_RUN_CMD = [ - "docker", - "run", - "--name", - "bidi_writes_container", - "--rm", - "-d", - "-p", - f"{_PORT}:9000", - "-p", - f"{_GRPC_PORT}:8888", - _DOCKER_IMAGE, -] -_DOCKER_STOP_CMD = [ - "docker", - "stop", - "bidi_writes_container", -] - @pytest.fixture(scope="module") def testbench(): - subprocess.run(_PULL_CMD) - proc = subprocess.Popen(_RUN_CMD) - time.sleep(10) - yield GRPC_ENDPOINT, HTTP_ENDPOINT - subprocess.run(_DOCKER_STOP_CMD) - proc.kill() + global TEST_BENCH_ENDPOINT, HTTP_ENDPOINT, GRPC_ENDPOINT + container_name = f"bidi-writes-{uuid.uuid4().hex[:8]}" + http_endpoint, grpc_endpoint = start_testbench( + container_name, + host_http_port=_HTTP_PORT, + host_grpc_port=_GRPC_PORT, + ) + TEST_BENCH_ENDPOINT = http_endpoint + HTTP_ENDPOINT = http_endpoint + GRPC_ENDPOINT = grpc_endpoint + try: + yield GRPC_ENDPOINT, HTTP_ENDPOINT + finally: + stop_testbench(container_name) CONTENT = b"A" * 1024 * 1024 * 10 # 10 MiB diff --git a/packages/google-cloud-storage/tests/conformance/test_conformance.py b/packages/google-cloud-storage/tests/conformance/test_conformance.py index 2263429acac4..fc1d662b28ed 100644 --- a/packages/google-cloud-storage/tests/conformance/test_conformance.py +++ b/packages/google-cloud-storage/tests/conformance/test_conformance.py @@ -17,10 +17,7 @@ import functools import logging import os -import subprocess import tempfile -import time -import urllib import uuid import pytest @@ -36,14 +33,6 @@ """Environment variable or default host for Storage testbench emulator.""" _HOST = os.environ.get("STORAGE_EMULATOR_HOST", "http://localhost:9000") -_PORT = urllib.parse.urlsplit(_HOST).port - -"""The storage testbench docker image info and commands.""" -_DEFAULT_IMAGE_NAME = "gcr.io/cloud-devrel-public-resources/storage-testbench" -_DEFAULT_IMAGE_TAG = "latest" -_DOCKER_IMAGE = f"{_DEFAULT_IMAGE_NAME}:{_DEFAULT_IMAGE_TAG}" -_PULL_CMD = ["docker", "pull", _DOCKER_IMAGE] -_RUN_CMD = ["docker", "run", "--rm", "-d", "-p", f"{_PORT}:9000", _DOCKER_IMAGE] _CONF_TEST_PROJECT_ID = "my-project-id" _CONF_TEST_SERVICE_ACCOUNT_EMAIL = ( @@ -922,6 +911,7 @@ def run_test_case( hmac_key, file_data, ): + host = host or os.environ.get("STORAGE_EMULATOR_HOST", _HOST) scenario = _CONFORMANCE_TESTS[scenario_id - 1] expect_success = scenario["expectSuccess"] precondition_provided = scenario["preconditionProvided"] @@ -979,29 +969,22 @@ def run_test_case( ### Run Conformance Tests for Retry Strategy ########################################################################################### ######################################################################################################################################## -# Pull storage-testbench docker image -subprocess.run(_PULL_CMD) -time.sleep(5) - -# Run docker image to start storage-testbench -with subprocess.Popen(_RUN_CMD) as proc: - # Run retry conformance tests - for scenario in _CONFORMANCE_TESTS: - id = scenario["id"] - methods = scenario["methods"] - cases = scenario["cases"] - for i, c in enumerate(cases): - for m in methods: - method_name = m["name"] - method_group = m["group"] if m.get("group", None) else m["name"] - if method_group not in method_mapping: - logging.info(f"No tests for operation {method_name}") - continue - - for lib_func in method_mapping[method_group]: - test_name = f"test-S{id}-{method_name}-{lib_func.__name__}-{i}" - globals()[test_name] = functools.partial( - run_test_case, id, m, c, lib_func, _HOST - ) - time.sleep(5) - proc.kill() +# Register retry conformance tests (storage-testbench lifecycle is managed +# once by conftest.pytest_configure / pytest_unconfigure). +for scenario in _CONFORMANCE_TESTS: + id = scenario["id"] + methods = scenario["methods"] + cases = scenario["cases"] + for i, c in enumerate(cases): + for m in methods: + method_name = m["name"] + method_group = m["group"] if m.get("group", None) else m["name"] + if method_group not in method_mapping: + logging.info(f"No tests for operation {method_name}") + continue + + for lib_func in method_mapping[method_group]: + test_name = f"test-S{id}-{method_name}-{lib_func.__name__}-{i}" + globals()[test_name] = functools.partial( + run_test_case, id, m, c, lib_func, None + ) diff --git a/packages/google-cloud-storage/tests/resumable_media/system/requests/conftest.py b/packages/google-cloud-storage/tests/resumable_media/system/requests/conftest.py index 67908795bded..8e3751db0833 100644 --- a/packages/google-cloud-storage/tests/resumable_media/system/requests/conftest.py +++ b/packages/google-cloud-storage/tests/resumable_media/system/requests/conftest.py @@ -13,6 +13,8 @@ # limitations under the License. """py.test fixtures to be shared across multiple system test modules.""" +import os + import google.auth # type: ignore import google.auth.transport.requests as tr_requests # type: ignore import pytest # type: ignore @@ -24,7 +26,12 @@ def ensure_bucket(transport): get_response = transport.get(utils.BUCKET_URL) if get_response.status_code == 404: credentials = transport.credentials - query_params = {"project": credentials.project_id} + project_id = ( + getattr(credentials, "project_id", None) + or os.environ.get("GOOGLE_CLOUD_PROJECT") + or os.environ.get("PROJECT_ID") + ) + query_params = {"project": project_id} payload = {"name": utils.BUCKET_NAME} post_response = transport.post( utils.BUCKET_POST_URL, params=query_params, json=payload diff --git a/packages/google-cloud-storage/tests/system/conftest.py b/packages/google-cloud-storage/tests/system/conftest.py index a74fd74f9652..827b1157367f 100644 --- a/packages/google-cloud-storage/tests/system/conftest.py +++ b/packages/google-cloud-storage/tests/system/conftest.py @@ -245,7 +245,7 @@ def signing_bucket_name(): @pytest.fixture(scope="session") -def signing_bucket(storage_client, signing_bucket_name): +def signing_bucket(storage_client, signing_bucket_name, service_account): bucket = storage_client.bucket(signing_bucket_name) # Create the bucket only if it doesn't yet exist. try: diff --git a/packages/google-cloud-storage/tests/system/test_bucket.py b/packages/google-cloud-storage/tests/system/test_bucket.py index ae194fe46793..39c65ae95237 100644 --- a/packages/google-cloud-storage/tests/system/test_bucket.py +++ b/packages/google-cloud-storage/tests/system/test_bucket.py @@ -456,7 +456,6 @@ def test_bucket_copy_blob_w_metageneration_match( payload = b"DEADBEEF" bucket_name = _helpers.unique_name("generation-match") bucket = storage_client.bucket(bucket_name) - bucket.requester_pays = True created = _helpers.retry_429_503(storage_client.create_bucket)(bucket) buckets_to_delete.append(created) assert created.name == bucket_name diff --git a/packages/google-resumable-media/tests/system/requests/conftest.py b/packages/google-resumable-media/tests/system/requests/conftest.py index 54ae10a2b5ca..7ea27e473755 100644 --- a/packages/google-resumable-media/tests/system/requests/conftest.py +++ b/packages/google-resumable-media/tests/system/requests/conftest.py @@ -13,6 +13,8 @@ # limitations under the License. """py.test fixtures to be shared across multiple system test modules.""" +import os + import google.auth # type: ignore import google.auth.transport.requests as tr_requests # type: ignore import pytest # type: ignore @@ -24,7 +26,12 @@ def ensure_bucket(transport): get_response = transport.get(utils.BUCKET_URL) if get_response.status_code == 404: credentials = transport.credentials - query_params = {"project": credentials.project_id} + project_id = ( + getattr(credentials, "project_id", None) + or os.environ.get("GOOGLE_CLOUD_PROJECT") + or os.environ.get("PROJECT_ID") + ) + query_params = {"project": project_id} payload = {"name": utils.BUCKET_NAME} post_response = transport.post( utils.BUCKET_POST_URL, params=query_params, json=payload diff --git a/packages/google-resumable-media/tests_async/system/requests/conftest.py b/packages/google-resumable-media/tests_async/system/requests/conftest.py index f9acc483223e..ba2395707374 100644 --- a/packages/google-resumable-media/tests_async/system/requests/conftest.py +++ b/packages/google-resumable-media/tests_async/system/requests/conftest.py @@ -13,6 +13,9 @@ # limitations under the License. """py.test fixtures to be shared across multiple system test modules.""" +import inspect +import os + import google.auth.transport._aiohttp_requests as tr_requests # type: ignore import pytest # type: ignore from google.auth._default_async import default_async # type: ignore @@ -24,7 +27,12 @@ async def ensure_bucket(transport): get_response = await transport.request("GET", utils.BUCKET_URL) if get_response.status == 404: credentials = transport.credentials - query_params = {"project": credentials.project_id} + project_id = ( + getattr(credentials, "project_id", None) + or os.environ.get("GOOGLE_CLOUD_PROJECT") + or os.environ.get("PROJECT_ID") + ) + query_params = {"project": project_id} payload = {"name": utils.BUCKET_NAME} post_response = await transport.request( "POST", utils.BUCKET_POST_URL, params=query_params, json=payload @@ -44,12 +52,22 @@ async def cleanup_bucket(transport): def _get_authorized_transport(): credentials, project_id = default_async(scopes=(utils.GCS_RW_SCOPE,)) + if not inspect.iscoroutinefunction(credentials.before_request): + import google.auth + import google.auth.transport.requests + from google.oauth2 import _credentials_async + + sync_creds, sync_project = google.auth.default(scopes=(utils.GCS_RW_SCOPE,)) + sync_creds.refresh(google.auth.transport.requests.Request()) + credentials = _credentials_async.Credentials(token=sync_creds.token) + credentials.project_id = ( + getattr(sync_creds, "project_id", None) or sync_project or project_id + ) return tr_requests.AuthorizedSession(credentials) @pytest.fixture(scope="module") async def authorized_transport(): - credentials, project_id = default_async(scopes=(utils.GCS_RW_SCOPE,)) yield _get_authorized_transport() diff --git a/packages/google-resumable-media/tests_async/system/requests/test_download.py b/packages/google-resumable-media/tests_async/system/requests/test_download.py index 5c5ac7d8582e..2602cfa6056f 100644 --- a/packages/google-resumable-media/tests_async/system/requests/test_download.py +++ b/packages/google-resumable-media/tests_async/system/requests/test_download.py @@ -23,7 +23,6 @@ import google.auth.transport._aiohttp_requests as tr_requests # type: ignore import multidict # type: ignore import pytest # type: ignore -from google.auth._default_async import default_async # type: ignore import google._async_resumable_media.requests as resumable_requests import google._async_resumable_media.requests.download as download_mod @@ -181,9 +180,8 @@ async def secret_file(authorized_transport, bucket): # Transport that returns corrupt data, so we can exercise checksum handling. @pytest.fixture(scope="module") -async def corrupting_transport(): - credentials, _ = default_async(scopes=(utils.GCS_RW_SCOPE,)) - yield CorruptingAuthorizedSession(credentials) +async def corrupting_transport(authorized_transport): + yield CorruptingAuthorizedSession(authorized_transport.credentials) @pytest.fixture(scope="module") diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py index 564625a92d73..264a63039864 100644 --- a/packages/pandas-gbq/tests/system/test_auth.py +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -11,7 +11,7 @@ from pandas_gbq import auth -IS_RUNNING_ON_CI = "KOKORO_BUILD_ID" in os.environ +IS_RUNNING_ON_CI = bool(os.environ.get("KOKORO_BUILD_ID") or os.environ.get("CI")) def mock_default_credentials(scopes=None, request=None):