diff --git a/dsub/_dsub_version.py b/dsub/_dsub_version.py index e116d5b..7389dfb 100644 --- a/dsub/_dsub_version.py +++ b/dsub/_dsub_version.py @@ -27,5 +27,5 @@ """ -DSUB_VERSION = '0.5.5' +DSUB_VERSION = '0.5.6' diff --git a/test/integration/e2e_accelerator.google-cls-v2.sh b/test/integration/e2e_accelerator.google-cls-v2.sh deleted file mode 100755 index 3c13a56..0000000 --- a/test/integration/e2e_accelerator.google-cls-v2.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -# Copyright 2020 Verily Life Sciences Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset - -# Basic test of using a small attached GPU. -# -# No input files. -# No output files. -# The stdout file is checked for expected output. - -readonly SCRIPT_DIR="$(dirname "${0}")" - -# Do standard test setup -source "${SCRIPT_DIR}/test_setup_e2e.sh" - -echo "Launching pipeline..." - -# Use the python:slim image just to demonstrate that no special image is needed. -# The necessary GPU libraries and files are mounted from the VM into the container. -run_dsub \ - --image 'python:slim' \ - --accelerator-type 'nvidia-tesla-p4' \ - --accelerator-count 1 \ - --command '\ - export LD_LIBRARY_PATH="/usr/local/nvidia/lib64" && \ - /usr/local/nvidia/bin/nvidia-smi' \ - --wait - -echo -echo "Checking output..." - -# Check the results -RESULT="$(gcloud storage cat "${STDOUT_LOG}")" -if ! echo "${RESULT}" | grep -qi "GPU Memory"; then - 1>&2 echo "GPU Memory not found in the dsub output!" - 1>&2 echo "${RESULT}" - exit 1 -fi - -echo -echo "Output file matches expected:" -echo "*****************************" -echo "${RESULT}" - echo "*****************************" -echo "SUCCESS" - diff --git a/test/integration/e2e_block_external_network.google-cls-v2.sh b/test/integration/e2e_block_external_network.google-cls-v2.sh deleted file mode 100755 index d3f2632..0000000 --- a/test/integration/e2e_block_external_network.google-cls-v2.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash - -# Copyright 2021 Verily Life Sciences Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset - -# Basic test of using the --block-external-network flag -# No input files. -# No output files. -# The stderr log file is checked for expected errors due to no network. - -readonly SCRIPT_DIR="$(dirname "${0}")" - -# Do standard test setup -source "${SCRIPT_DIR}/test_setup_e2e.sh" - -# The user script runs "gcloud storage ls", so it needs an image with gcloud. -# Read dsub's own constant rather than duplicating the image reference here; -# that keeps this test from going stale when the constant is updated, and lets -# DSUB_CLOUD_SDK_IMAGE steer the test too. -readonly CLOUD_SDK_IMAGE="$(python3 -c \ - 'from dsub.providers import google_utils; print(google_utils.CLOUD_SDK_IMAGE)')" -if [[ -z "${CLOUD_SDK_IMAGE}" ]]; then - 1>&2 echo "Could not read CLOUD_SDK_IMAGE from dsub.providers.google_utils." - exit 1 -fi -echo "Using image: ${CLOUD_SDK_IMAGE}" - -echo "Launching pipeline..." - -set +o errexit - -# script_block_external_network.sh sets CLOUDSDK_STORAGE_MAX_RETRIES=0 when -# running "gcloud storage ls" below. Otherwise, gcloud storage will retry -# due to the network error. -JOB_ID="$(run_dsub \ - --image "${CLOUD_SDK_IMAGE}" \ - --block-external-network \ - --script "${SCRIPT_DIR}/script_block_external_network.sh" \ - --retries 1 \ - --wait)" -if [[ $? -eq 0 ]]; then - 1>&2 echo "dsub did not report the failure as it should have." - exit 1 -fi -set -o errexit - -echo -echo "Checking stderr of both attempts..." - -# Check the results -readonly ATTEMPT_1_STDERR_LOG="$(dirname "${LOGGING}")/${TEST_NAME}.1-stderr.log" -readonly ATTEMPT_2_STDERR_LOG="$(dirname "${LOGGING}")/${TEST_NAME}.2-stderr.log" - -# A blocked network reaches the log as one of several messages depending on the -# gcloud version in the image: older releases report a handled urllib3 "Max -# retries exceeded", while current ones surface an unhandled "gcloud crashed -# (ConnectionError)" traceback. Assert on the invariant -- gcloud could not -# reach the GCS endpoint -- rather than on one release's prose, so that a -# reworded gcloud error does not read as a dsub regression. -readonly GCLOUD_NETWORK_ERROR_RE='max retries exceeded|connectionerror|connection refused|failed to establish a new connection|could not resolve|name or service not known|network is unreachable' -# curl reports its failure on a single line, so keep the host in the pattern. -# Otherwise a gcloud network error elsewhere in the log could satisfy this -# check while curl's own error is missing. -readonly CURL_NETWORK_ERROR_RE='(could not resolve host|failed to connect to|resolving timed out.*) *:? *google\.com' - -for STDERR_LOG_FILE in "${ATTEMPT_1_STDERR_LOG}" "${ATTEMPT_2_STDERR_LOG}" ; do - RESULT="$(gcloud storage cat "${STDERR_LOG_FILE}")" - if ! echo "${RESULT}" | grep -qi "storage.googleapis.com" \ - || ! echo "${RESULT}" | grep -qiE "${GCLOUD_NETWORK_ERROR_RE}"; then - 1>&2 echo "Network error from gcloud not found in the dsub stderr log!" - 1>&2 echo "${RESULT}" - exit 1 - fi - - if ! echo "${RESULT}" | grep -qiE "${CURL_NETWORK_ERROR_RE}"; then - 1>&2 echo "Network error from curl not found in the dsub stderr log!" - 1>&2 echo "${RESULT}" - exit 1 - fi -done - -echo -echo "Checking dstat output..." -ATTEMPT_1_DSTAT_OUTPUT=$(run_dstat --attempts 1 --status 'FAILURE' --full --jobs "${JOB_ID}" 2>&1); -ATTEMPT_2_DSTAT_OUTPUT=$(run_dstat --attempts 2 --status 'FAILURE' --full --jobs "${JOB_ID}" 2>&1); -for DSTAT_OUTPUT in "${ATTEMPT_1_DSTAT_OUTPUT}" "${ATTEMPT_1_DSTAT_OUTPUT}" ; do - if ! echo "${DSTAT_OUTPUT}" | grep -qi "block-external-network: true"; then - 1>&2 echo "block-external-network not found in dstat output!" - 1>&2 echo "${DSTAT_OUTPUT}" - exit 1 - fi -done - -echo -echo "stderr log contains the expected errors." -echo "dstat output contains the expected block-external-network flag." -echo "SUCCESS" - diff --git a/test/integration/e2e_io_mount_bucket.google-cls-v2.sh b/test/integration/e2e_io_mount_bucket.google-cls-v2.sh deleted file mode 100755 index 4340d9a..0000000 --- a/test/integration/e2e_io_mount_bucket.google-cls-v2.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset - -# Test gcsfuse abilities. -# -# This test is designed to verify that named GCS bucket (mount) -# command-line parameters work correctly. -# -# The actual operation performed here is to mount to a bucket containing a BAM -# and compute its md5, writing it to .bam.md5. - -readonly SCRIPT_DIR="$(dirname "${0}")" - -# Do standard test setup -source "${SCRIPT_DIR}/test_setup_e2e.sh" - -# Do io setup -source "${SCRIPT_DIR}/io_setup.sh" - -echo "Launching pipeline..." - -JOB_ID="$(io_setup::run_dsub_with_mount "${GENOMICS_PUBLIC_BUCKET}")" - -# Do validation -io_setup::check_output -io_setup::check_dstat "${JOB_ID}" false "${GENOMICS_PUBLIC_BUCKET}" diff --git a/test/integration/e2e_io_mount_bucket_requester_pays.google-cls-v2.sh b/test/integration/e2e_io_mount_bucket_requester_pays.google-cls-v2.sh deleted file mode 100755 index 94c8d5c..0000000 --- a/test/integration/e2e_io_mount_bucket_requester_pays.google-cls-v2.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# Copyright 2023 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset - -# Test gcsfuse abilities. -# -# This test is designed to verify that named GCS bucket (mount) -# command-line parameters work correctly. -# -# The actual operation performed here is to mount to a bucket containing a BAM -# and compute its md5, writing it to .bam.md5. - -readonly SCRIPT_DIR="$(dirname "${0}")" - -# Do standard test setup -source "${SCRIPT_DIR}/test_setup_e2e.sh" - -# Do io setup -source "${SCRIPT_DIR}/io_setup.sh" - -echo "Launching pipeline..." - -JOB_ID="$(io_setup::run_dsub_with_mount "gs://${DSUB_BUCKET_REQUESTER_PAYS}" "true")" -echo "JOB_ID = $JOB_ID" - -# Do validation -io_setup::check_output -io_setup::check_dstat "${JOB_ID}" false "gs://${DSUB_BUCKET_REQUESTER_PAYS}" diff --git a/test/integration/e2e_io_mount_disk.google-cls-v2.sh b/test/integration/e2e_io_mount_disk.google-cls-v2.sh deleted file mode 100755 index 9100645..0000000 --- a/test/integration/e2e_io_mount_disk.google-cls-v2.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# Copyright 2018 Verily Life Sciences Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset - -# Test persistent disk abilities. -# -# This test is designed to verify that mounting a Google Persistent Disk works. -# Input files have been placed inside persistent disk sourced by the test image -# used here. -# -# The actual operation performed here is to download a BAM and compute -# the md5, writing it to .bam.md5. -# and compute its md5, writing it to .bam.md5. - -readonly SCRIPT_DIR="$(dirname "${0}")" - -# Do standard test setup -source "${SCRIPT_DIR}/test_setup_e2e.sh" - -# Do io setup -source "${SCRIPT_DIR}/io_setup.sh" - -io_setup::image_setup -echo "Launching pipeline..." -JOB_ID="$(io_setup::run_dsub_with_mount "${TEST_IMAGE_URL} 50")" - -# Do validation -io_setup::check_output -io_setup::check_dstat "${JOB_ID}" false "${TEST_IMAGE_URL}" diff --git a/test/integration/e2e_io_mount_existing_disk.google-cls-v2.sh b/test/integration/e2e_io_mount_existing_disk.google-cls-v2.sh deleted file mode 100755 index 9d77d4b..0000000 --- a/test/integration/e2e_io_mount_existing_disk.google-cls-v2.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# Copyright 2022 Verily Life Sciences Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset - -# Test existing persistent disk abilities. -# -# This test is designed to verify that mounting an existing Google Persistent -# Disk works. -# Input files have been placed inside the persistent disk ahead of time. -# -# The actual operation performed here is to download a BAM and compute -# the md5, writing it to .bam.md5. -# and compute its md5, writing it to .bam.md5. - -readonly SCRIPT_DIR="$(dirname "${0}")" - -# Do standard test setup -source "${SCRIPT_DIR}/test_setup_e2e.sh" - -# Do io setup -source "${SCRIPT_DIR}/io_setup.sh" - -readonly ZONES="${TEST_EXISTING_DISK_ZONE}" - -io_setup::existing_disk_setup -echo "Launching pipeline..." -JOB_ID="$(io_setup::run_dsub_with_mount "${TEST_EXISTING_DISK_URL}")" - -# Do validation -io_setup::check_output -io_setup::check_dstat "${JOB_ID}" false "${TEST_EXISTING_DISK_URL}" diff --git a/test/integration/e2e_preemptible_retries_fail.google-cls-v2.sh b/test/integration/e2e_preemptible_retries_fail.google-cls-v2.sh deleted file mode 100755 index 566947f..0000000 --- a/test/integration/e2e_preemptible_retries_fail.google-cls-v2.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Copyright 2019 Verily Life Sciences Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset - -# Test the --retries with preemptible dsub flag. - -readonly SCRIPT_DIR="$(dirname "${0}")" - -# Do standard test setup -source "${SCRIPT_DIR}/test_setup_e2e.sh" - -# Do retries setup -source "${SCRIPT_DIR}/retries_setup.sh" - -readonly JOB_NAME="$(retries_setup::get_job_name "$(basename "${0}")")" - -echo "Launch a job that should fail with --retries 1 --preemptible 1" -retries_setup::run_dsub_preemptible "${JOB_NAME}" 1 1 false - -echo - -echo "Checking task metadata for job that should switch to non-preemptible after failure" -retries_setup::check_job_attr "${JOB_NAME}" status "FAILURE FAILURE" -retries_setup::check_job_attr "${JOB_NAME}" task-attempt "2 1" -retries_setup::check_job_attr "${JOB_NAME}" provider-attributes.preemptible "False True" - -echo "SUCCESS" diff --git a/test/integration/e2e_python_api.py b/test/integration/e2e_python_api.py index 7c50a9c..6bbd699 100644 --- a/test/integration/e2e_python_api.py +++ b/test/integration/e2e_python_api.py @@ -25,7 +25,6 @@ from dsub.lib import param_util from dsub.lib import resources from dsub.providers import google_batch -from dsub.providers import google_cls_v2 from dsub.providers import local # Because this may be invoked from another directory (treated as a library) or @@ -43,9 +42,6 @@ def get_dsub_provider(): """Return the appropriate google_base.JobProvider instance.""" if test.DSUB_PROVIDER == 'local': return local.LocalJobProvider(resources) - elif test.DSUB_PROVIDER == 'google-cls-v2': - return google_cls_v2.GoogleCLSV2JobProvider(False, test.PROJECT_ID, - 'us-central1') elif test.DSUB_PROVIDER == 'google-batch': return google_batch.GoogleBatchJobProvider( False, test.PROJECT_ID, 'us-central1' diff --git a/test/integration/e2e_user.google-cls-v2.sh b/test/integration/e2e_user.google-cls-v2.sh deleted file mode 100755 index 7b3f146..0000000 --- a/test/integration/e2e_user.google-cls-v2.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash - -# Copyright 2019 Verily Life Sciences Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# For google-cls-v2, user-ids and job names are used as labels. -# Test that --user and --name works properly when it has underscores -# and capital letters. - -# Do standard test setup -readonly SCRIPT_DIR="$(dirname "${0}")" -source "${SCRIPT_DIR}/test_setup_e2e.sh" - -readonly RANDOM_STRING="$(cat /dev/urandom | env LC_CTYPE=C tr -cd 'a-z0-9' \ - | head -c 8)" -readonly USER_NAME="UNDERSCORE_USER" -readonly USER_NAME_2="UNDERSCORE_USER_2" -readonly JOB_NAME="UNDERSCORE_JOB_${RANDOM_STRING}" - -# (1) Launch 2 simple jobs with a user_id and name containing -# an underscore and caps -echo "Launching two jobs, one with user ${USER_NAME}," -echo "And one with user ${USER_NAME_2} (and don't --wait)..." -JOB_ID="$(run_dsub \ - --command 'sleep 5s' \ - --user "${USER_NAME}" \ - --name "${JOB_NAME}")" - -JOB_ID_2="$(run_dsub \ - --command 'sleep 5s' \ - --user "${USER_NAME_2}" \ - --name "${JOB_NAME}")" - -sleep 5s - -# (2) Validate dstat can find the job by job id and username -echo "Check that the job can be found with user" -DSTAT_OUTPUT="$(run_dstat \ - --status '*' \ - --jobs "${JOB_ID}" \ - --users "${USER_NAME}")" -if [[ -z "${DSTAT_OUTPUT}" ]]; then - echo "ERROR: dstat output for ${JOB_ID} not found." - exit 1 -fi - -# (3) Validate dstat can find the job by job name and username -echo "Check that the job can be found with name" -DSTAT_OUTPUT="$(run_dstat \ - --status '*' \ - --names "${JOB_NAME}" \ - --users "${USER_NAME}")" -if [[ -z "${DSTAT_OUTPUT}" ]]; then - echo "ERROR: dstat output for ${JOB_ID} not found." - exit 1 -fi - -# (4) Validate that dstat can find the job with --users '*' -echo "Check that the job can be found with --users '*'" -DSTAT_OUTPUT="$(run_dstat \ - --status '*' \ - --names "${JOB_NAME}" \ - --users "*")" -if [[ -z "${DSTAT_OUTPUT}" ]]; then - echo "ERROR: dstat output for ${JOB_ID} not found." - exit 1 -fi - -# (5) Validate that dstat can find jobs for multiple specified users -DSTAT_OUTPUT="$(run_dstat \ - --status '*' \ - --names "${JOB_NAME}" \ - --users "${USER_NAME}" "${USER_NAME_2}" \ - --format yaml)" - -TASK_COUNT=$(echo "${DSTAT_OUTPUT}" | grep -w "job-name" | wc -l) - -if [[ "${TASK_COUNT}" -ne 2 ]]; then - echo "Number of tasks returned by dstat --users not 2!" - echo "${DSTAT_OUTPUT}" - exit 1 -fi - -echo "dstat output was not empty." -echo "Success!" diff --git a/test/integration/test_setup.sh b/test/integration/test_setup.sh index 769e0f6..f7b45f4 100644 --- a/test/integration/test_setup.sh +++ b/test/integration/test_setup.sh @@ -37,8 +37,6 @@ export USER="${USER:-$(whoami)}" # If the script name is ..sh, pull out the provider. # If the script name is .sh, use "local". # If the DSUB_PROVIDER is set, make sure it is correct for a provider test. -# Special-case the google-cls-v2 tests to be runnable for google-batch -# and google-batch. readonly SCRIPT_NAME="$(basename "$0")" readonly SCRIPT_DEFAULT_PROVIDER=$( @@ -49,10 +47,7 @@ readonly SCRIPT_DEFAULT_PROVIDER=$( if [[ -z "${DSUB_PROVIDER:-}" ]]; then readonly DSUB_PROVIDER="${SCRIPT_DEFAULT_PROVIDER:-local}" elif [[ -n "${SCRIPT_DEFAULT_PROVIDER}" ]]; then - if [[ "${DSUB_PROVIDER}" == "google-batch" ]] && \ - [[ "${SCRIPT_DEFAULT_PROVIDER}" == "google-cls-v2" ]]; then - echo "Running google-cls-v2 e2e/unit tests with provider google-batch" - elif [[ "${DSUB_PROVIDER}" != "${SCRIPT_DEFAULT_PROVIDER}" ]]; then + if [[ "${DSUB_PROVIDER}" != "${SCRIPT_DEFAULT_PROVIDER}" ]]; then 1>&2 echo "DSUB_PROVIDER is '${DSUB_PROVIDER:-}' not '${SCRIPT_DEFAULT_PROVIDER}'" exit 1 fi @@ -112,21 +107,6 @@ function dsub_google-batch() { "${@}" } -function dsub_google-cls-v2() { - local location="${LOCATION:-}" - local zones="${ZONES:-}" - local regions="${REGIONS:-}" - - dsub \ - --provider google-cls-v2 \ - --project "${PROJECT_ID}" \ - ${location:+--location "${location}"} \ - --logging "${LOGGING_OVERRIDE:-${LOGGING}}" \ - ${regions:+--regions "${regions}"} \ - ${zones:+--zones "${zones}"} \ - "${@}" -} - function dsub_local() { dsub \ --provider local \ @@ -168,16 +148,6 @@ function dstat_google-batch() { "${@}" } -function dstat_google-cls-v2() { - local location="${LOCATION:-}" - - dstat \ - --provider google-cls-v2 \ - --project "${PROJECT_ID}" \ - ${location:+--location "${location}"} \ - "${@}" -} - function dstat_local() { dstat \ --provider local \ @@ -202,16 +172,6 @@ function run_ddel() { run_ddel_age "45m" "${@}" } -function ddel_google-cls-v2() { - local location="${LOCATION:-}" - - ddel \ - --provider google-cls-v2 \ - --project "${PROJECT_ID}" \ - ${location:+--location "${location}"} \ - "${@}" -} - function ddel_google-batch() { local location="${LOCATION:-}" diff --git a/test/integration/test_setup_e2e.py b/test/integration/test_setup_e2e.py index f5df9e8..ab2bec1 100644 --- a/test/integration/test_setup_e2e.py +++ b/test/integration/test_setup_e2e.py @@ -195,31 +195,6 @@ def dsub_google_batch(dsub_args): # pyformat: enable -def dsub_google_cls_v2(dsub_args): - """Call dsub appending google-cls-v2 required arguments.""" - # pyformat: disable - google_cls_v2_opt_args = [ - ("BOOT_DISK_SIZE", "--boot-disk-size"), - ("DISK_SIZE", "--disk-size") - ] - # pyformat: enable - - opt_args = [] - for var in google_cls_v2_opt_args: - val = globals().get(var[0]) - if val: - opt_args.append(var[1], val) - - # pyformat: disable - return dsub_command.call([ - "--provider", "google-cls-v2", - "--project", PROJECT_ID, - "--logging", LOGGING, - "--regions", "us-central1" - ] + opt_args + dsub_args) - # pyformat: enable - - def dsub_local(dsub_args): """Call dsub appending local-provider required arguments.""" diff --git a/test/integration/test_unit_util.sh b/test/integration/test_unit_util.sh index 3fe63f8..bbe4515 100644 --- a/test/integration/test_unit_util.sh +++ b/test/integration/test_unit_util.sh @@ -147,12 +147,24 @@ readonly -f assert_err_not_contains function assert_output_empty() { - [[ ! -s "${TEST_STDOUT}" ]] || "Assert: stdout is not empty" + if [[ -s "${TEST_STDOUT}" ]]; then + 1>&2 echo "Assert: stdout is not empty" + 1>&2 echo "ACTUAL:" + 1>&2 echo "$(<"${TEST_STDOUT}")" + + exit 1 + fi } readonly -f assert_output_empty function assert_err_empty() { - [[ ! -s "${TEST_STDERR}" ]] || "Assert: stderr is not empty" + if [[ -s "${TEST_STDERR}" ]]; then + 1>&2 echo "Assert: stderr is not empty" + 1>&2 echo "ACTUAL:" + 1>&2 echo "$(<"${TEST_STDERR}")" + + exit 1 + fi } readonly -f assert_err_empty diff --git a/test/integration/unit_flags.google-cls-v2.sh b/test/integration/unit_flags.google-cls-v2.sh deleted file mode 100755 index 067c576..0000000 --- a/test/integration/unit_flags.google-cls-v2.sh +++ /dev/null @@ -1,1002 +0,0 @@ -#!/bin/bash - -# Copyright 2018 Verily Life Sciences Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset - -# unit_flags.google-cls-v2.sh -# -# Collection of unit tests for dsub command-line flags -# specific to the google-cls-v2 provider. - -readonly SCRIPT_DIR="$(dirname "${0}")" - -# Do standard test setup -source "${SCRIPT_DIR}/test_setup_unit.sh" - -function call_dsub() { - local image="${DOCKER_IMAGE_OVERRIDE:-dummy-image}" - - dsub \ - --provider "${DSUB_PROVIDER}" \ - --project "${PROJECT_ID}" \ - --logging "${LOGGING_OVERRIDE:-${LOGGING}}" \ - --image "${image}" \ - "${@}" \ - --dry-run \ - 1> "${TEST_STDOUT}" \ - 2> "${TEST_STDERR}" -} -readonly -f call_dsub - -readonly NETWORK_NAME_KEY="network" -readonly CONTAINER_NAME_KEY="containerName" - -# Define tests - -function test_preemptible_zero() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --preemptible 0; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.preemptible" "False" - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_preemptible_zero - -function test_preemptible_off() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.preemptible" "False" - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_preemptible_off - -function test_preemptible_on() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --preemptible; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.preemptible" "True" - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_preemptible_on - -# A google-cls-v2 test that the location value is settable and used -# for the region. -function test_location() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --location us-west2 \ - --command 'echo "${TEST_NAME}"'; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.regions.[0]" "us-west2" - assert_err_value_equals \ - "[0].pipeline.resources.zones" "[]" - - test_passed "${subtest}" - else - 1>&2 echo "Using the location flag generated an error" - - test_failed "${subtest}" - fi -} -readonly -f test_location - -function test_neither_region_nor_zone() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"'; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.regions.[0]" "us-central1" - assert_err_value_equals \ - "[0].pipeline.resources.zones" "[]" - - test_passed "${subtest}" - else - 1>&2 echo "Location not used as default region" - - test_failed "${subtest}" - fi -} -readonly -f test_neither_region_nor_zone - -function test_region_and_zone() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --zones us-central1-f \ - --regions us-central1; then - - 1>&2 echo "Both regions and zones specified - not detected" - - test_failed "${subtest}" - else - - assert_output_empty - - assert_err_contains \ - "ValueError: At most one of --regions and --zones may be specified" - - test_passed "${subtest}" - fi -} -readonly -f test_region_and_zone - -function test_regions() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.regions.[0]" "us-central1" - assert_err_value_equals \ - "[0].pipeline.resources.zones" "[]" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_regions - -function test_zones() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --zones us-central1-a; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.regions" "[]" - assert_err_value_equals \ - "[0].pipeline.resources.zones.[0]" "us-central1-a" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_zones - -function test_min_cores() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --min-cores 1; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.machineType" "custom-1-3840" - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_min_cores - -function test_min_ram() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --min-ram 1; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.machineType" "custom-1-1024" - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_min_ram - -function test_machine_type() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --machine-type "n1-highmem-2"; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.machineType" "n1-highmem-2" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_machine_type - -function test_no_machine_type() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.machineType" "n1-standard-1" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_no_machine_type - -function test_machine_type_with_ram_and_cpu() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --machine-type "n1-highmem-2" \ - --min-cores 1 \ - --min-ram 1; then - - 1>&2 echo "min-ram/min-cores set with machine-type on google-cls-v2 provider - not detected" - - test_failed "${subtest}" - else - assert_output_empty - - assert_err_contains \ - "ValueError: --machine-type not supported together with --min-cores or --min-ram." - - test_passed "${subtest}" - fi -} -readonly -f test_machine_type_with_ram_and_cpu - -function test_accelerator_type_and_count() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --accelerator-type "nvidia-tesla-k80" \ - --nvidia-driver-version "390.46" \ - --accelerator-count 2; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.accelerators.[0].type" "nvidia-tesla-k80" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.accelerators.[0].count" "2" - - # The nvidia-driver-version should be ignored (deprecated Sept 2020) - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.nvidiaDriverVersion" "None" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_accelerator_type_and_count - -function test_no_accelerator_type_and_count() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.accelerators" "None" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.nvidiaDriverVersion" "None" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_no_accelerator_type_and_count - -function test_network() { - local subtest="${FUNCNAME[0]}" - - if DOCKER_IMAGE_OVERRIDE="marketplace.gcr.io/google/debian9" call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --network 'network-name-foo' \ - --subnetwork 'subnetwork-name-foo' \ - --use-private-address; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.network.${NETWORK_NAME_KEY}" "network-name-foo" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.network.subnetwork" "subnetwork-name-foo" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.network.usePrivateAddress" "True" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_network - -function test_no_network() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.network.${NETWORK_NAME_KEY}" "None" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.network.subnetwork" "None" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.network.usePrivateAddress" "False" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_no_network - -function test_use_private_address_with_public_image() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --use-private-address; then - - 1>&2 echo "Public image used with no public address was not detected" - - test_failed "${subtest}" - else - - assert_output_empty - - assert_err_contains \ - "ValueError: --use-private-address must specify a --image with a gcr.io or pkg.dev host" - - test_passed "${subtest}" - fi -} -readonly -f test_use_private_address_with_public_image - -function test_use_private_address_with_gcr_io() { - local subtest="${FUNCNAME[0]}" - - if DOCKER_IMAGE_OVERRIDE="marketplace.gcr.io/google/debian9" call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --use-private-address; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.actions.[3].imageUri" "marketplace.gcr.io/google/debian9" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_use_private_address_with_gcr_io - -function test_use_private_address_with_pkg_dev() { - local subtest="${FUNCNAME[0]}" - - if DOCKER_IMAGE_OVERRIDE="us-central1-docker.pkg.dev/my-project/my-repo/my-image" call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --use-private-address; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.actions.[3].imageUri" "us-central1-docker.pkg.dev/my-project/my-repo/my-image" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_use_private_address_with_pkg_dev - -function test_cpu_platform() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --cpu-platform 'Intel Skylake'; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.cpuPlatform" "Intel Skylake" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_cpu_platform - -function test_no_cpu_platform() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.cpuPlatform" "None" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_no_cpu_platform - -function test_timeout() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --timeout '1h'; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.timeout" "3600.0s" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_timeout - -function test_no_timeout() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.timeout" "None" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_no_timeout - -function test_log_interval() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --log-interval '1h'; then - - # Check that the output contains expected values - assert_err_value_matches \ - "[0].pipeline.actions.[0].commands.[1]" "3600.0s" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_log_interval - -function test_no_log_interval() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1; then - - # Check that the output contains expected values - assert_err_value_matches \ - "[0].pipeline.actions.[0].commands.[1]" "60s" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_no_log_interval - -function test_ssh() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --ssh; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.actions.[1].entrypoint" "ssh-server" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_ssh - -function test_no_ssh() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1; then - - # Check that the output does not contain ssh values - assert_err_not_contains \ - "ssh-server" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_no_ssh - -function test_user_project() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --user-project 'sample-project-name'; then - - # Check for the USER_PROJECT in the environment for the - # logging, localization, delocalization, and final_logging actions - assert_err_value_matches \ - "[0].pipeline.actions.[0].environment.USER_PROJECT" "sample-project-name" - assert_err_value_matches \ - "[0].pipeline.actions.[2].environment.USER_PROJECT" "sample-project-name" - assert_err_value_matches \ - "[0].pipeline.actions.[4].environment.USER_PROJECT" "sample-project-name" - assert_err_value_matches \ - "[0].pipeline.actions.[5].environment.USER_PROJECT" "sample-project-name" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_user_project - -function test_no_user_project() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1; then - - # Check that the output contains expected values - assert_err_not_contains "sample-project-name" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_no_user_project - -function test_service_account() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --service-account 'foo@bar.com'; then - - # Check for the service account email - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.serviceAccount.email" "foo@bar.com" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_service_account - -function test_no_service_account() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.serviceAccount.email" "default" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_no_service_account - -function test_disk_type() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --disk-type 'pd-ssd'; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].volume" "datadisk" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].persistentDisk.sizeGb" "200" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].persistentDisk.type" "pd-ssd" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].persistentDisk.sourceImage" "None" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_disk_type - -function test_no_disk_type() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 ; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].volume" "datadisk" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].persistentDisk.sizeGb" "200" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].persistentDisk.type" "pd-standard" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].persistentDisk.sourceImage" "None" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_no_disk_type - -function test_mount_image() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --mount 'MOUNT_POINT=https://www.googleapis.com/compute/v1/projects/my-project/global/images/my-image 250'; then - - # Check that the output contains expected values - - # The volumes aren't order dependent, but we know the code adds the - # data disk first - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].volume" "datadisk" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].persistentDisk.sizeGb" "200" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].persistentDisk.type" "pd-standard" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].persistentDisk.sourceImage" "None" - - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[1].volume" "MOUNT-POINT" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[1].persistentDisk.sizeGb" "250" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[1].persistentDisk.type" "pd-standard" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[1].persistentDisk.sourceImage" "https://www.googleapis.com/compute/v1/projects/my-project/global/images/my-image" - - # Check the mount points and environment variables for the user-command - assert_err_value_equals \ - "[0].pipeline.actions.[3].${CONTAINER_NAME_KEY}" "user-command" - assert_err_value_equals \ - "[0].pipeline.actions.[3].environment.MOUNT_POINT" "/mnt/data/mount/https/www.googleapis.com/compute/v1/projects/my-project/global/images/my-image" - - # The mounts aren't order dependent, but we know the code adds the - # data disk first - assert_err_value_equals \ - "[0].pipeline.actions.[3].mounts.[0].disk" "datadisk" - assert_err_value_equals \ - "[0].pipeline.actions.[3].mounts.[0].path" "/mnt/data" - assert_err_value_equals \ - "[0].pipeline.actions.[3].mounts.[0].readOnly" "False" - assert_err_value_equals \ - "[0].pipeline.actions.[3].mounts.[1].disk" "MOUNT-POINT" - assert_err_value_equals \ - "[0].pipeline.actions.[3].mounts.[1].path" "/mnt/data/mount/https/www.googleapis.com/compute/v1/projects/my-project/global/images/my-image" - assert_err_value_equals \ - "[0].pipeline.actions.[3].mounts.[1].readOnly" "True" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_mount_image - - -function test_mount_existing_disk() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --mount 'MOUNT_POINT=https://www.googleapis.com/compute/v1/projects/my-project/zones/us-central1-a/disks/my-existing-disk'; then - - # Check that the output contains expected values - - # The volumes aren't order dependent, but we know the code adds the - # data disk first - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].volume" "datadisk" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].persistentDisk.sizeGb" "200" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].persistentDisk.type" "pd-standard" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[0].persistentDisk.sourceImage" "None" - - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[1].volume" "MOUNT-POINT" - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.volumes.[1].existingDisk.disk" "https://www.googleapis.com/compute/v1/projects/my-project/zones/us-central1-a/disks/my-existing-disk" - - # Check the mount points and environment variables for the user-command - assert_err_value_equals \ - "[0].pipeline.actions.[3].${CONTAINER_NAME_KEY}" "user-command" - assert_err_value_equals \ - "[0].pipeline.actions.[3].environment.MOUNT_POINT" "/mnt/data/mount/https/www.googleapis.com/compute/v1/projects/my-project/zones/us-central1-a/disks/my-existing-disk" - - # The mounts aren't order dependent, but we know the code adds the - # data disk first - assert_err_value_equals \ - "[0].pipeline.actions.[3].mounts.[0].disk" "datadisk" - assert_err_value_equals \ - "[0].pipeline.actions.[3].mounts.[0].path" "/mnt/data" - assert_err_value_equals \ - "[0].pipeline.actions.[3].mounts.[0].readOnly" "False" - assert_err_value_equals \ - "[0].pipeline.actions.[3].mounts.[1].disk" "MOUNT-POINT" - assert_err_value_equals \ - "[0].pipeline.actions.[3].mounts.[1].path" "/mnt/data/mount/https/www.googleapis.com/compute/v1/projects/my-project/zones/us-central1-a/disks/my-existing-disk" - assert_err_value_equals \ - "[0].pipeline.actions.[3].mounts.[1].readOnly" "True" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_mount_existing_disk - -function test_stackdriver() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --enable-stackdriver-monitoring; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.enableStackdriverMonitoring" "True" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_stackdriver - -function test_no_stackdriver() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1; then - - # Check that the output contains expected values - assert_err_value_equals \ - "[0].pipeline.resources.virtualMachine.enableStackdriverMonitoring" "False" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_no_stackdriver - -function test_block_external_network() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1 \ - --block-external-network; then - - assert_err_value_equals \ - "[0].pipeline.actions.[3].blockExternalNetwork" "True" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_block_external_network - -function test_no_block_external_network() { - local subtest="${FUNCNAME[0]}" - - if call_dsub \ - --command 'echo "${TEST_NAME}"' \ - --regions us-central1; then - - # Check that the output does not contain block network flag - assert_err_value_equals \ - "[0].pipeline.actions.[3].blockExternalNetwork" "False" - - test_passed "${subtest}" - else - test_failed "${subtest}" - fi -} -readonly -f test_no_block_external_network - - -# Run the tests -trap "exit_handler" EXIT - -mkdir -p "${TEST_TMP}" - -echo -test_location - -echo -test_preemptible_zero -test_preemptible_off -test_preemptible_on - -echo -test_neither_region_nor_zone -test_region_and_zone -test_regions -test_zones - -echo -test_min_cores -test_min_ram -test_machine_type -test_no_machine_type -test_machine_type_with_ram_and_cpu - -echo -test_accelerator_type_and_count -test_no_accelerator_type_and_count - -echo -test_network -test_no_network -test_use_private_address_with_public_image -test_use_private_address_with_gcr_io -test_use_private_address_with_pkg_dev - -echo -test_cpu_platform -test_no_cpu_platform - -echo -test_timeout -test_no_timeout - -echo -test_log_interval -test_no_log_interval - -echo -test_ssh -test_no_ssh - -echo -test_user_project -test_no_user_project - -echo -test_service_account -test_no_service_account - -echo -test_disk_type -test_no_disk_type - -echo -test_mount_image -test_mount_existing_disk - -echo -test_stackdriver -test_no_stackdriver - -echo -test_block_external_network -test_no_block_external_network diff --git a/test/integration/unit_flags.test-fails.sh b/test/integration/unit_flags.test-fails.sh index e76a3c8..ac460d1 100755 --- a/test/integration/unit_flags.test-fails.sh +++ b/test/integration/unit_flags.test-fails.sh @@ -37,7 +37,7 @@ run_dsub \ --label CAPS=bad \ --command 'echo "Hi"' \ 1> "${TEST_STDOUT}" \ - 2> "${TEST_STDERR}" || /bin/true + 2> "${TEST_STDERR}" || true assert_err_contains \ 'ValueError: Invalid name for label: "CAPS"' @@ -47,7 +47,7 @@ run_dsub \ --label lowercase_understores=ok-dashes_too \ --command 'echo "Hi"' \ 1> "${TEST_STDOUT}" \ - 2> "${TEST_STDERR}" || /bin/true + 2> "${TEST_STDERR}" || true # Fancier testing of the label syntax is done in test_param_util.py diff --git a/test/integration/unit_version.sh b/test/integration/unit_version.sh index f4b09ec..67e2164 100755 --- a/test/integration/unit_version.sh +++ b/test/integration/unit_version.sh @@ -39,7 +39,21 @@ function test_shell_util_version() { 1> "${TEST_STDOUT}" \ 2> "${TEST_STDERR}" - # Stderr must be empty and stdout should have the version. + # Some environments print benign warnings from third-party libraries on + # every invocation (e.g. urllib3's OpenSSL check, google-cloud-batch's + # Python-version notice). Strip those known, harmless warning stanzas + # before asserting stderr is otherwise empty. Match the specific known + # message text, not the bare warning category, so unrelated warnings of + # the same category are not silently hidden. + grep -v \ + -e 'NotOpenSSLWarning' \ + -e 'FutureWarning: You are using a Python version.*will stop supporting in google.cloud.batch_v1' \ + -e '^[[:space:]]*warnings\.warn(' \ + "${TEST_STDERR}" > "${TEST_STDERR}.filtered" || true + mv "${TEST_STDERR}.filtered" "${TEST_STDERR}" + + # Stderr must be empty (aside from the known warnings filtered above) and + # stdout should have the version. assert_err_empty if [[ "$(cat "${TEST_STDOUT}")" != "${EXPECTED_STRING}" ]] ; then 1>&2 echo "Assert: version was not printed by '$@ --version'" diff --git a/test/run_tests.sh b/test/run_tests.sh index bc355ea..98d13d3 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -216,7 +216,7 @@ function get_test_providers() { return fi if [[ "${NO_GOOGLE_BATCH_TESTS:-0}" -eq 1 ]]; then - echo -n "local google-cls-v2" + echo -n "local" return fi case "${test_file}" in @@ -238,8 +238,6 @@ function get_test_providers() { e2e_io.sh | \ e2e_io_auto.sh | \ e2e_io_gcs_tasks.sh | \ - e2e_io_mount_bucket.google-cls-v2.sh | \ - e2e_io_mount_bucket_requester_pays.google-cls-v2.sh | \ e2e_io_recursive.sh | \ e2e_io_tasks.py | \ e2e_io_tasks.sh | \ @@ -253,7 +251,6 @@ function get_test_providers() { e2e_logging_paths_retry_failure_tasks.sh | \ e2e_logging_paths_retry_tasks.sh | \ e2e_non_root.sh | \ - e2e_preemptible_retries_fail.google-cls-v2.sh | \ e2e_python.sh | \ e2e_python_api.py | \ e2e_requester_pays_buckets.sh | \ @@ -262,12 +259,11 @@ function get_test_providers() { e2e_retries_fail_2.sh | \ e2e_runtime.sh | \ e2e_skip.sh | \ - e2e_skip_tasks.sh | \ - e2e_user.google-cls-v2.sh) - local all_provider_list="${DSUB_PROVIDER:-local google-cls-v2 google-batch}" + e2e_skip_tasks.sh) + local all_provider_list="${DSUB_PROVIDER:-local google-batch}" ;; *) - local all_provider_list="${DSUB_PROVIDER:-local google-cls-v2}" + local all_provider_list="${DSUB_PROVIDER:-local}" ;; esac