diff --git a/storage/cloudbuild/run_zonal_tests.sh b/storage/cloudbuild/run_rapid_tests.sh similarity index 83% rename from storage/cloudbuild/run_zonal_tests.sh rename to storage/cloudbuild/run_rapid_tests.sh index 25a8e714c7d..5fb2e687a87 100644 --- a/storage/cloudbuild/run_zonal_tests.sh +++ b/storage/cloudbuild/run_rapid_tests.sh @@ -40,9 +40,11 @@ pip install google-cloud-testutils google-cloud-kms pip install google-cloud-storage[grpc,testing] echo '--- Setting up environment variables on VM ---' -export ZONAL_BUCKET=${_ZONAL_BUCKET} -export RUN_ZONAL_SYSTEM_TESTS=True +export RAPID_BUCKET="${_RAPID_BUCKET:-${_ZONAL_BUCKET:-}}" +export ZONAL_BUCKET="${RAPID_BUCKET}" +export RUN_RAPID_SYSTEM_TESTS=True +export RUN_BIDI_SYSTEM_TESTS=True export GCE_METADATA_MTLS_MODE=None CURRENT_ULIMIT=$(ulimit -n) -echo '--- Running Zonal tests on VM with ulimit set to ---' $CURRENT_ULIMIT -pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' samples/snippets/zonal_buckets/zonal_snippets_test.py +echo '--- Running Rapid bucket (bidi) tests on VM with ulimit set to ---' $CURRENT_ULIMIT +pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' samples/snippets/bidi/bidi_snippets_test.py diff --git a/storage/cloudbuild/zb-system-tests-cloudbuild.yaml b/storage/cloudbuild/zb-system-tests-cloudbuild.yaml index b968a0df18a..04b5d2188ab 100644 --- a/storage/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/storage/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -19,6 +19,10 @@ substitutions: _SHORT_BUILD_ID: ${BUILD_ID:0:8} _VM_NAME: "py-sdk-sys-test-${_SHORT_BUILD_ID}" _ULIMIT: "10000" # 10k, for gRPC bidi streams + # Name of the rapid bucket used by the bidi samples tests. Set by the trigger. + _ZONAL_BUCKET: "" + # Service account attached to the test VM. Set by the trigger. + _ZONAL_VM_SERVICE_ACCOUNT: "" @@ -113,12 +117,12 @@ steps: sleep 15 done # copy the script to the VM - gcloud compute scp storage/cloudbuild/run_zonal_tests.sh ${_VM_NAME}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine + gcloud compute scp storage/cloudbuild/run_rapid_tests.sh ${_VM_NAME}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine # Execute the script on the VM via SSH. # Capture the exit code to ensure cleanup happens before the build fails. set +e - gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n ${_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} CROSS_REGION_BUCKET=${_CROSS_REGION_BUCKET} _PR_NUMBER=${_PR_NUMBER} bash run_zonal_tests.sh" + gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n ${_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _RAPID_BUCKET=${_ZONAL_BUCKET} _ZONAL_BUCKET=${_ZONAL_BUCKET} CROSS_REGION_BUCKET=${_CROSS_REGION_BUCKET} _PR_NUMBER=${_PR_NUMBER} bash run_rapid_tests.sh" EXIT_CODE=$? set -e diff --git a/storage/samples/snippets/bidi/README.md b/storage/samples/snippets/bidi/README.md new file mode 100644 index 00000000000..5dc4419adb4 --- /dev/null +++ b/storage/samples/snippets/bidi/README.md @@ -0,0 +1,91 @@ +# Google Cloud Storage - Bidirectional Streaming (Bidi) Snippets + +This directory contains snippets that use the Google Cloud Storage gRPC +bidirectional streaming (bidi) APIs, including appendable object writes and +multi-range reads. + +## Prerequisites + +- A Google Cloud Platform project with the Cloud Storage API enabled. +- For the appendable object snippets (create, finalize, pause and resume, tail, + and the pre-warmed writer pool), a rapid bucket (a bucket with the Rapid + storage class). +- The ranged read snippets work with any Google Cloud Storage bucket. + +## Running the snippets + +### Create and write to an appendable object + +This snippet uploads an appendable object to a rapid bucket. + +```bash +python samples/snippets/bidi/storage_create_and_write_appendable_object.py --bucket_name --object_name +``` + +### Finalize an appendable object upload + +This snippet creates, writes to, and finalizes an appendable object. + +```bash +python samples/snippets/bidi/storage_finalize_appendable_object_upload.py --bucket_name --object_name +``` + +### Pause and resume an appendable object upload + +This snippet demonstrates pausing and resuming an appendable object upload. + +```bash +python samples/snippets/bidi/storage_pause_and_resume_appendable_upload.py --bucket_name --object_name +``` + +### Tail an appendable object + +This snippet demonstrates tailing an appendable GCS object, similar to `tail -f`. + +```bash +python samples/snippets/bidi/storage_read_appendable_object_tail.py --bucket_name --object_name --duration +``` + + +### Download a range of bytes from an object + +This snippet downloads a range of bytes from an object. + +```bash +python samples/snippets/bidi/storage_open_object_single_ranged_read.py --bucket_name --object_name --start_byte --size +``` + + +### Download multiple ranges of bytes from a single object + +This snippet downloads multiple ranges of bytes from a single object into different buffers. + +```bash +python samples/snippets/bidi/storage_open_object_multiple_ranged_read.py --bucket_name --object_name +``` + +### Download the entire content of an object + +This snippet downloads the entire content of an object using a multi-range downloader. + +```bash +python samples/snippets/bidi/storage_open_object_read_full_object.py --bucket_name --object_name +``` + + + +### Download a range of bytes from multiple objects concurrently + +This snippet downloads a range of bytes from multiple objects concurrently. + +```bash +python samples/snippets/bidi/storage_open_multiple_objects_ranged_read.py --bucket_name --object_names +``` + +### Optimize write latency with a pre-warmed writer pool + +This snippet uses a pre-warmed pool of writers for appendable objects in a rapid bucket. + +```bash +python samples/snippets/bidi/storage_optimize_write_latency_pool.py --bucket_name --key_prefix +``` \ No newline at end of file diff --git a/storage/samples/snippets/zonal_buckets/zonal_snippets_test.py b/storage/samples/snippets/bidi/bidi_snippets_test.py similarity index 82% rename from storage/samples/snippets/zonal_buckets/zonal_snippets_test.py rename to storage/samples/snippets/bidi/bidi_snippets_test.py index 3cb40b98cb8..99f15b3cbb9 100644 --- a/storage/samples/snippets/zonal_buckets/zonal_snippets_test.py +++ b/storage/samples/snippets/bidi/bidi_snippets_test.py @@ -36,14 +36,21 @@ import storage_read_appendable_object_tail pytestmark = pytest.mark.skipif( - os.getenv("RUN_ZONAL_SYSTEM_TESTS") != "True", - reason="Zonal system tests need to be explicitly enabled. This helps scheduling tests in Kokoro and Cloud Build.", + not any( + os.getenv(v) == "True" + for v in ( + "RUN_RAPID_SYSTEM_TESTS", + "RUN_BIDI_SYSTEM_TESTS", + "RUN_ZONAL_SYSTEM_TESTS", + ) + ), + reason="Rapid bucket (bidi) system tests need to be explicitly enabled. This helps scheduling tests in Kokoro and Cloud Build.", ) -# TODO: replace this with a fixture once zonal bucket creation / deletion +# TODO: replace this with a fixture once rapid bucket creation / deletion # is supported in grpc client or json client client. -_ZONAL_BUCKET = os.getenv("ZONAL_BUCKET") +_RAPID_BUCKET = os.getenv("RAPID_BUCKET") or os.getenv("ZONAL_BUCKET") async def create_async_grpc_client(): @@ -77,7 +84,7 @@ def json_client(): async def create_appendable_object(grpc_client, object_name, data): writer = AsyncAppendableObjectWriter( client=grpc_client, - bucket_name=_ZONAL_BUCKET, + bucket_name=_RAPID_BUCKET, object_name=object_name, generation=0, # throws `FailedPrecondition` if object already exists. ) @@ -87,25 +94,20 @@ async def create_appendable_object(grpc_client, object_name, data): return writer.generation -# TODO: replace this with a fixture once zonal bucket creation / deletion -# is supported in grpc client or json client client. -_ZONAL_BUCKET = os.getenv("ZONAL_BUCKET") - - def test_storage_create_and_write_appendable_object( async_grpc_client, json_client, event_loop, capsys ): - object_name = f"zonal-snippets-test-{uuid.uuid4()}" + object_name = f"bidi-snippets-test-{uuid.uuid4()}" event_loop.run_until_complete( storage_create_and_write_appendable_object.storage_create_and_write_appendable_object( - _ZONAL_BUCKET, object_name, grpc_client=async_grpc_client + _RAPID_BUCKET, object_name, grpc_client=async_grpc_client ) ) out, _ = capsys.readouterr() assert f"Appended object {object_name} created of size" in out - blob = json_client.bucket(_ZONAL_BUCKET).blob(object_name) + blob = json_client.bucket(_RAPID_BUCKET).blob(object_name) blob.delete() @@ -115,12 +117,12 @@ def test_storage_finalize_appendable_object_upload( object_name = f"test-finalize-appendable-{uuid.uuid4()}" event_loop.run_until_complete( storage_finalize_appendable_object_upload.storage_finalize_appendable_object_upload( - _ZONAL_BUCKET, object_name, grpc_client=async_grpc_client + _RAPID_BUCKET, object_name, grpc_client=async_grpc_client ) ) out, _ = capsys.readouterr() assert f"Appendable object {object_name} created and finalized." in out - blob = json_client.bucket(_ZONAL_BUCKET).get_blob(object_name) + blob = json_client.bucket(_RAPID_BUCKET).get_blob(object_name) blob.delete() @@ -130,14 +132,14 @@ def test_storage_pause_and_resume_appendable_upload( object_name = f"test-pause-resume-{uuid.uuid4()}" event_loop.run_until_complete( storage_pause_and_resume_appendable_upload.storage_pause_and_resume_appendable_upload( - _ZONAL_BUCKET, object_name, grpc_client=async_grpc_client + _RAPID_BUCKET, object_name, grpc_client=async_grpc_client ) ) out, _ = capsys.readouterr() assert "First writer closed. Upload is 'paused'." in out assert "Second writer closed. Full object uploaded." in out - blob = json_client.bucket(_ZONAL_BUCKET).get_blob(object_name) + blob = json_client.bucket(_RAPID_BUCKET).get_blob(object_name) blob.delete() @@ -147,7 +149,7 @@ def test_storage_read_appendable_object_tail( object_name = f"test-read-tail-{uuid.uuid4()}" event_loop.run_until_complete( storage_read_appendable_object_tail.read_appendable_object_tail( - _ZONAL_BUCKET, object_name, duration=3, grpc_client=async_grpc_client + _RAPID_BUCKET, object_name, duration=3, grpc_client=async_grpc_client ) ) out, _ = capsys.readouterr() @@ -158,7 +160,7 @@ def test_storage_read_appendable_object_tail( assert "Tailer finished." in out assert "Writer closed." in out - bucket = json_client.bucket(_ZONAL_BUCKET) + bucket = json_client.bucket(_RAPID_BUCKET) blob = bucket.blob(object_name) blob.delete() @@ -173,15 +175,15 @@ def test_storage_open_object_read_full_object( ) event_loop.run_until_complete( storage_open_object_read_full_object.storage_open_object_read_full_object( - _ZONAL_BUCKET, object_name, grpc_client=async_grpc_client + _RAPID_BUCKET, object_name, grpc_client=async_grpc_client ) ) out, _ = capsys.readouterr() assert ( - f"Downloaded all {len(data)} bytes from object {object_name} in bucket {_ZONAL_BUCKET}." + f"Downloaded all {len(data)} bytes from object {object_name} in bucket {_RAPID_BUCKET}." in out ) - blob = json_client.bucket(_ZONAL_BUCKET).blob(object_name) + blob = json_client.bucket(_RAPID_BUCKET).blob(object_name) blob.delete() @@ -197,7 +199,7 @@ def test_storage_open_object_single_ranged_read( download_size = 5 event_loop.run_until_complete( storage_open_object_single_ranged_read.storage_open_object_single_ranged_read( - _ZONAL_BUCKET, + _RAPID_BUCKET, object_name, start_byte=0, size=download_size, @@ -206,7 +208,7 @@ def test_storage_open_object_single_ranged_read( ) out, _ = capsys.readouterr() assert f"Downloaded {download_size} bytes from {object_name}" in out - blob = json_client.bucket(_ZONAL_BUCKET).blob(object_name) + blob = json_client.bucket(_RAPID_BUCKET).blob(object_name) blob.delete() @@ -220,7 +222,7 @@ def test_storage_open_object_multiple_ranged_read( ) event_loop.run_until_complete( storage_open_object_multiple_ranged_read.storage_open_object_multiple_ranged_read( - _ZONAL_BUCKET, object_name, grpc_client=async_grpc_client + _RAPID_BUCKET, object_name, grpc_client=async_grpc_client ) ) out, _ = capsys.readouterr() @@ -228,7 +230,7 @@ def test_storage_open_object_multiple_ranged_read( assert "Downloaded 10 bytes into buffer 2 from start byte 20: b'aaaaaaaaaa'" in out assert "Downloaded 10 bytes into buffer 3 from start byte 40: b'aaaaaaaaaa'" in out assert "Downloaded 10 bytes into buffer 4 from start byte 60: b'aaaaaaaaaa'" in out - blob = json_client.bucket(_ZONAL_BUCKET).blob(object_name) + blob = json_client.bucket(_RAPID_BUCKET).blob(object_name) blob.delete() @@ -248,14 +250,14 @@ def test_storage_open_multiple_objects_ranged_read( event_loop.run_until_complete( storage_open_multiple_objects_ranged_read.storage_open_multiple_objects_ranged_read( - _ZONAL_BUCKET, [blob1_name, blob2_name], grpc_client=async_grpc_client + _RAPID_BUCKET, [blob1_name, blob2_name], grpc_client=async_grpc_client ) ) out, _ = capsys.readouterr() assert f"Downloaded {len(data1)} bytes from {blob1_name}" in out assert f"Downloaded {len(data2)} bytes from {blob2_name}" in out - blob1 = json_client.bucket(_ZONAL_BUCKET).blob(blob1_name) - blob2 = json_client.bucket(_ZONAL_BUCKET).blob(blob2_name) + blob1 = json_client.bucket(_RAPID_BUCKET).blob(blob1_name) + blob2 = json_client.bucket(_RAPID_BUCKET).blob(blob2_name) blob1.delete() blob2.delete() @@ -266,11 +268,11 @@ def test_storage_optimize_write_latency_pool( key_prefix = f"test-writer-pool-{uuid.uuid4()}" event_loop.run_until_complete( storage_optimize_write_latency_pool.storage_optimize_write_latency_pool( - _ZONAL_BUCKET, key_prefix, pool_size=3, grpc_client=async_grpc_client + _RAPID_BUCKET, key_prefix, pool_size=3, grpc_client=async_grpc_client ) ) out, _ = capsys.readouterr() assert f"Read unfinalized object {key_prefix}_0: 0123456789" in out - bucket = json_client.bucket(_ZONAL_BUCKET) + bucket = json_client.bucket(_RAPID_BUCKET) for i in range(4): bucket.blob(f"{key_prefix}_{i}").delete() diff --git a/storage/samples/snippets/zonal_buckets/storage_create_and_write_appendable_object.py b/storage/samples/snippets/bidi/storage_create_and_write_appendable_object.py similarity index 97% rename from storage/samples/snippets/zonal_buckets/storage_create_and_write_appendable_object.py rename to storage/samples/snippets/bidi/storage_create_and_write_appendable_object.py index 725eeb2bd98..94e46a4e931 100644 --- a/storage/samples/snippets/zonal_buckets/storage_create_and_write_appendable_object.py +++ b/storage/samples/snippets/bidi/storage_create_and_write_appendable_object.py @@ -29,7 +29,7 @@ async def storage_create_and_write_appendable_object( bucket_name, object_name, grpc_client=None ): - """Uploads an appendable object to zonal bucket. + """Uploads an appendable object to rapid bucket. grpc_client: an existing grpc_client to use, this is only for testing. """ diff --git a/storage/samples/snippets/zonal_buckets/storage_finalize_appendable_object_upload.py b/storage/samples/snippets/bidi/storage_finalize_appendable_object_upload.py similarity index 100% rename from storage/samples/snippets/zonal_buckets/storage_finalize_appendable_object_upload.py rename to storage/samples/snippets/bidi/storage_finalize_appendable_object_upload.py diff --git a/storage/samples/snippets/zonal_buckets/storage_open_multiple_objects_ranged_read.py b/storage/samples/snippets/bidi/storage_open_multiple_objects_ranged_read.py similarity index 97% rename from storage/samples/snippets/zonal_buckets/storage_open_multiple_objects_ranged_read.py rename to storage/samples/snippets/bidi/storage_open_multiple_objects_ranged_read.py index bed580d3662..0f7d83760bd 100644 --- a/storage/samples/snippets/zonal_buckets/storage_open_multiple_objects_ranged_read.py +++ b/storage/samples/snippets/bidi/storage_open_multiple_objects_ranged_read.py @@ -16,7 +16,7 @@ """Downloads a range of bytes from multiple objects concurrently. Example usage: - ```python samples/snippets/zonal_buckets/storage_open_multiple_objects_ranged_read.py \ + ```python samples/snippets/bidi/storage_open_multiple_objects_ranged_read.py \ --bucket_name \ --object_names ``` """ diff --git a/storage/samples/snippets/zonal_buckets/storage_open_object_multiple_ranged_read.py b/storage/samples/snippets/bidi/storage_open_object_multiple_ranged_read.py similarity index 100% rename from storage/samples/snippets/zonal_buckets/storage_open_object_multiple_ranged_read.py rename to storage/samples/snippets/bidi/storage_open_object_multiple_ranged_read.py diff --git a/storage/samples/snippets/zonal_buckets/storage_open_object_read_full_object.py b/storage/samples/snippets/bidi/storage_open_object_read_full_object.py similarity index 100% rename from storage/samples/snippets/zonal_buckets/storage_open_object_read_full_object.py rename to storage/samples/snippets/bidi/storage_open_object_read_full_object.py diff --git a/storage/samples/snippets/zonal_buckets/storage_open_object_single_ranged_read.py b/storage/samples/snippets/bidi/storage_open_object_single_ranged_read.py similarity index 100% rename from storage/samples/snippets/zonal_buckets/storage_open_object_single_ranged_read.py rename to storage/samples/snippets/bidi/storage_open_object_single_ranged_read.py diff --git a/storage/samples/snippets/zonal_buckets/storage_optimize_write_latency_pool.py b/storage/samples/snippets/bidi/storage_optimize_write_latency_pool.py similarity index 95% rename from storage/samples/snippets/zonal_buckets/storage_optimize_write_latency_pool.py rename to storage/samples/snippets/bidi/storage_optimize_write_latency_pool.py index b37f96aaaf2..1d0556e9fda 100644 --- a/storage/samples/snippets/zonal_buckets/storage_optimize_write_latency_pool.py +++ b/storage/samples/snippets/bidi/storage_optimize_write_latency_pool.py @@ -32,11 +32,11 @@ async def storage_optimize_write_latency_pool( bucket_name: str, key_prefix: str, pool_size: int = 3, grpc_client=None ): - """Uses a pre-warmed pool of writers for a zonal bucket. + """Uses a pre-warmed pool of writers for appendable objects in a rapid bucket. grpc_client: an existing grpc_client to use, this is only for testing. """ - # The ID of your GCS zonal bucket + # The ID of your GCS rapid bucket # bucket_name = "your-unique-bucket-name" # The prefix for your pooled GCS objects @@ -117,7 +117,7 @@ async def maintain_pool( formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( - "--bucket_name", help="Your Cloud Storage zonal bucket name." + "--bucket_name", help="Your Cloud Storage rapid bucket name." ) parser.add_argument( "--key_prefix", help="Prefix for pooled object names." diff --git a/storage/samples/snippets/zonal_buckets/storage_pause_and_resume_appendable_upload.py b/storage/samples/snippets/bidi/storage_pause_and_resume_appendable_upload.py similarity index 100% rename from storage/samples/snippets/zonal_buckets/storage_pause_and_resume_appendable_upload.py rename to storage/samples/snippets/bidi/storage_pause_and_resume_appendable_upload.py diff --git a/storage/samples/snippets/zonal_buckets/storage_read_appendable_object_tail.py b/storage/samples/snippets/bidi/storage_read_appendable_object_tail.py similarity index 100% rename from storage/samples/snippets/zonal_buckets/storage_read_appendable_object_tail.py rename to storage/samples/snippets/bidi/storage_read_appendable_object_tail.py diff --git a/storage/samples/snippets/zonal_buckets/README.md b/storage/samples/snippets/zonal_buckets/README.md deleted file mode 100644 index 21811697e4b..00000000000 --- a/storage/samples/snippets/zonal_buckets/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# Google Cloud Storage - Zonal Buckets Snippets - -This directory contains snippets for interacting with Google Cloud Storage zonal buckets. - -## Prerequisites - -- A Google Cloud Platform project with the Cloud Storage API enabled. -- A zonal Google Cloud Storage bucket. - -## Running the snippets - -### Create and write to an appendable object - -This snippet uploads an appendable object to a zonal bucket. - -```bash -python samples/snippets/zonal_buckets/storage_create_and_write_appendable_object.py --bucket_name --object_name -``` - -### Finalize an appendable object upload - -This snippet creates, writes to, and finalizes an appendable object. - -```bash -python samples/snippets/zonal_buckets/storage_finalize_appendable_object_upload.py --bucket_name --object_name -``` - -### Pause and resume an appendable object upload - -This snippet demonstrates pausing and resuming an appendable object upload. - -```bash -python samples/snippets/zonal_buckets/storage_pause_and_resume_appendable_upload.py --bucket_name --object_name -``` - -### Tail an appendable object - -This snippet demonstrates tailing an appendable GCS object, similar to `tail -f`. - -```bash -python samples/snippets/zonal_buckets/storage_read_appendable_object_tail.py --bucket_name --object_name --duration -``` - - -### Download a range of bytes from an object - -This snippet downloads a range of bytes from an object. - -```bash -python samples/snippets/zonal_buckets/storage_open_object_single_ranged_read.py --bucket_name --object_name --start_byte --size -``` - - -### Download multiple ranges of bytes from a single object - -This snippet downloads multiple ranges of bytes from a single object into different buffers. - -```bash -python samples/snippets/zonal_buckets/storage_open_object_multiple_ranged_read.py --bucket_name --object_name -``` - -### Download the entire content of an object - -This snippet downloads the entire content of an object using a multi-range downloader. - -```bash -python samples/snippets/zonal_buckets/storage_open_object_read_full_object.py --bucket_name --object_name -``` - - - -### Download a range of bytes from multiple objects concurrently - -This snippet downloads a range of bytes from multiple objects concurrently. - -```bash -python samples/snippets/zonal_buckets/storage_open_multiple_objects_ranged_read.py --bucket_name --object_names -``` - -### Optimize write latency with a pre-warmed writer pool - -This snippet uses a pre-warmed pool of writers for a zonal bucket. - -```bash -python samples/snippets/zonal_buckets/storage_optimize_write_latency_pool.py --bucket_name --key_prefix -``` \ No newline at end of file