From 5b42705e2465b78d91ffc6c0fb8e9a4456575238 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Mon, 2 Feb 2026 12:16:46 +0000 Subject: [PATCH 1/3] merging commits and main branch --- .../lib/google/cloud/storage/bucket.rb | 12 ++- .../lib/google/cloud/storage/file/verifier.rb | 48 ++++++---- .../cloud/storage/bucket_encryption_test.rb | 7 +- .../test/google/cloud/storage/bucket_test.rb | 89 +++++++++++++++++-- .../google/cloud/storage/lazy/bucket_test.rb | 66 +++++++++++++- google-cloud-storage/test/helper.rb | 8 ++ 6 files changed, 202 insertions(+), 28 deletions(-) diff --git a/google-cloud-storage/lib/google/cloud/storage/bucket.rb b/google-cloud-storage/lib/google/cloud/storage/bucket.rb index ca8461354bb7..6d71edf54e77 100644 --- a/google-cloud-storage/lib/google/cloud/storage/bucket.rb +++ b/google-cloud-storage/lib/google/cloud/storage/bucket.rb @@ -1628,7 +1628,7 @@ def file path, # changed to a time in the future. If custom_time must be unset, you # must either perform a rewrite operation, or upload the data again # and create a new file. - # @param [Symbol, nil] checksum The type of checksum for the client to + # @param [Symbol, nil, Boolean] checksum The type of checksum for the client to # automatically calculate and send with the create request to verify # the integrity of the object. If provided, Cloud Storage will only # create the file if the value calculated by the client matches the @@ -1636,11 +1636,12 @@ def file path, # # Acceptable values are: # + # * `true` [Boolean] - Calculate and provide a checksum using the CRC32c hash. + # * `false` [Boolean] - Do not calculate or provide a checksum. # * `md5` - Calculate and provide a checksum using the MD5 hash. # * `crc32c` - Calculate and provide a checksum using the CRC32c hash. # * `all` - Calculate and provide checksums for all available verifications. - # - # Optional. The default is `nil`. Do not provide if also providing a + # Optional. The default is `crc32c`. Do not provide if also providing a # corresponding `crc32c` or `md5` argument. See # [Validation](https://cloud.google.com/storage/docs/hashes-etags) # for more information. @@ -1805,6 +1806,11 @@ def create_file file, path ||= file.path if file.respond_to? :path path ||= file if file.is_a? String raise ArgumentError, "must provide path" if path.nil? + # If no checksum type or specific value is provided, the default will be set to crc32c. + # If the checksum is set to false, it will be disabled. + if [checksum, crc32c, md5].all?(&:nil?) || checksum == true + checksum = :crc32c + end crc32c = crc32c_for file, checksum, crc32c md5 = md5_for file, checksum, md5 diff --git a/google-cloud-storage/lib/google/cloud/storage/file/verifier.rb b/google-cloud-storage/lib/google/cloud/storage/file/verifier.rb index dcfefc6c616d..c65f33bb7c08 100644 --- a/google-cloud-storage/lib/google/cloud/storage/file/verifier.rb +++ b/google-cloud-storage/lib/google/cloud/storage/file/verifier.rb @@ -49,29 +49,47 @@ def self.verify_crc32c gcloud_file, local_file gcloud_file.crc32c == crc32c_for(local_file) end + # Calculates MD5 digest using either file path or open stream. def self.md5_for local_file - if local_file.respond_to? :to_path - ::File.open Pathname(local_file).to_path, "rb" do |f| - ::Digest::MD5.file(f).base64digest - end - else # StringIO - local_file.rewind - md5 = ::Digest::MD5.base64digest local_file.read - local_file.rewind - md5 - end + _digest_for local_file, ::Digest::MD5 end + # Calculates CRC32c digest using either file path or open stream. def self.crc32c_for local_file - if local_file.respond_to? :to_path + _digest_for local_file, ::Digest::CRC32c + end + + # @private + # Computes a base64-encoded digest for a local file or IO stream. + # + # This method handles two types of inputs for `local_file`: + # 1. A file path (String or Pathname): It efficiently streams the file + # to compute the digest without loading the entire file into memory. + # 2. An IO-like stream (e.g., File, StringIO): It reads the stream's + # content to compute the digest. The stream is rewound before and after + # reading to ensure its position is not permanently changed. + # + # @param local_file [String, Pathname, IO] The local file path or IO + # stream for which to compute the digest. + # @param digest_class [Class] The digest class to use for the + # calculation (e.g., `Digest::MD5`). It must respond to `.file` and + # `.base64digest`. + # + # @return [String] The base64-encoded digest of the file's content. + # + def self._digest_for local_file, digest_class + + if local_file.respond_to?(:to_path) || local_file.is_a?(String) + # Case 1: Input is a file path (String, Pathname, or object that responds to :to_path). ::File.open Pathname(local_file).to_path, "rb" do |f| - ::Digest::CRC32c.file(f).base64digest + digest_class.file(f).base64digest end - else # StringIO + else + # Case 2: Input is an open stream (File or StringIO). local_file.rewind - crc32c = ::Digest::CRC32c.base64digest local_file.read + digest = digest_class.base64digest local_file.read local_file.rewind - crc32c + digest end end end diff --git a/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb b/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb index ce617da110e6..e7c10e8b5e04 100644 --- a/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb @@ -130,7 +130,12 @@ def create_file_gapi bucket=nil, name = nil def empty_file_gapi cache_control: nil, content_disposition: nil, content_encoding: nil, content_language: nil, content_type: nil, crc32c: nil, md5: nil, metadata: nil, - storage_class: nil + storage_class: nil, checksum: nil + + # If no checksum type or specific value is provided, the default will be set to crc32c. + # If the checksum is set to false, it will be disabled. + crc32c ||= set_crc32c_as_default md5, crc32c, checksum + params = { cache_control: cache_control, content_type: content_type, content_disposition: content_disposition, md5_hash: md5, diff --git a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb index 203e423467e5..340283319ff0 100644 --- a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb @@ -101,6 +101,41 @@ _(bucket_complete.autoclass_enabled).must_equal bucket_autoclass_enabled _(bucket_complete.autoclass_terminal_storage_class).must_equal bucket_autoclass_terminal_storage_class end + + it "creates a file with checksum: :crc32c by default" do + new_file_name = random_file_path + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world!" + tmpfile.rewind + + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile + + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + + bucket.service.mocked_service = mock + bucket.create_file tmpfile, new_file_name + + mock.verify + end + end + + it "creates a file with a StringIO and checksum: :crc32c by default" do + new_file_name = random_file_path + new_file_contents = StringIO.new "Hello world" + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for new_file_contents + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: new_file_contents, options: {retries: 0}) + + bucket.service.mocked_service = mock + + bucket.create_file new_file_contents, new_file_name + + mock.verify + end it "returns frozen cors" do bucket_complete.cors.each do |cors| @@ -405,6 +440,42 @@ end end + it "creates a file with no checksum" do + new_file_name = random_file_path + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world!" + tmpfile.rewind + + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(checksum: false)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + + bucket.service.mocked_service = mock + + bucket.create_file tmpfile, new_file_name, checksum: false + mock.verify + end + end + + it "creates a file with crc32c if checksum is true" do + new_file_name = random_file_path + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world!" + tmpfile.rewind + + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(checksum: true, crc32c: "e5jnUQ==")], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + bucket.service.mocked_service = mock + + bucket.create_file tmpfile, new_file_name, checksum: true + + mock.verify + end + end + it "creates a file with attributes" do new_file_name = random_file_path @@ -595,9 +666,11 @@ new_file_name = random_file_path Tempfile.create ["google-cloud", ".txt"] do |tmpfile| + + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile mock = Minitest::Mock.new mock.expect :insert_object, create_file_gapi(bucket_user_project.name, new_file_name), - [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) + [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) bucket_user_project.service.mocked_service = mock @@ -608,13 +681,13 @@ end end - it "creates an file with a StringIO" do + it "creates a file with StringIO" do new_file_name = random_file_path - new_file_contents = StringIO.new - + new_file_contents = StringIO.new("Hello world string_io") + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for new_file_contents mock = Minitest::Mock.new mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), - [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: new_file_contents, options: {retries: 0}) + [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: new_file_contents, options: {retries: 0}) bucket.service.mocked_service = mock @@ -1416,7 +1489,11 @@ def empty_file_gapi cache_control: nil, content_disposition: nil, content_encoding: nil, content_language: nil, content_type: nil, crc32c: nil, md5: nil, metadata: nil, storage_class: nil, temporary_hold: nil, - event_based_hold: nil + event_based_hold: nil, checksum: nil + + # If no checksum type or specific value is provided, the default will be set to crc32c. + # If the checksum is set to false, it will be disabled. + crc32c ||= set_crc32c_as_default md5, crc32c, checksum params = { cache_control: cache_control, content_type: content_type, content_disposition: content_disposition, md5_hash: md5, diff --git a/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb b/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb index c304f54b3838..ea8e93c7df94 100644 --- a/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb @@ -244,6 +244,62 @@ mock.verify end end + + it "creates a file with checksum: :crc32c by default" do + new_file_name = random_file_path + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world 123" + tmpfile.rewind + + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + + bucket.service.mocked_service = mock + bucket.create_file tmpfile, new_file_name + + mock.verify + end + end + + it "creates a file with no checksum" do + new_file_name = random_file_path + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world!" + tmpfile.rewind + + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(checksum: false)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + + bucket.service.mocked_service = mock + + bucket.create_file tmpfile, new_file_name, checksum: false + mock.verify + end + end + + it "creates a file with crc32c if checksum is true" do + new_file_name = random_file_path + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world!" + tmpfile.rewind + + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(checksum: true, crc32c: "e5jnUQ==")], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + + bucket.service.mocked_service = mock + + bucket.create_file tmpfile, new_file_name, checksum: true + + mock.verify + end + end it "creates a file with attributes" do new_file_name = random_file_path @@ -279,7 +335,6 @@ Tempfile.open ["google-cloud", ".txt"] do |tmpfile| tmpfile.write "Hello world" tmpfile.rewind - metadata = { "player" => "Bob", score: 10 @@ -340,9 +395,10 @@ new_file_name = random_file_path Tempfile.create ["google-cloud", ".txt"] do |tmpfile| + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile mock = Minitest::Mock.new mock.expect :insert_object, create_file_gapi(bucket_user_project.name, new_file_name), - [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) + [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) bucket_user_project.service.mocked_service = mock @@ -1090,7 +1146,11 @@ def create_file_gapi bucket=nil, name = nil def empty_file_gapi cache_control: nil, content_disposition: nil, content_encoding: nil, content_language: nil, content_type: nil, crc32c: nil, md5: nil, metadata: nil, - storage_class: nil + storage_class: nil, checksum: nil + + # If no checksum type or specific value is provided, the default will be set to crc32c. + # If the checksum is set to false, it will be disabled. + crc32c ||= set_crc32c_as_default md5, crc32c, checksum params = { cache_control: cache_control, content_type: content_type, content_disposition: content_disposition, md5_hash: md5, diff --git a/google-cloud-storage/test/helper.rb b/google-cloud-storage/test/helper.rb index 5d37f92feb26..1f635b8b62bb 100644 --- a/google-cloud-storage/test/helper.rb +++ b/google-cloud-storage/test/helper.rb @@ -612,4 +612,12 @@ def restore_file_gapi bucket, file_name, generation=nil file_hash = random_file_hash(bucket, file_name, generation).to_json Google::Apis::StorageV1::Object.from_json file_hash end + + def set_crc32c_as_default md5, crc32c, checksum + # If no checksum type or specific value is provided, the default will be set to crc32c. + # If the checksum is set to false, it will be disabled. if [checksum, crc32c, md5].all?(&:nil?) || checksum == true + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for(StringIO.new("Hello world")) + end + crc32c + end end From 8a693681d62736b37e7be7be1079c094b6fe6c3b Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Mon, 2 Feb 2026 12:22:10 +0000 Subject: [PATCH 2/3] fix --- google-cloud-storage/test/helper.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/google-cloud-storage/test/helper.rb b/google-cloud-storage/test/helper.rb index 1f635b8b62bb..44cbfb822ac5 100644 --- a/google-cloud-storage/test/helper.rb +++ b/google-cloud-storage/test/helper.rb @@ -614,8 +614,9 @@ def restore_file_gapi bucket, file_name, generation=nil end def set_crc32c_as_default md5, crc32c, checksum - # If no checksum type or specific value is provided, the default will be set to crc32c. - # If the checksum is set to false, it will be disabled. if [checksum, crc32c, md5].all?(&:nil?) || checksum == true + # If no checksum type or specific value is provided, the default will be set to crc32c. + # If the checksum is set to false, it will be disabled. + if [checksum, crc32c, md5].all?(&:nil?) || checksum == true crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for(StringIO.new("Hello world")) end crc32c From 902506ed404721a0827ee42b3ad14b08cee8af87 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Fri, 6 Feb 2026 11:32:07 +0000 Subject: [PATCH 3/3] sending content --- .../google/cloud/storage/bucket_encryption_test.rb | 4 ++-- .../test/google/cloud/storage/bucket_test.rb | 11 +++++------ .../test/google/cloud/storage/lazy/bucket_test.rb | 6 +++--- google-cloud-storage/test/helper.rb | 5 +++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb b/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb index e7c10e8b5e04..be834f311492 100644 --- a/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb @@ -130,11 +130,11 @@ def create_file_gapi bucket=nil, name = nil def empty_file_gapi cache_control: nil, content_disposition: nil, content_encoding: nil, content_language: nil, content_type: nil, crc32c: nil, md5: nil, metadata: nil, - storage_class: nil, checksum: nil + storage_class: nil, checksum: nil, content: nil # If no checksum type or specific value is provided, the default will be set to crc32c. # If the checksum is set to false, it will be disabled. - crc32c ||= set_crc32c_as_default md5, crc32c, checksum + crc32c ||= set_crc32c_as_default md5, crc32c, checksum, content params = { cache_control: cache_control, content_type: content_type, diff --git a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb index 340283319ff0..71447a5f6f24 100644 --- a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb @@ -110,10 +110,9 @@ tmpfile.rewind crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile - mock = Minitest::Mock.new mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), - [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + [bucket.name, empty_file_gapi(content: tmpfile.read)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) bucket.service.mocked_service = mock bucket.create_file tmpfile, new_file_name @@ -128,7 +127,7 @@ crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for new_file_contents mock = Minitest::Mock.new mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), - [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: new_file_contents, options: {retries: 0}) + [bucket.name, empty_file_gapi(content: new_file_contents.read)], **insert_object_args(name: new_file_name, upload_source: new_file_contents, options: {retries: 0}) bucket.service.mocked_service = mock @@ -1489,11 +1488,11 @@ def empty_file_gapi cache_control: nil, content_disposition: nil, content_encoding: nil, content_language: nil, content_type: nil, crc32c: nil, md5: nil, metadata: nil, storage_class: nil, temporary_hold: nil, - event_based_hold: nil, checksum: nil + event_based_hold: nil, checksum: nil, content: nil # If no checksum type or specific value is provided, the default will be set to crc32c. # If the checksum is set to false, it will be disabled. - crc32c ||= set_crc32c_as_default md5, crc32c, checksum + crc32c ||= set_crc32c_as_default md5, crc32c, checksum, content params = { cache_control: cache_control, content_type: content_type, content_disposition: content_disposition, md5_hash: md5, @@ -1512,4 +1511,4 @@ def list_files_gapi count = 2, token = nil, prefixes = nil, include_folders_as_p files = count.times.map { Google::Apis::StorageV1::Object.from_json random_file_hash.to_json } Google::Apis::StorageV1::Objects.new kind: "storage#objects", items: files, next_page_token: token, prefixes: prefixes, include_folders_as_prefixes: include_folders_as_prefixes end -end +end \ No newline at end of file diff --git a/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb b/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb index ea8e93c7df94..359141e1b22a 100644 --- a/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb @@ -273,7 +273,7 @@ mock = Minitest::Mock.new mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), - [bucket.name, empty_file_gapi(checksum: false)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + [bucket.name, empty_file_gapi(checksum: false, content: tmpfile.read)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) bucket.service.mocked_service = mock @@ -1146,11 +1146,11 @@ def create_file_gapi bucket=nil, name = nil def empty_file_gapi cache_control: nil, content_disposition: nil, content_encoding: nil, content_language: nil, content_type: nil, crc32c: nil, md5: nil, metadata: nil, - storage_class: nil, checksum: nil + storage_class: nil, checksum: nil, content: nil # If no checksum type or specific value is provided, the default will be set to crc32c. # If the checksum is set to false, it will be disabled. - crc32c ||= set_crc32c_as_default md5, crc32c, checksum + crc32c ||= set_crc32c_as_default md5, crc32c, checksum, content params = { cache_control: cache_control, content_type: content_type, content_disposition: content_disposition, md5_hash: md5, diff --git a/google-cloud-storage/test/helper.rb b/google-cloud-storage/test/helper.rb index 44cbfb822ac5..99f11e464010 100644 --- a/google-cloud-storage/test/helper.rb +++ b/google-cloud-storage/test/helper.rb @@ -613,11 +613,12 @@ def restore_file_gapi bucket, file_name, generation=nil Google::Apis::StorageV1::Object.from_json file_hash end - def set_crc32c_as_default md5, crc32c, checksum + def set_crc32c_as_default md5, crc32c, checksum, content = nil # If no checksum type or specific value is provided, the default will be set to crc32c. # If the checksum is set to false, it will be disabled. if [checksum, crc32c, md5].all?(&:nil?) || checksum == true - crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for(StringIO.new("Hello world")) + # if content is present and crc32c is not provided, calculate crc32c based on content + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for(StringIO.new(content || "Hello world")) end crc32c end