diff --git a/google-cloud-storage/lib/google/cloud/storage/bucket.rb b/google-cloud-storage/lib/google/cloud/storage/bucket.rb index ca8461354bb7..6d71edf54e77 100644 --- a/google-cloud-storage/lib/google/cloud/storage/bucket.rb +++ b/google-cloud-storage/lib/google/cloud/storage/bucket.rb @@ -1628,7 +1628,7 @@ def file path, # changed to a time in the future. If custom_time must be unset, you # must either perform a rewrite operation, or upload the data again # and create a new file. - # @param [Symbol, nil] checksum The type of checksum for the client to + # @param [Symbol, nil, Boolean] checksum The type of checksum for the client to # automatically calculate and send with the create request to verify # the integrity of the object. If provided, Cloud Storage will only # create the file if the value calculated by the client matches the @@ -1636,11 +1636,12 @@ def file path, # # Acceptable values are: # + # * `true` [Boolean] - Calculate and provide a checksum using the CRC32c hash. + # * `false` [Boolean] - Do not calculate or provide a checksum. # * `md5` - Calculate and provide a checksum using the MD5 hash. # * `crc32c` - Calculate and provide a checksum using the CRC32c hash. # * `all` - Calculate and provide checksums for all available verifications. - # - # Optional. The default is `nil`. Do not provide if also providing a + # Optional. The default is `crc32c`. Do not provide if also providing a # corresponding `crc32c` or `md5` argument. See # [Validation](https://cloud.google.com/storage/docs/hashes-etags) # for more information. @@ -1805,6 +1806,11 @@ def create_file file, path ||= file.path if file.respond_to? :path path ||= file if file.is_a? String raise ArgumentError, "must provide path" if path.nil? + # If no checksum type or specific value is provided, the default will be set to crc32c. + # If the checksum is set to false, it will be disabled. + if [checksum, crc32c, md5].all?(&:nil?) || checksum == true + checksum = :crc32c + end crc32c = crc32c_for file, checksum, crc32c md5 = md5_for file, checksum, md5 diff --git a/google-cloud-storage/lib/google/cloud/storage/file/verifier.rb b/google-cloud-storage/lib/google/cloud/storage/file/verifier.rb index dcfefc6c616d..c65f33bb7c08 100644 --- a/google-cloud-storage/lib/google/cloud/storage/file/verifier.rb +++ b/google-cloud-storage/lib/google/cloud/storage/file/verifier.rb @@ -49,29 +49,47 @@ def self.verify_crc32c gcloud_file, local_file gcloud_file.crc32c == crc32c_for(local_file) end + # Calculates MD5 digest using either file path or open stream. def self.md5_for local_file - if local_file.respond_to? :to_path - ::File.open Pathname(local_file).to_path, "rb" do |f| - ::Digest::MD5.file(f).base64digest - end - else # StringIO - local_file.rewind - md5 = ::Digest::MD5.base64digest local_file.read - local_file.rewind - md5 - end + _digest_for local_file, ::Digest::MD5 end + # Calculates CRC32c digest using either file path or open stream. def self.crc32c_for local_file - if local_file.respond_to? :to_path + _digest_for local_file, ::Digest::CRC32c + end + + # @private + # Computes a base64-encoded digest for a local file or IO stream. + # + # This method handles two types of inputs for `local_file`: + # 1. A file path (String or Pathname): It efficiently streams the file + # to compute the digest without loading the entire file into memory. + # 2. An IO-like stream (e.g., File, StringIO): It reads the stream's + # content to compute the digest. The stream is rewound before and after + # reading to ensure its position is not permanently changed. + # + # @param local_file [String, Pathname, IO] The local file path or IO + # stream for which to compute the digest. + # @param digest_class [Class] The digest class to use for the + # calculation (e.g., `Digest::MD5`). It must respond to `.file` and + # `.base64digest`. + # + # @return [String] The base64-encoded digest of the file's content. + # + def self._digest_for local_file, digest_class + + if local_file.respond_to?(:to_path) || local_file.is_a?(String) + # Case 1: Input is a file path (String, Pathname, or object that responds to :to_path). ::File.open Pathname(local_file).to_path, "rb" do |f| - ::Digest::CRC32c.file(f).base64digest + digest_class.file(f).base64digest end - else # StringIO + else + # Case 2: Input is an open stream (File or StringIO). local_file.rewind - crc32c = ::Digest::CRC32c.base64digest local_file.read + digest = digest_class.base64digest local_file.read local_file.rewind - crc32c + digest end end end diff --git a/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb b/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb index ce617da110e6..be834f311492 100644 --- a/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb @@ -130,7 +130,12 @@ def create_file_gapi bucket=nil, name = nil def empty_file_gapi cache_control: nil, content_disposition: nil, content_encoding: nil, content_language: nil, content_type: nil, crc32c: nil, md5: nil, metadata: nil, - storage_class: nil + storage_class: nil, checksum: nil, content: nil + + # If no checksum type or specific value is provided, the default will be set to crc32c. + # If the checksum is set to false, it will be disabled. + crc32c ||= set_crc32c_as_default md5, crc32c, checksum, content + params = { cache_control: cache_control, content_type: content_type, content_disposition: content_disposition, md5_hash: md5, diff --git a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb index 203e423467e5..71447a5f6f24 100644 --- a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb @@ -101,6 +101,40 @@ _(bucket_complete.autoclass_enabled).must_equal bucket_autoclass_enabled _(bucket_complete.autoclass_terminal_storage_class).must_equal bucket_autoclass_terminal_storage_class end + + it "creates a file with checksum: :crc32c by default" do + new_file_name = random_file_path + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world!" + tmpfile.rewind + + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(content: tmpfile.read)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + + bucket.service.mocked_service = mock + bucket.create_file tmpfile, new_file_name + + mock.verify + end + end + + it "creates a file with a StringIO and checksum: :crc32c by default" do + new_file_name = random_file_path + new_file_contents = StringIO.new "Hello world" + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for new_file_contents + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(content: new_file_contents.read)], **insert_object_args(name: new_file_name, upload_source: new_file_contents, options: {retries: 0}) + + bucket.service.mocked_service = mock + + bucket.create_file new_file_contents, new_file_name + + mock.verify + end it "returns frozen cors" do bucket_complete.cors.each do |cors| @@ -405,6 +439,42 @@ end end + it "creates a file with no checksum" do + new_file_name = random_file_path + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world!" + tmpfile.rewind + + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(checksum: false)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + + bucket.service.mocked_service = mock + + bucket.create_file tmpfile, new_file_name, checksum: false + mock.verify + end + end + + it "creates a file with crc32c if checksum is true" do + new_file_name = random_file_path + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world!" + tmpfile.rewind + + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(checksum: true, crc32c: "e5jnUQ==")], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + bucket.service.mocked_service = mock + + bucket.create_file tmpfile, new_file_name, checksum: true + + mock.verify + end + end + it "creates a file with attributes" do new_file_name = random_file_path @@ -595,9 +665,11 @@ new_file_name = random_file_path Tempfile.create ["google-cloud", ".txt"] do |tmpfile| + + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile mock = Minitest::Mock.new mock.expect :insert_object, create_file_gapi(bucket_user_project.name, new_file_name), - [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) + [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) bucket_user_project.service.mocked_service = mock @@ -608,13 +680,13 @@ end end - it "creates an file with a StringIO" do + it "creates a file with StringIO" do new_file_name = random_file_path - new_file_contents = StringIO.new - + new_file_contents = StringIO.new("Hello world string_io") + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for new_file_contents mock = Minitest::Mock.new mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), - [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: new_file_contents, options: {retries: 0}) + [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: new_file_contents, options: {retries: 0}) bucket.service.mocked_service = mock @@ -1416,7 +1488,11 @@ def empty_file_gapi cache_control: nil, content_disposition: nil, content_encoding: nil, content_language: nil, content_type: nil, crc32c: nil, md5: nil, metadata: nil, storage_class: nil, temporary_hold: nil, - event_based_hold: nil + event_based_hold: nil, checksum: nil, content: nil + + # If no checksum type or specific value is provided, the default will be set to crc32c. + # If the checksum is set to false, it will be disabled. + crc32c ||= set_crc32c_as_default md5, crc32c, checksum, content params = { cache_control: cache_control, content_type: content_type, content_disposition: content_disposition, md5_hash: md5, @@ -1435,4 +1511,4 @@ def list_files_gapi count = 2, token = nil, prefixes = nil, include_folders_as_p files = count.times.map { Google::Apis::StorageV1::Object.from_json random_file_hash.to_json } Google::Apis::StorageV1::Objects.new kind: "storage#objects", items: files, next_page_token: token, prefixes: prefixes, include_folders_as_prefixes: include_folders_as_prefixes end -end +end \ No newline at end of file diff --git a/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb b/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb index c304f54b3838..359141e1b22a 100644 --- a/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb @@ -244,6 +244,62 @@ mock.verify end end + + it "creates a file with checksum: :crc32c by default" do + new_file_name = random_file_path + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world 123" + tmpfile.rewind + + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + + bucket.service.mocked_service = mock + bucket.create_file tmpfile, new_file_name + + mock.verify + end + end + + it "creates a file with no checksum" do + new_file_name = random_file_path + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world!" + tmpfile.rewind + + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(checksum: false, content: tmpfile.read)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + + bucket.service.mocked_service = mock + + bucket.create_file tmpfile, new_file_name, checksum: false + mock.verify + end + end + + it "creates a file with crc32c if checksum is true" do + new_file_name = random_file_path + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world!" + tmpfile.rewind + + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(checksum: true, crc32c: "e5jnUQ==")], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + + bucket.service.mocked_service = mock + + bucket.create_file tmpfile, new_file_name, checksum: true + + mock.verify + end + end it "creates a file with attributes" do new_file_name = random_file_path @@ -279,7 +335,6 @@ Tempfile.open ["google-cloud", ".txt"] do |tmpfile| tmpfile.write "Hello world" tmpfile.rewind - metadata = { "player" => "Bob", score: 10 @@ -340,9 +395,10 @@ new_file_name = random_file_path Tempfile.create ["google-cloud", ".txt"] do |tmpfile| + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile mock = Minitest::Mock.new mock.expect :insert_object, create_file_gapi(bucket_user_project.name, new_file_name), - [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) + [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) bucket_user_project.service.mocked_service = mock @@ -1090,7 +1146,11 @@ def create_file_gapi bucket=nil, name = nil def empty_file_gapi cache_control: nil, content_disposition: nil, content_encoding: nil, content_language: nil, content_type: nil, crc32c: nil, md5: nil, metadata: nil, - storage_class: nil + storage_class: nil, checksum: nil, content: nil + + # If no checksum type or specific value is provided, the default will be set to crc32c. + # If the checksum is set to false, it will be disabled. + crc32c ||= set_crc32c_as_default md5, crc32c, checksum, content params = { cache_control: cache_control, content_type: content_type, content_disposition: content_disposition, md5_hash: md5, diff --git a/google-cloud-storage/test/helper.rb b/google-cloud-storage/test/helper.rb index 5d37f92feb26..99f11e464010 100644 --- a/google-cloud-storage/test/helper.rb +++ b/google-cloud-storage/test/helper.rb @@ -612,4 +612,14 @@ def restore_file_gapi bucket, file_name, generation=nil file_hash = random_file_hash(bucket, file_name, generation).to_json Google::Apis::StorageV1::Object.from_json file_hash end + + def set_crc32c_as_default md5, crc32c, checksum, content = nil + # If no checksum type or specific value is provided, the default will be set to crc32c. + # If the checksum is set to false, it will be disabled. + if [checksum, crc32c, md5].all?(&:nil?) || checksum == true + # if content is present and crc32c is not provided, calculate crc32c based on content + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for(StringIO.new(content || "Hello world")) + end + crc32c + end end