diff --git a/CHANGES.rst b/CHANGES.rst index bb38bc214e..88b983e4bf 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -34,6 +34,16 @@ Compute (#2147) [Miguel Caballer - @micafer] +Storage +~~~~~~~ + +- [Azure Blobs] Fix ``chunk_size`` argument being ignored by + ``download_object_as_stream`` and ``download_object_range_as_stream``. The + requested chunk size is now forwarded to the underlying iterator instead of + always using ``AZURE_DOWNLOAD_CHUNK_SIZE``. + (GITHUB-1698) + [Sanjay Santhanam - @Sanjays2402] + Changes in Apache Libcloud 3.9.1 -------------------------------- diff --git a/libcloud/storage/drivers/azure_blobs.py b/libcloud/storage/drivers/azure_blobs.py index adaf3d81fa..dfb686f82f 100644 --- a/libcloud/storage/drivers/azure_blobs.py +++ b/libcloud/storage/drivers/azure_blobs.py @@ -793,7 +793,7 @@ def download_object_as_stream(self, obj, chunk_size=None): """ obj_path = self._get_object_path(obj.container, obj.name) response = self.connection.request(obj_path, method="GET", stream=True, raw=True) - iterator = response.iter_content(AZURE_DOWNLOAD_CHUNK_SIZE) + iterator = response.iter_content(chunk_size or AZURE_DOWNLOAD_CHUNK_SIZE) return self._get_object( obj=obj, @@ -848,7 +848,7 @@ def download_object_range_as_stream(self, obj, start_bytes, end_bytes=None, chun response = self.connection.request( obj_path, method="GET", headers=headers, stream=True, raw=True ) - iterator = response.iter_content(AZURE_DOWNLOAD_CHUNK_SIZE) + iterator = response.iter_content(chunk_size or AZURE_DOWNLOAD_CHUNK_SIZE) success_status_codes = [httplib.OK, httplib.PARTIAL_CONTENT] return self._get_object( diff --git a/libcloud/test/storage/test_azure_blobs.py b/libcloud/test/storage/test_azure_blobs.py index cf8c06614d..0598a254fb 100644 --- a/libcloud/test/storage/test_azure_blobs.py +++ b/libcloud/test/storage/test_azure_blobs.py @@ -697,6 +697,40 @@ def test_download_object_as_stream_success(self): consumed_stream = "".join(chunk.decode("utf-8") for chunk in stream) self.assertEqual(len(consumed_stream), obj.size) + def test_download_object_as_stream_uses_chunk_size(self): + container = Container(name="foo_bar_container", extra={}, driver=self.driver) + + obj = Object( + name="foo_bar_object", + size=1000, + hash=None, + extra={}, + container=container, + meta_data=None, + driver=self.driver_type, + ) + + used_chunk_sizes = [] + + def mock_get_object( + self, obj, callback, callback_kwargs, response, success_status_code=None + ): + iterator = callback_kwargs["iterator"] + used_chunk_sizes.append(iterator.gi_frame.f_locals.get("chunk_size")) + return iterator + + old_func = self.driver_type._get_object + self.driver_type._get_object = mock_get_object + try: + self.driver.download_object_as_stream(obj=obj, chunk_size=1234) + self.driver.download_object_range_as_stream( + obj=obj, start_bytes=0, end_bytes=10, chunk_size=4321 + ) + finally: + self.driver_type._get_object = old_func + + self.assertEqual(used_chunk_sizes, [1234, 4321]) + def test_download_object_range_success(self): container = Container(name="foo_bar_container", extra={}, driver=self.driver) obj = Object(