diff --git a/CHANGES.rst b/CHANGES.rst index bb38bc214e..a7fc6b4fec 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -34,6 +34,16 @@ Compute (#2147) [Miguel Caballer - @micafer] +Storage +~~~~~~~ + +- [S3] Fix ``chunk_size`` argument being ignored by + ``download_object_as_stream`` and ``download_object_range_as_stream``. The + requested chunk size is now passed through to the underlying iterator + instead of the hardcoded default. + (GITHUB-1798) + [Sanjay Santhanam - @Sanjays2402] + Changes in Apache Libcloud 3.9.1 -------------------------------- diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index bd9212ea50..4b59e2df9b 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -521,7 +521,7 @@ def download_object_as_stream(self, obj, chunk_size=None): callback=read_in_chunks, response=response, callback_kwargs={ - "iterator": response.iter_content(CHUNK_SIZE), + "iterator": response.iter_content(chunk_size or CHUNK_SIZE), "chunk_size": chunk_size, }, success_status_code=httplib.OK, @@ -573,7 +573,7 @@ def download_object_range_as_stream(self, obj, start_bytes, end_bytes=None, chun callback=read_in_chunks, response=response, callback_kwargs={ - "iterator": response.iter_content(CHUNK_SIZE), + "iterator": response.iter_content(chunk_size or CHUNK_SIZE), "chunk_size": chunk_size, }, success_status_code=httplib.PARTIAL_CONTENT, diff --git a/libcloud/test/storage/test_s3.py b/libcloud/test/storage/test_s3.py index be70898c84..406a5fa356 100644 --- a/libcloud/test/storage/test_s3.py +++ b/libcloud/test/storage/test_s3.py @@ -861,6 +861,30 @@ def mock_get_object( finally: self.driver_type._get_object = old_func + def test_download_object_as_stream_uses_chunk_size(self): + # Regression test: the chunk_size passed by the caller must be + # forwarded to iter_content instead of the hardcoded CHUNK_SIZE. + container = Container(name="foo_bar_container", extra={}, driver=self.driver) + obj = Object( + name="foo_bar_object", + size=1000, + hash=None, + extra={}, + container=container, + meta_data=None, + driver=self.driver_type, + ) + + requested_chunk_size = CHUNK_SIZE * 2 + mock_response = Mock(name="mock response") + mock_response.iter_content.return_value = iter([b"a"]) + + with mock.patch.object(self.driver.connection, "request", return_value=mock_response): + with mock.patch.object(self.driver, "_get_object", side_effect=lambda **kw: kw): + self.driver.download_object_as_stream(obj=obj, chunk_size=requested_chunk_size) + + mock_response.iter_content.assert_called_once_with(requested_chunk_size) + def test_upload_object_invalid_ex_storage_class(self): # Invalid hash is detected on the amazon side and BAD_REQUEST is # returned