diff --git a/changes/3526.feature.md b/changes/3526.feature.md deleted file mode 100644 index 7746611855..0000000000 --- a/changes/3526.feature.md +++ /dev/null @@ -1 +0,0 @@ -Increased the default value of `async.concurrency` from 10 to 64 to improve parallelism and throughput for concurrent I/O operations. This change enables better performance out-of-the-box for most workloads. Users with specific resource constraints or when using many Dask threads may want to lower this value via the `ZARR_ASYNC_CONCURRENCY` environment variable or by setting `zarr.config.set({'async.concurrency': N})`. diff --git a/docs/user-guide/performance.md b/docs/user-guide/performance.md index 6760cf055c..c39ca7d25d 100644 --- a/docs/user-guide/performance.md +++ b/docs/user-guide/performance.md @@ -175,13 +175,18 @@ Coming soon. ## Parallel computing and synchronization -Zarr is designed to support parallel computing and enables concurrent reads and writes to arrays. This section covers how to optimize Zarr's concurrency settings for different parallel computing scenarios. +Zarr is designed to support parallel computing and enables concurrent reads and writes to arrays. +This section covers how to optimize Zarr's concurrency settings for different parallel computing +scenarios. ### Concurrent I/O operations -Zarr uses asynchronous I/O internally to enable concurrent reads and writes across multiple chunks. The level of concurrency is controlled by the `async.concurrency` configuration setting, which determines the maximum number of concurrent I/O operations. +Zarr uses asynchronous I/O internally to enable concurrent reads and writes across multiple chunks. +The level of concurrency is controlled by the `async.concurrency` configuration setting, which +determines the maximum number of concurrent I/O operations. -The default value is 64, which provides good performance for most workloads. You can adjust this value based on your specific needs: +The default value is 10, which is a conservative value. You may get improved performance by tuning +the concurrency limit. You can adjust this value based on your specific needs: ```python import zarr diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index 5acf242ef7..908922a5f2 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -97,7 +97,7 @@ def enable_gpu(self) -> ConfigSet: "order": "C", "write_empty_chunks": False, }, - "async": {"concurrency": 64, "timeout": None}, + "async": {"concurrency": 10, "timeout": None}, "threading": {"max_workers": None}, "json_indent": 2, "codec_pipeline": { diff --git a/tests/test_config.py b/tests/test_config.py index 150aca7c96..ec72967b98 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -54,7 +54,7 @@ def test_config_defaults_set() -> None: "order": "C", "write_empty_chunks": False, }, - "async": {"concurrency": 64, "timeout": None}, + "async": {"concurrency": 10, "timeout": None}, "threading": {"max_workers": None}, "json_indent": 2, "codec_pipeline": { @@ -100,7 +100,7 @@ def test_config_defaults_set() -> None: ] ) assert config.get("array.order") == "C" - assert config.get("async.concurrency") == 64 + assert config.get("async.concurrency") == 10 assert config.get("async.timeout") is None assert config.get("codec_pipeline.batch_size") == 1 assert config.get("json_indent") == 2 @@ -108,7 +108,7 @@ def test_config_defaults_set() -> None: @pytest.mark.parametrize( ("key", "old_val", "new_val"), - [("array.order", "C", "F"), ("async.concurrency", 64, 128), ("json_indent", 2, 0)], + [("array.order", "C", "F"), ("async.concurrency", 10, 128), ("json_indent", 2, 0)], ) def test_config_defaults_can_be_overridden(key: str, old_val: Any, new_val: Any) -> None: assert config.get(key) == old_val