From f89ba54198b27d1dfc233f126ee8bcfd09c65d34 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 10 Nov 2025 13:31:24 +0100 Subject: [PATCH 1/4] concurrency limit 64 -> 10 --- src/zarr/core/config.py | 2 +- tests/test_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index 5acf242ef7..908922a5f2 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -97,7 +97,7 @@ def enable_gpu(self) -> ConfigSet: "order": "C", "write_empty_chunks": False, }, - "async": {"concurrency": 64, "timeout": None}, + "async": {"concurrency": 10, "timeout": None}, "threading": {"max_workers": None}, "json_indent": 2, "codec_pipeline": { diff --git a/tests/test_config.py b/tests/test_config.py index 150aca7c96..55d2349e49 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -54,7 +54,7 @@ def test_config_defaults_set() -> None: "order": "C", "write_empty_chunks": False, }, - "async": {"concurrency": 64, "timeout": None}, + "async": {"concurrency": 10, "timeout": None}, "threading": {"max_workers": None}, "json_indent": 2, "codec_pipeline": { From 9ac0e6bae960e99b80633c367c2cbb5afc3b8b4c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 10 Nov 2025 13:43:05 +0100 Subject: [PATCH 2/4] update reference to concurrency limit in docs --- docs/user-guide/performance.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/user-guide/performance.md b/docs/user-guide/performance.md index 88d8e69936..8b338547fb 100644 --- a/docs/user-guide/performance.md +++ b/docs/user-guide/performance.md @@ -175,13 +175,18 @@ Coming soon. ## Parallel computing and synchronization -Zarr is designed to support parallel computing and enables concurrent reads and writes to arrays. This section covers how to optimize Zarr's concurrency settings for different parallel computing scenarios. +Zarr is designed to support parallel computing and enables concurrent reads and writes to arrays. +This section covers how to optimize Zarr's concurrency settings for different parallel computing +scenarios. ### Concurrent I/O operations -Zarr uses asynchronous I/O internally to enable concurrent reads and writes across multiple chunks. The level of concurrency is controlled by the `async.concurrency` configuration setting, which determines the maximum number of concurrent I/O operations. +Zarr uses asynchronous I/O internally to enable concurrent reads and writes across multiple chunks. +The level of concurrency is controlled by the `async.concurrency` configuration setting, which +determines the maximum number of concurrent I/O operations. -The default value is 64, which provides good performance for most workloads. You can adjust this value based on your specific needs: +The default value is 10, which is a conservative value. You may get improved performance by tuning +the concurrency limit. You can adjust this value based on your specific needs: ```python import zarr From a502daeb3b959b5073af8b3667075aea824af880 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 13 Nov 2025 16:10:12 +0100 Subject: [PATCH 3/4] update tests --- tests/test_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 55d2349e49..ec72967b98 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -100,7 +100,7 @@ def test_config_defaults_set() -> None: ] ) assert config.get("array.order") == "C" - assert config.get("async.concurrency") == 64 + assert config.get("async.concurrency") == 10 assert config.get("async.timeout") is None assert config.get("codec_pipeline.batch_size") == 1 assert config.get("json_indent") == 2 @@ -108,7 +108,7 @@ def test_config_defaults_set() -> None: @pytest.mark.parametrize( ("key", "old_val", "new_val"), - [("array.order", "C", "F"), ("async.concurrency", 64, 128), ("json_indent", 2, 0)], + [("array.order", "C", "F"), ("async.concurrency", 10, 128), ("json_indent", 2, 0)], ) def test_config_defaults_can_be_overridden(key: str, old_val: Any, new_val: Any) -> None: assert config.get(key) == old_val From 51a8ead8f78226d8149666b5b85107f337cdd24d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 13 Nov 2025 16:15:52 +0100 Subject: [PATCH 4/4] remove release note --- changes/3526.feature.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 changes/3526.feature.md diff --git a/changes/3526.feature.md b/changes/3526.feature.md deleted file mode 100644 index 7746611855..0000000000 --- a/changes/3526.feature.md +++ /dev/null @@ -1 +0,0 @@ -Increased the default value of `async.concurrency` from 10 to 64 to improve parallelism and throughput for concurrent I/O operations. This change enables better performance out-of-the-box for most workloads. Users with specific resource constraints or when using many Dask threads may want to lower this value via the `ZARR_ASYNC_CONCURRENCY` environment variable or by setting `zarr.config.set({'async.concurrency': N})`.