List group members level by level instead of enumerating all keys

src/main/java/dev/zarr/zarrjava/core/Group.java

@@ -3,12 +3,19 @@
 import dev.zarr.zarrjava.ZarrException;
 import dev.zarr.zarrjava.store.FilesystemStore;
 import dev.zarr.zarrjava.store.StoreHandle;
+import dev.zarr.zarrjava.utils.Utils;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.AbstractMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 public abstract class Group extends AbstractNode {
@@ -70,13 +77,100 @@ public Node get(String key) throws ZarrException, IOException {
         return get(new String[]{key});
     }
 
-    public abstract Stream<Node> list();
+    /**
+     * Keys that hold metadata of the group itself and never point at a child node.
+     */
+    protected abstract Set<String> metadataKeys();
+
+    /**
+     * Lists the immediate children (arrays and subgroups) of this group.
+     * <p>
+     * This costs a single listing request on the underlying store, plus one metadata read per
+     * child. Keys that do not hold a Zarr node are skipped.
+     *
+     * @return a stream of the direct children of this group
+     * @throws UnsupportedOperationException if the underlying store does not support listing
+     */
+    public Stream<Node> members() {
+        return descendantKeys(new String[0]).parallelStream()
+                .map(this::openDescendant)
+                .filter(Objects::nonNull)
+                .collect(Collectors.toList())
+                .stream();
+    }
+
+    public Node[] membersAsArray() {
+        try (Stream<Node> nodeStream = members()) {
+            return nodeStream.toArray(Node[]::new);
+        }
+    }
+
+    /**
+     * Recursively lists all descendants (arrays and groups) of this group, at any depth.
+     * <p>
+     * The group hierarchy is walked one level at a time, so only group keys are listed and chunk
+     * keys are never enumerated. Descending into an array is not necessary and does not happen.
+     *
+     * @return a stream of all descendants of this group, excluding the group itself
+     * @throws UnsupportedOperationException if the underlying store does not support listing
+     */
+    public Stream<Node> list() {
+        return listDescendants(new String[0]);
+    }
 
     public Node[] listAsArray() {
         try (Stream<Node> nodeStream = list()) {
             return nodeStream.toArray(Node[]::new);
         }
     }
 
+    private Stream<Node> listDescendants(String[] prefix) {
+        List<Map.Entry<String[], Node>> children = descendantKeys(prefix).parallelStream()
+                .map(key -> new AbstractMap.SimpleEntry<String[], Node>(key, openDescendant(key)))
+                .collect(Collectors.toList());
+
+        return children.stream().flatMap(child -> {
+            Node node = child.getValue();
+            if (node == null) {
+                // Not a node itself, but it may still contain nodes further down.
+                return listDescendants(child.getKey());
+            }
+            if (node instanceof Group) {
+                return Stream.concat(Stream.of(node), listDescendants(child.getKey()));
+            }
+            return Stream.of(node);
+        });
+    }
+
+    /**
+     * Lists the keys directly below {@code prefix} that may hold a child node, relative to this
+     * group.
+     */
+    private List<String[]> descendantKeys(String[] prefix) {
+        Set<String> metadataKeys = metadataKeys();
+        try (Stream<String> children = storeHandle.resolve(prefix).listChildren()) {
+            return children
+                    .filter(name -> !metadataKeys.contains(name))
+                    .map(name -> Utils.concatArrays(prefix, new String[]{name}))
+                    .collect(Collectors.toList());
+        }
+    }
+
+    /**
+     * Opens the node at {@code key}, or returns null if there is no node there.
+     */
+    @Nullable
+    private Node openDescendant(String[] key) {
+        try {
+            return get(key);
+        } catch (IOException e) {
+            throw new RuntimeException(
+                    "Failed to read node metadata for key '" + String.join("/", key) + "': " + e.getMessage(), e);
+        } catch (ZarrException e) {
+            throw new RuntimeException(
+                    "Failed to parse node metadata for key '" + String.join("/", key) + "': " + e.getMessage(), e);
+        }
+    }
+
     public abstract GroupMetadata metadata();
 }

src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java

@@ -44,6 +44,16 @@ public boolean exists(String[] keys) {
         return Files.isRegularFile(resolveKeys(keys));
     }
 
+    /**
+     * Whether there is no file at the given keys. Reading such a key does not always fail with a
+     * {@link NoSuchFileException}: if one of the path components is a file rather than a
+     * directory, the filesystem reports "Not a directory" instead. Either way the key holds no
+     * data, so {@link #get} has to return null for it.
+     */
+    private boolean isMissing(String[] keys) {
+        return !Files.isRegularFile(resolveKeys(keys));
+    }
+
     @Nullable
     @Override
     public ByteBuffer get(String[] keys) {
@@ -52,6 +62,9 @@ public ByteBuffer get(String[] keys) {
         } catch (NoSuchFileException e) {
             return null;
         } catch (IOException e) {
+            if (isMissing(keys)) {
+                return null;
+            }
             throw StoreException.readFailed(this.toString(), keys, e);
         }
     }
@@ -75,6 +88,9 @@ public ByteBuffer get(String[] keys, long start) {
         } catch (NoSuchFileException e) {
             return null;
         } catch (IOException e) {
+            if (isMissing(keys)) {
+                return null;
+            }
             throw StoreException.readFailed(this.toString(), keys, e);
         }
     }
@@ -97,6 +113,9 @@ public ByteBuffer get(String[] keys, long start, long end) {
         } catch (NoSuchFileException e) {
             return null;
         } catch (IOException e) {
+            if (isMissing(keys)) {
+                return null;
+            }
             throw StoreException.readFailed(this.toString(), keys, e);
         }
     }

src/main/java/dev/zarr/zarrjava/store/S3Store.java

@@ -3,6 +3,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.stream.Stream;
 
 import javax.annotation.Nonnull;
@@ -12,7 +14,6 @@
 import software.amazon.awssdk.core.ResponseInputStream;
 import software.amazon.awssdk.core.sync.RequestBody;
 import software.amazon.awssdk.services.s3.S3Client;
-import software.amazon.awssdk.services.s3.model.CommonPrefix;
 import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectResponse;
@@ -191,15 +192,20 @@ public Stream<String> listChildren(String[] keys) {
                 .delimiter("/")
                 .build();
 
-        ListObjectsV2Response res = s3client.listObjectsV2(req);
-
-        // Combine CommonPrefixes (folders) and Contents (files)
-        Stream<String> folders = res.commonPrefixes().stream().map(CommonPrefix::prefix);
         final String finalFullPrefix = fullPrefix;
-        Stream<String> files = res.contents().stream().map(S3Object::key)
-                .filter(key -> !key.equals(finalFullPrefix));
+        // Combine CommonPrefixes (folders) and Contents (files) across all pages. A single
+        // listObjectsV2 call returns at most 1000 entries, which would silently truncate the
+        // children of a large group.
+        List<String> children = new ArrayList<>();
+        for (ListObjectsV2Response res : s3client.listObjectsV2Paginator(req)) {
+            res.commonPrefixes().forEach(commonPrefix -> children.add(commonPrefix.prefix()));
+            res.contents().stream()
+                    .map(S3Object::key)
+                    .filter(key -> !key.equals(finalFullPrefix))
+                    .forEach(children::add);
+        }
 
-        return Stream.concat(folders, files)
+        return children.stream()
                 .map(k -> keyToRelativeArray(k, finalFullPrefix)[0]);
     }
 

src/main/java/dev/zarr/zarrjava/v2/Group.java

@@ -17,14 +17,22 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
-import java.util.Objects;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.function.Function;
-import java.util.stream.Stream;
 
 import static dev.zarr.zarrjava.v2.Node.makeObjectMapper;
 import static dev.zarr.zarrjava.v2.Node.makeObjectWriter;
 
 public class Group extends dev.zarr.zarrjava.core.Group implements Node {
+
+    /**
+     * Keys that hold metadata of a v2 group itself and never point at a child node.
+     */
+    private static final Set<String> METADATA_KEYS = Collections.unmodifiableSet(
+            new HashSet<>(Arrays.asList(ZGROUP, ZATTRS, ZARRAY)));
+
     public GroupMetadata metadata;
 
     protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) {
@@ -182,26 +190,6 @@ public Node get(String[] key) throws ZarrException, IOException {
         }
     }
 
-    @Override
-    public Stream<dev.zarr.zarrjava.core.Node> list() {
-        return storeHandle.list().map(key -> {
-            if (key.length <= 1) return null; // exclude root from list
-            String fileName = key[key.length - 1];
-            StoreHandle parent = storeHandle.resolve(Arrays.copyOf(key, key.length - 1));
-            try {
-                if (fileName.equals(ZARRAY)) {
-                    return Array.open(parent);
-                }
-                if (fileName.equals(ZGROUP)) {
-                    return (dev.zarr.zarrjava.core.Node) Group.open(parent);
-                }
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-            return null;
-        }).filter(Objects::nonNull);
-    }
-
     /**
      * Creates a new subgroup with default metadata at the specified key.
      *
@@ -293,6 +281,11 @@ public String toString() {
         return String.format("<v2.Group {%s}>", storeHandle);
     }
 
+    @Override
+    protected Set<String> metadataKeys() {
+        return METADATA_KEYS;
+    }
+
     @Override
     public GroupMetadata metadata() {
         return metadata;

src/main/java/dev/zarr/zarrjava/v3/Group.java

@@ -15,16 +15,21 @@
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
 import java.util.function.Function;
-import java.util.stream.Stream;
 
 import static dev.zarr.zarrjava.v3.Node.makeObjectMapper;
 import static dev.zarr.zarrjava.v3.Node.makeObjectWriter;
 
 
 public class Group extends dev.zarr.zarrjava.core.Group implements Node {
 
+    /**
+     * Keys that hold metadata of a v3 group itself and never point at a child node.
+     */
+    private static final Set<String> METADATA_KEYS = Collections.singleton(ZARR_JSON);
+
     public GroupMetadata metadata;
 
     protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) throws IOException {
@@ -192,25 +197,6 @@ public Node get(String[] key) throws ZarrException, IOException {
         }
     }
 
-    @Override
-    public Stream<dev.zarr.zarrjava.core.Node> list() {
-        Stream<String[]> metadataKeys = storeHandle.list()
-                .filter(key -> key[key.length - 1].equals(ZARR_JSON))
-                .filter(key -> key.length > 1); // exclude root from list
-        return metadataKeys.map(key -> {
-            try {
-                return get(Arrays.copyOf(key, key.length - 1));
-            } catch (IOException e) {
-                throw new RuntimeException(
-                        "Failed to read node metadata for key '" + String.join("/", key) + "': " + e.getMessage(), e);
-            } catch (ZarrException e) {
-                throw new RuntimeException(
-                        "Failed to parse node metadata for key '" + String.join("/", key) + "': " + e.getMessage(), e);
-            }
-        });
-    }
-
-
     /**
      * Creates a new subgroup with the provided metadata at the specified key.
      *
@@ -312,6 +298,11 @@ public String toString() {
         return String.format("<v3.Group {%s}>", storeHandle);
     }
 
+    @Override
+    protected Set<String> metadataKeys() {
+        return METADATA_KEYS;
+    }
+
     @Override
     public GroupMetadata metadata() {
         return metadata;