diff --git a/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocationS3.java b/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocationS3.java index 9eef8a1fcb..a312568f46 100644 --- a/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocationS3.java +++ b/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocationS3.java @@ -1,3 +1,8 @@ +/* + * Copyright (c) 2023-2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + package ucar.nc2.grib; import static com.google.common.truth.Truth.assertThat; @@ -51,7 +56,7 @@ public static List getTestParameters() { public TestGribIndexLocationS3(String filename, String fragment) { this.filename = filename + fragment; - this.indexFilename = filename + GribIndex.GBX9_IDX; + this.indexFilename = GribIndex.makeIndexFileName(filename); this.isGrib1 = filename.endsWith(".grib1"); } @@ -86,7 +91,7 @@ public void shouldCreateIndexInDefaultCache() throws IOException { assertThat(index).isNotNull(); assertThat(index.getNRecords()).isNotEqualTo(0); - assertThat(diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX).exists()).isTrue(); + assertThat(diskCache.getCacheFile(GribIndex.makeIndexFileName(s3File.getPath())).exists()).isTrue(); } @Test @@ -98,13 +103,13 @@ public void shouldReuseCachedIndex() throws IOException { final GribIndex index = GribIndex.readOrCreateIndexFromSingleFile(isGrib1, s3File, CollectionUpdateType.always, logger); assertThat(index).isNotNull(); - assertThat(diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX).exists()).isTrue(); - final long cacheLastModified = diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX).lastModified(); + assertThat(diskCache.getCacheFile(GribIndex.makeIndexFileName(s3File.getPath())).exists()).isTrue(); + final long cacheLastModified = diskCache.getCacheFile(GribIndex.makeIndexFileName(s3File.getPath())).lastModified(); final GribIndex rereadIndex = GribIndex.readOrCreateIndexFromSingleFile(isGrib1, s3File, CollectionUpdateType.never, logger); assertThat(rereadIndex).isNotNull(); - final File rereadCachedIndex = diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX); + final File rereadCachedIndex = diskCache.getCacheFile(GribIndex.makeIndexFileName(s3File.getPath())); assertThat(rereadCachedIndex.lastModified()).isEqualTo(cacheLastModified); } } diff --git a/cdm/core/src/main/java/thredds/filesystem/ControllerOS.java b/cdm/core/src/main/java/thredds/filesystem/ControllerOS.java index 1aaa024377..6c401084ff 100644 --- a/cdm/core/src/main/java/thredds/filesystem/ControllerOS.java +++ b/cdm/core/src/main/java/thredds/filesystem/ControllerOS.java @@ -78,6 +78,20 @@ public DirectoryStream getSubdirs(CollectionConfig mc, boolean recheck) { return new MFileDirectoryStream(new FilteredIterator(mc, new MFileIterator(cd), true)); // return only subdirs } + @Nullable + @Override + public DirectoryStream getFullInventoryAtLocation(String location) { + if (location.startsWith("file:")) { + location = location.substring(5); + } + + File cd = new File(location); + if (!cd.exists()) + return null; + if (!cd.isDirectory()) + return null; + return new MFileDirectoryStream(new MFileIterator(cd)); + } public void close() {} // NOOP diff --git a/cdm/core/src/main/java/thredds/filesystem/ControllerOS7.java b/cdm/core/src/main/java/thredds/filesystem/ControllerOS7.java index 82543e04d1..61352bcd9d 100644 --- a/cdm/core/src/main/java/thredds/filesystem/ControllerOS7.java +++ b/cdm/core/src/main/java/thredds/filesystem/ControllerOS7.java @@ -5,6 +5,7 @@ package thredds.filesystem; +import javax.annotation.Nullable; import thredds.inventory.CollectionConfig; import thredds.inventory.MController; import thredds.inventory.MFile; @@ -57,6 +58,27 @@ public DirectoryStream getSubdirs(CollectionConfig mc, boolean recheck) { return null; } + @Nullable + @Override + public DirectoryStream getFullInventoryAtLocation(String location) { + if (location.startsWith("file:")) { + location = location.substring(5); + } + + Path cd = Paths.get(location); + if (!Files.exists(cd)) + return null; + if (!Files.isDirectory(cd)) + return null; + + MFileDirectoryStream stream = null; + try { + stream = new MFileDirectoryStream(new MFileIterator(cd, null)); + } catch (IOException ioe) { + logger.warn(ioe.getMessage(), ioe); + } + return stream; + } public void close() {} // NOOP diff --git a/cdm/core/src/main/java/thredds/inventory/CollectionAbstract.java b/cdm/core/src/main/java/thredds/inventory/CollectionAbstract.java index a21fca8bfe..304e710192 100644 --- a/cdm/core/src/main/java/thredds/inventory/CollectionAbstract.java +++ b/cdm/core/src/main/java/thredds/inventory/CollectionAbstract.java @@ -121,7 +121,11 @@ public String getCollectionName() { @Override public String getIndexFilename(String suffix) { - return getRoot() + "/" + collectionName + suffix; + MFile rootMFile = MFiles.create(getRoot()); + MFile mfile = rootMFile.getChild(collectionName + suffix); + if (mfile == null) + throw new IllegalStateException("Cannot determine location of collection index file"); + return mfile.getPath(); } public void setStreamFilter(MFileFilter filter) { diff --git a/cdm/core/src/main/java/thredds/inventory/CollectionList.java b/cdm/core/src/main/java/thredds/inventory/CollectionList.java index a895dacc12..5690ff7f2f 100644 --- a/cdm/core/src/main/java/thredds/inventory/CollectionList.java +++ b/cdm/core/src/main/java/thredds/inventory/CollectionList.java @@ -42,7 +42,6 @@ public CollectionList(String collectionName, String list, Logger logger) { Collections.sort(mfiles); this.lastModified = lastModified; - this.root = System.getProperty("user.dir"); } public CollectionList(String collectionName, String root, List mfiles, Logger logger) { diff --git a/cdm/core/src/main/java/thredds/inventory/CollectionManagerCatalog.java b/cdm/core/src/main/java/thredds/inventory/CollectionManagerCatalog.java index 6e157f9d19..07c9090141 100644 --- a/cdm/core/src/main/java/thredds/inventory/CollectionManagerCatalog.java +++ b/cdm/core/src/main/java/thredds/inventory/CollectionManagerCatalog.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -18,6 +18,7 @@ import java.util.Date; import java.util.Formatter; import java.util.List; +import ucar.nc2.util.DiskCache2; /** * CollectionManager of datasets from a catalog. @@ -45,7 +46,7 @@ public CollectionManagerCatalog(String collectionName, String collectionSpec, St } this.catalogUrl = collectionSpec; - this.root = System.getProperty("user.dir"); + this.root = DiskCache2.getDefault().getRootDirectory(); } @Override diff --git a/cdm/core/src/main/java/thredds/inventory/CollectionSingleFile.java b/cdm/core/src/main/java/thredds/inventory/CollectionSingleFile.java index faa81b9d6f..0be93a70ea 100644 --- a/cdm/core/src/main/java/thredds/inventory/CollectionSingleFile.java +++ b/cdm/core/src/main/java/thredds/inventory/CollectionSingleFile.java @@ -5,6 +5,8 @@ package thredds.inventory; +import ucar.nc2.util.DiskCache2; + /** * A CollectionManager consisting of a single file * @@ -18,9 +20,9 @@ public CollectionSingleFile(MFile file, org.slf4j.Logger logger) { mfiles.add(file); try { MFile p = file.getParent(); - this.root = p != null ? p.getPath() : System.getProperty("user.dir"); + this.root = p != null ? p.getPath() : DiskCache2.getDefault().getRootDirectory(); } catch (java.io.IOException e) { - this.root = System.getProperty("user.dir"); + this.root = DiskCache2.getDefault().getRootDirectory(); } this.lastModified = file.getLastModified(); diff --git a/cdm/core/src/main/java/thredds/inventory/CollectionSpecParser.java b/cdm/core/src/main/java/thredds/inventory/CollectionSpecParser.java index d35d33cc94..f91db7d915 100644 --- a/cdm/core/src/main/java/thredds/inventory/CollectionSpecParser.java +++ b/cdm/core/src/main/java/thredds/inventory/CollectionSpecParser.java @@ -1,11 +1,13 @@ /* - * Copyright (c) 1998-2022 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ + package thredds.inventory; import java.io.File; import java.util.Formatter; +import ucar.nc2.util.DiskCache2; /** * Parses the collection specification string for local files. @@ -39,7 +41,7 @@ public class CollectionSpecParser extends CollectionSpecParserAbstract { private final static String DELIMITER = "/"; private final static String FRAGMENT = ""; - private final static String DEFAULT_DIR = System.getProperty("user.dir"); + private final static String DEFAULT_DIR = DiskCache2.getDefault().getRootDirectory(); /** * Single spec : "/topdir/** /#dateFormatMark#regExp" diff --git a/cdm/core/src/main/java/thredds/inventory/MController.java b/cdm/core/src/main/java/thredds/inventory/MController.java index 1e42dd0f23..e47d206f2f 100644 --- a/cdm/core/src/main/java/thredds/inventory/MController.java +++ b/cdm/core/src/main/java/thredds/inventory/MController.java @@ -48,15 +48,13 @@ public interface MController extends Closeable { DirectoryStream getSubdirs(CollectionConfig mc, boolean recheck); /** - * Get an MFile for a specific location. - * - * @param location the location - * @return MFile or null + * Returns all subdirectories and leaves in a given location, not recursing into subdirectories. + * + * @param location defines the location to scan + * @return DirectoryStream over Mfiles, or null if the location does not exist */ @Nullable - default MFile getMFile(String location) { - return MFiles.create(location); - } + DirectoryStream getFullInventoryAtLocation(String location); @Override void close(); diff --git a/cdm/core/src/main/java/thredds/inventory/MControllers.java b/cdm/core/src/main/java/thredds/inventory/MControllers.java index 6a703b4599..1dfbe54784 100644 --- a/cdm/core/src/main/java/thredds/inventory/MControllers.java +++ b/cdm/core/src/main/java/thredds/inventory/MControllers.java @@ -43,8 +43,7 @@ public static MController create(String location) { */ public static DirectoryStream newDirectoryStream(String location) throws IOException { MController controller = create(location); - CollectionConfig config = new CollectionConfig(location, location, false, null, null); - DirectoryStream stream = controller.getInventoryTop(config, true); + DirectoryStream stream = controller.getFullInventoryAtLocation(location); controller.close(); if (stream == null) { throw new IOException("Could not create DirectoryStream for " + location); diff --git a/cdm/core/src/main/java/thredds/inventory/partition/DirectoryBuilder.java b/cdm/core/src/main/java/thredds/inventory/partition/DirectoryBuilder.java index 3e8ad497da..154d6ba760 100644 --- a/cdm/core/src/main/java/thredds/inventory/partition/DirectoryBuilder.java +++ b/cdm/core/src/main/java/thredds/inventory/partition/DirectoryBuilder.java @@ -71,7 +71,7 @@ private enum PartitionStatus { private PartitionStatus partitionStatus = PartitionStatus.unknown; public DirectoryBuilder(String topCollectionName, String dirFilename, String suffix) throws IOException { - this(topCollectionName, MControllers.create(dirFilename).getMFile(dirFilename), suffix); + this(topCollectionName, MFiles.create(dirFilename), suffix); } /** diff --git a/cdm/core/src/test/data/directory_stream/sub_directory/sub_directory_2/subdir2_file b/cdm/core/src/test/data/directory_stream/sub_directory/sub_directory_2/subdir2_file new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cdm/core/src/test/data/directory_stream/sub_directory/subdir_file b/cdm/core/src/test/data/directory_stream/sub_directory/subdir_file new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cdm/core/src/test/data/directory_stream/top_level_file b/cdm/core/src/test/data/directory_stream/top_level_file new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cdm/core/src/test/java/thredds/inventory/TestCollectionSpecParser.java b/cdm/core/src/test/java/thredds/inventory/TestCollectionSpecParser.java index 35e5bc9ec9..3e4bc4ff64 100644 --- a/cdm/core/src/test/java/thredds/inventory/TestCollectionSpecParser.java +++ b/cdm/core/src/test/java/thredds/inventory/TestCollectionSpecParser.java @@ -1,3 +1,8 @@ +/* + * Copyright (c) 2019-2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + package thredds.inventory; import static com.google.common.truth.Truth.assertThat; @@ -12,6 +17,7 @@ import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import ucar.nc2.util.DiskCache2; @RunWith(Parameterized.class) public class TestCollectionSpecParser { @@ -67,11 +73,11 @@ public static List getTestParameters() throws URISyntaxException { {"/data/ldm/pub/decoded/netcdf/surface/metar/T*.T", "/data/ldm/pub/decoded/netcdf/surface/metar", false, "T*.T", null}, - {"", System.getProperty("user.dir"), false, null, null}, + {"", DiskCache2.getDefault().getRootDirectory(), false, null, null}, - {".*grib1", System.getProperty("user.dir"), false, ".*grib1", null}, + {".*grib1", DiskCache2.getDefault().getRootDirectory(), false, ".*grib1", null}, - {".*\\.grib1", System.getProperty("user.dir"), false, ".*\\.grib1", null}, + {".*\\.grib1", DiskCache2.getDefault().getRootDirectory(), false, ".*\\.grib1", null}, {"dir/**/subdir/.*grib1", "dir", true, "subdir/.*grib1", null},}); } diff --git a/cdm/core/src/test/java/thredds/inventory/TestMControllers.java b/cdm/core/src/test/java/thredds/inventory/TestMControllers.java new file mode 100644 index 0000000000..aa0613afc7 --- /dev/null +++ b/cdm/core/src/test/java/thredds/inventory/TestMControllers.java @@ -0,0 +1,52 @@ +package thredds.inventory; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import org.junit.Test; +import ucar.unidata.util.test.TestDir; + +public class TestMControllers { + + @Test + public void testDirectoryStream() throws IOException { + String testDir = TestDir.localTestDataDir + "directory_stream"; + + ArrayList dsMfiles = new ArrayList<>(); + DirectoryStream ds = MControllers.newDirectoryStream(testDir); + for (MFile item : ds) { + dsMfiles.add(Paths.get(item.getPath()).toAbsolutePath().toString()); + } + ds.close(); + + ArrayList dsFiles = new ArrayList<>(); + DirectoryStream ds2 = + Files.newDirectoryStream(Paths.get(TestDir.localTestDataDir + "directory_stream").toAbsolutePath()); + for (Path item : ds2) { + dsFiles.add(item.toString()); + } + ds2.close(); + + dsMfiles.sort(null); + dsFiles.sort(null); + assertThat(dsMfiles).containsExactlyElementsIn(dsFiles); + } + + @Test + public void testSubdirStream() throws IOException { + String testDir = TestDir.localTestDataDir + "directory_stream"; + ArrayList dsMfiles = new ArrayList<>(); + DirectoryStream ds = MControllers.newSubdirStream(testDir); + for (MFile item : ds) { + dsMfiles.add(Paths.get(item.getPath()).toAbsolutePath().toString()); + } + ds.close(); + + assertThat(dsMfiles).hasSize(1); + } +} diff --git a/cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java b/cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java index 0af11b58a5..6cdf4ace41 100644 --- a/cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java +++ b/cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java @@ -51,11 +51,17 @@ public ControllerS3() {} private void init(CollectionConfig mc) { if (mc != null) { + init(mc.getDirectoryName()); + } + } + + private void init(String location) { + if (location != null) { try { - initUri(mc.getDirectoryName()); + initUri(location); initClient(); } catch (IOException e) { - logger.error("Error initializing ControllerS3 for {}.", mc.getDirectoryName(), e); + logger.error("Error initializing ControllerS3 for {}.", location, e); } } } @@ -128,6 +134,16 @@ public DirectoryStream getSubdirs(CollectionConfig mc, boolean recheck) { return new MFileDirectoryStream(new FilteredIterator(mc, mFiles.iterator(), true)); } + @Override + public DirectoryStream getFullInventoryAtLocation(String location) { + init(location); + String prefix = null; + if (initialUri.getKey().isPresent()) { + prefix = initialUri.getKey().get(); + } + return new MFileDirectoryStream(new MFileS3Iterator(client, initialUri, prefix, limit, false)); + } + @Override public void close() {} // NO-OP diff --git a/cdm/s3/src/main/java/thredds/inventory/s3/MFileS3.java b/cdm/s3/src/main/java/thredds/inventory/s3/MFileS3.java index 8d9ab9bd08..1b5773b421 100644 --- a/cdm/s3/src/main/java/thredds/inventory/s3/MFileS3.java +++ b/cdm/s3/src/main/java/thredds/inventory/s3/MFileS3.java @@ -1,7 +1,8 @@ /* - * Copyright (c) 2020 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 2020-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ + package thredds.inventory.s3; import java.io.IOException; @@ -218,9 +219,20 @@ public String getName() { @Nullable public MFile getParent() throws IOException { // In general, objects to do not have parents. However, if a delimiter is set, we have a pseudo path, and then - // the object can have a parent. + // the object can have a parent. If a delimiter is not set, but the key is not null, then the parent is + // the top of the bucket. If a delimiter is not set, and the key is null, then we are at the top of the bucket + // already, and then there is no parent. MFile parentMfile = null; - if (delimiter != null) { + if (delimiter == null) { + String key = getKey(); + if (key != null && !key.isEmpty()) { + String uriString = getPath(); + int queryIdx = uriString.indexOf("?"); + if (queryIdx > 0) { + parentMfile = new MFileS3(uriString.substring(0, queryIdx)); + } + } + } else { // get the full path String currentUri = getPath(); String frag = ""; diff --git a/cdm/s3/src/test/java/thredds/inventory/s3/TestMFileS3.java b/cdm/s3/src/test/java/thredds/inventory/s3/TestMFileS3.java index 6c7a2b1c07..676efa5490 100644 --- a/cdm/s3/src/test/java/thredds/inventory/s3/TestMFileS3.java +++ b/cdm/s3/src/test/java/thredds/inventory/s3/TestMFileS3.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 2020-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -455,7 +455,8 @@ private void checkWithBucketAndKey(String cdmS3Uri, String expectedName, String if (delimiter != null) { assertThat(mFile.getParent()).isNotNull(); } else { - assertThat(mFile.getParent()).isNull(); + assertThat(mFile.getParent()).isNotNull(); + assertThat(mFile.getParent().getPath()).isEqualTo(cdmS3Uri.split("\\?")[0]); } assertThat(mFile.isDirectory()).isFalse(); assertThat(mFile.getLength()).isEqualTo(G16_OBJECT_1_SIZE); @@ -468,8 +469,9 @@ private void dirCheckNoDelim(String cdmS3Uri, String expectedName) throws IOExce assertThat(mFile.getPath()).isEqualTo(cdmS3Uri); // Without a delimiter, the name is the key. assertThat(mFile.getName()).isEqualTo(expectedName); - // Without a delimiter, there is no parent. - assertThat(mFile.getParent()).isNull(); + // Without a delimiter, the parent is the bucket. + assertThat(mFile.getParent()).isNotNull(); + assertThat(mFile.getParent().getPath()).isEqualTo(cdmS3Uri.split("\\?")[0]); // Without a delimiter, there is no concept of a directory. assertThat(mFile.isDirectory()).isFalse(); } diff --git a/cdm/zarr/src/main/java/thredds/filesystem/zarr/ControllerZip.java b/cdm/zarr/src/main/java/thredds/filesystem/zarr/ControllerZip.java index 365f87345c..0269645230 100644 --- a/cdm/zarr/src/main/java/thredds/filesystem/zarr/ControllerZip.java +++ b/cdm/zarr/src/main/java/thredds/filesystem/zarr/ControllerZip.java @@ -79,6 +79,21 @@ public DirectoryStream getSubdirs(CollectionConfig mc, boolean recheck) { } } + @Override + public DirectoryStream getFullInventoryAtLocation(String location) { + if (location.startsWith(prefix)) { + location = location.substring(prefix.length()); + } + + try { + MFileZip mfile = new MFileZip(location); + return new MFileDirectoryStream(new MFileIteratorLeaves(mfile)); + } catch (IOException ioe) { + logger.warn(ioe.getMessage(), ioe); + return null; + } + } + @Override public void close() {} // NOOP diff --git a/grib/build.gradle.kts b/grib/build.gradle.kts index 4e381669e3..a9a51d6e67 100644 --- a/grib/build.gradle.kts +++ b/grib/build.gradle.kts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 2025-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -39,6 +39,7 @@ dependencies { testCompileOnly(ncjLibs.junit4) + testRuntimeOnly(project(":cdm-s3")) testRuntimeOnly(project(":libaec-native")) testRuntimeOnly(ncjLibs.junit5.platformLauncher) diff --git a/grib/src/main/java/ucar/nc2/grib/GribIndex.java b/grib/src/main/java/ucar/nc2/grib/GribIndex.java index 4724e7b309..d5a51ed4b3 100644 --- a/grib/src/main/java/ucar/nc2/grib/GribIndex.java +++ b/grib/src/main/java/ucar/nc2/grib/GribIndex.java @@ -29,9 +29,7 @@ public abstract class GribIndex { private static final CollectionManager.ChangeChecker gribCC = new CollectionManager.ChangeChecker() { public boolean hasChangedSince(MFile file, long when) { - String idxPath = file.getPath(); - if (!idxPath.endsWith(GBX9_IDX)) - idxPath += GBX9_IDX; + String idxPath = makeIndexFileName(file.getPath()); MFile idxFile = GribIndexCache.getExistingFileOrCache(idxPath); if (idxFile == null) return true; @@ -43,9 +41,7 @@ public boolean hasChangedSince(MFile file, long when) { } public boolean hasntChangedSince(MFile file, long when) { - String idxPath = file.getPath(); - if (!idxPath.endsWith(GBX9_IDX)) - idxPath += GBX9_IDX; + String idxPath = makeIndexFileName(file.getPath()); MFile idxFile = GribIndexCache.getExistingFileOrCache(idxPath); if (idxFile == null) return true; @@ -91,14 +87,31 @@ public static GribIndex readOrCreateIndexFromSingleFile(boolean isGrib1, MFile m if (!index.readIndex(mfile.getPath(), mfile.getLastModified(), force)) { // heres where the index date is checked // against the data file index.makeIndex(mfile.getPath(), null); - logger.debug(" Index written: {} == {} records", mfile.getName() + GBX9_IDX, index.getNRecords()); + logger.debug(" Index written: {} ({}) == {} records", mfile.getName(), GBX9_IDX, index.getNRecords()); } else if (debug) { - logger.debug(" Index read: {} == {} records", mfile.getName() + GBX9_IDX, index.getNRecords()); + logger.debug(" Index read: {} ({}) == {} records", mfile.getName(), GBX9_IDX, index.getNRecords()); } return index; } + /** + * Make the gbx9 index name from the grib file name. + * + * Handles special cases where the grib filename is not a file path. + * For example + * cdms3:thredds-test-data?test-grib-without-index/cosmo-eu.grib2#delimiter=/ + * + * becomes + * cdms3:thredds-test-data?test-grib-without-index/cosmo-eu.grib2.gbx9#delimiter=/ + * + * @param filename + * @return the gbx9 index file + */ + public static String makeIndexFileName(String filename) { + return GribUtils.makeIndexFileName(filename, GBX9_IDX); + } + ////////////////////////////////////////// /** diff --git a/grib/src/main/java/ucar/nc2/grib/GribUtils.java b/grib/src/main/java/ucar/nc2/grib/GribUtils.java index 5ac7059f29..c01e4fe21f 100755 --- a/grib/src/main/java/ucar/nc2/grib/GribUtils.java +++ b/grib/src/main/java/ucar/nc2/grib/GribUtils.java @@ -1,10 +1,12 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ package ucar.nc2.grib; +import thredds.inventory.MFile; +import thredds.inventory.MFiles; import ucar.nc2.time.CalendarDate; import ucar.nc2.time.CalendarPeriod; import ucar.unidata.util.StringUtil2; @@ -194,4 +196,28 @@ public static boolean scanModeSameDirection(int scanMode) { return !GribNumbers.testGribBitIsSet(scanMode, 4); } + /** + * Make an index filename with the given suffix from the grib filename. + * + * Handles special cases where the grib filename is not a file path. + * For example, + * cdms3:thredds-test-data?test-grib-without-index/cosmo-eu.grib2#delimiter=/ + * + * becomes + * cdms3:thredds-test-data?test-grib-without-index/cosmo-eu.grib2.gbx9#delimiter=/ + * + * @param filename + * @return the index filename + */ + public static String makeIndexFileName(String filename, String idxNameSuffix) { + String idxPath = filename; + if (!filename.endsWith(idxNameSuffix)) { + MFile testIdxMFile = MFiles.create(filename); + String mfileName = testIdxMFile.getName(); + String idxName = testIdxMFile.getName() + idxNameSuffix; + int i = idxPath.lastIndexOf(mfileName); + idxPath = idxPath.substring(0, i) + idxName + idxPath.substring(i + mfileName.length(), filename.length()); + } + return idxPath; + } } diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GcMFile.java b/grib/src/main/java/ucar/nc2/grib/collection/GcMFile.java index 733f5d8846..d379fdc959 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/GcMFile.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/GcMFile.java @@ -33,7 +33,7 @@ static List makeFiles(MFile directory, List files, Set MFile file = files.get(index); String filename; if (file.getPath().startsWith(dirPath)) { - filename = file.getPath().substring(dirPath.length()); + filename = file.getName(); if (filename.startsWith("/")) filename = filename.substring(1); } else diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GribCdmIndex.java b/grib/src/main/java/ucar/nc2/grib/collection/GribCdmIndex.java index 9413c790c9..de1ed2176d 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/GribCdmIndex.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/GribCdmIndex.java @@ -19,6 +19,7 @@ import thredds.inventory.partition.*; import ucar.nc2.dataset.DatasetUrl; import ucar.nc2.grib.GribIndexCache; +import ucar.nc2.grib.GribUtils; import ucar.nc2.grib.grib1.Grib1RecordScanner; import ucar.nc2.grib.grib2.Grib2RecordScanner; import ucar.nc2.stream.NcStream; @@ -120,15 +121,25 @@ private static MFile makeTopIndexFileFromConfig(FeatureCollectionConfig config) static MFile makeIndexFile(String collectionName, MFile directory) { String nameNoBlanks = StringUtil2.replace(collectionName, ' ', "_"); - return directory.getChild(nameNoBlanks + NCX_SUFFIX); + String indexFileName = GribUtils.makeIndexFileName(nameNoBlanks, NCX_SUFFIX); + return directory.getChild(indexFileName); } - private static String makeNameFromIndexFilename(String idxPathname) { - idxPathname = StringUtil2.replace(idxPathname, '\\', "/"); - int pos = idxPathname.lastIndexOf('/'); - String idxFilename = (pos < 0) ? idxPathname : idxPathname.substring(pos + 1); - assert idxFilename.endsWith(NCX_SUFFIX) : idxFilename; - return idxFilename.substring(0, idxFilename.length() - NCX_SUFFIX.length()); + static String makeNameFromIndexFilename(String idxPathname) { + String name; + if (!idxPathname.endsWith(NCX_SUFFIX)) { + MFile testIdxMFile = MFiles.create(idxPathname); + String idxName = testIdxMFile.getName(); + assert idxName.endsWith(NCX_SUFFIX) : idxName; + name = idxName.substring(0, idxName.length() - NCX_SUFFIX.length()); + } else { + idxPathname = StringUtil2.replace(idxPathname, '\\', "/"); + int pos = idxPathname.lastIndexOf('/'); + String idxFilename = (pos < 0) ? idxPathname : idxPathname.substring(pos + 1); + assert idxFilename.endsWith(NCX_SUFFIX) : idxFilename; + name = idxFilename.substring(0, idxFilename.length() - NCX_SUFFIX.length()); + } + return name; } /////////////////////////////////////////// @@ -338,7 +349,7 @@ public static boolean updateGribCollection(FeatureCollectionConfig config, Colle if (specp.wantSubdirs()) { // its a partition try (DirectoryPartition dpart = - new DirectoryPartition(config, rootPath.toString(), true, new GribCdmIndex(logger), NCX_SUFFIX, logger)) { + new DirectoryPartition(config, rootPath, true, new GribCdmIndex(logger), NCX_SUFFIX, logger)) { dpart.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); changed = updateDirectoryCollectionRecurse(isGrib1, dpart, config, updateType, logger); } @@ -834,9 +845,7 @@ private static GribCollectionImmutable openGribCollectionFromDataFile(boolean is throws IOException { String filename = dataRaf.getLocation(); - File dataFile = new File(filename); - - MFile mfile = new MFileOS(dataFile); + MFile mfile = MFiles.create(filename); return openGribCollectionFromDataFile(isGrib1, mfile, updateType, config, errlog, logger); } diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionImmutable.java b/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionImmutable.java index 67f8f43269..792ed5d66a 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionImmutable.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionImmutable.java @@ -1082,9 +1082,7 @@ public MFile findMFileByName(String filename) { RandomAccessFile getDataRaf(int fileno) throws IOException { // absolute location - MFile mfile = fileMap.get(fileno); - String filename = mfile.getPath(); - MFile dataFile = MFiles.create(filename); + MFile dataFile = fileMap.get(fileno); // if data file does not exist, check relative location - eg may be /upc/share instead of Q: if (!dataFile.exists()) { @@ -1103,7 +1101,7 @@ RandomAccessFile getDataRaf(int fileno) throws IOException { throw new FileNotFoundException("data file not found = " + dataFile.getPath()); } - RandomAccessFile want = RandomAccessFile.acquire(dataFile.getPath()); + RandomAccessFile want = NetcdfFiles.getRaf(dataFile.getPath(), -1); want.order(RandomAccessFile.BIG_ENDIAN); return want; } diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionMutable.java b/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionMutable.java index fb542a319f..480e13be3e 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionMutable.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionMutable.java @@ -51,9 +51,9 @@ public class GribCollectionMutable implements Closeable { ////////////////////////////////////////////////////////// static MFile makeIndexMFile(String collectionName, MFile directory) { - String nameNoBlanks = StringUtil2.replace(collectionName, ' ', "_"); - return new GcMFile(directory, nameNoBlanks + GribCdmIndex.NCX_SUFFIX, -1, -1, -1); // LOOK dont know lastMod, size. - // can it be added later? + MFile nameNoBlanks = GribCdmIndex.makeIndexFile(collectionName, directory); + return new GcMFile(directory, nameNoBlanks.getName(), -1, -1, -1); // LOOK dont know lastMod, size. + // can it be added later? } private static final CalendarDateFormatter cf = diff --git a/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java b/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java index 8614a7ae17..7ee69e5505 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java +++ b/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java @@ -15,6 +15,7 @@ import ucar.nc2.NetcdfFiles; import ucar.nc2.grib.GribIndex; import ucar.nc2.grib.GribIndexCache; +import ucar.nc2.grib.GribUtils; import ucar.nc2.stream.NcStream; import ucar.unidata.io.RandomAccessFile; import java.io.File; @@ -88,9 +89,7 @@ public boolean readIndex(String filename, long gribLastModified) { } public boolean readIndex(String filename, long gribLastModified, CollectionUpdateType force) { - String idxPath = filename; - if (!idxPath.endsWith(GBX9_IDX)) - idxPath += GBX9_IDX; + String idxPath = GribIndex.makeIndexFileName(filename); MFile idxFile = GribIndexCache.getExistingFileOrCache(idxPath); if (idxFile == null) return false; @@ -169,11 +168,10 @@ private Grib1SectionGridDefinition readGds(Grib1IndexProto.Grib1GdsSection proto // LOOK what about extending an index ?? public boolean makeIndex(String filename, RandomAccessFile dataRaf) throws IOException { - String idxPath = filename; - if (!idxPath.endsWith(GBX9_IDX)) - idxPath += GBX9_IDX; + String idxPath = GribIndex.makeIndexFileName(filename); MFile idxMFile = GribIndexCache.getFileOrCache(idxPath); - MFile idxMFileTmp = GribIndexCache.getFileOrCache(idxPath + ".tmp"); + String idxPathTmp = GribUtils.makeIndexFileName(idxPath, ".tmp"); + MFile idxMFileTmp = GribIndexCache.getFileOrCache(idxPathTmp); if (!(idxMFile instanceof MFileOS || idxMFile instanceof MFileOS7)) { throw new IllegalArgumentException("Only local file systems are supported for index creation"); diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java index f2488f343f..da4f2a94bd 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java @@ -15,6 +15,7 @@ import ucar.nc2.NetcdfFiles; import ucar.nc2.grib.GribIndex; import ucar.nc2.grib.GribIndexCache; +import ucar.nc2.grib.GribUtils; import ucar.nc2.stream.NcStream; import ucar.unidata.io.RandomAccessFile; import java.io.File; @@ -92,9 +93,7 @@ public boolean readIndex(String filename, long gribLastModified) { } public boolean readIndex(String filename, long gribLastModified, CollectionUpdateType force) { - String idxPath = filename; - if (!idxPath.endsWith(GBX9_IDX)) - idxPath += GBX9_IDX; + String idxPath = GribIndex.makeIndexFileName(filename); MFile idxFile = GribIndexCache.getExistingFileOrCache(idxPath); if (idxFile == null) return false; @@ -204,11 +203,10 @@ private Grib2SectionGridDefinition readGds(Grib2IndexProto.GribGdsSection proto) // LOOK what about extending an index ?? // Only support creation of index files locally public boolean makeIndex(String filename, RandomAccessFile dataRaf) throws IOException { - String idxPath = filename; - if (!idxPath.endsWith(GBX9_IDX)) - idxPath += GBX9_IDX; + String idxPath = GribIndex.makeIndexFileName(filename); MFile idxMFile = GribIndexCache.getFileOrCache(idxPath); - MFile idxMFileTmp = GribIndexCache.getFileOrCache(idxPath + ".tmp"); + String idxPathTmp = GribUtils.makeIndexFileName(idxPath, ".tmp"); + MFile idxMFileTmp = GribIndexCache.getFileOrCache(idxPathTmp); if (!(idxMFile instanceof MFileOS || idxMFile instanceof MFileOS7)) { throw new IllegalArgumentException("Only local file systems are supported for index creation"); diff --git a/grib/src/test/java/ucar/nc2/grib/TestSingleGribFileS3.java b/grib/src/test/java/ucar/nc2/grib/TestSingleGribFileS3.java new file mode 100644 index 0000000000..3598d884b1 --- /dev/null +++ b/grib/src/test/java/ucar/nc2/grib/TestSingleGribFileS3.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + +package ucar.nc2.grib; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.Formatter; +import org.junit.Test; +import ucar.ma2.Array; +import ucar.nc2.Variable; +import ucar.nc2.dataset.NetcdfDataset; +import ucar.nc2.dataset.NetcdfDatasets; +import ucar.nc2.util.CompareNetcdf2; +import ucar.unidata.util.test.TestDir; + +public class TestSingleGribFileS3 { + + private static final String GRIB1_BUCKET_KEY = "thredds-test-data?test-grib-without-index/radar_national.grib1"; + private static final String GRIB2_BUCKET_KEY = "thredds-test-data?test-grib-without-index/cosmo-eu.grib2"; + private static final String GRIB1_LOCAL = TestDir.localTestDataDir + "radar_national.grib1"; + private static final String GRIB2_LOCAL = TestDir.localTestDataDir + "cosmo-eu.grib2"; + + @Test + public void testGrib1S3Full() throws IOException { + String location = String.format("cdms3://s3.us-east-1.amazonaws.com/%s#delimiter=/", GRIB1_BUCKET_KEY); + basicDatasetValidation(location); + } + + @Test + public void testGrib1S3Short() throws IOException { + String location = String.format("cdms3:%s#delimiter=/", GRIB1_BUCKET_KEY); + basicDatasetValidation(location); + } + + @Test + public void testGrib1SFullNoDelimiter() throws IOException { + String location = String.format("cdms3://s3.us-east-1.amazonaws.com/%s", GRIB1_BUCKET_KEY); + basicDatasetValidation(location); + } + + @Test + public void testGrib1S3ShortNoDelimiter() throws IOException { + String location = String.format("cdms3:%s", GRIB1_BUCKET_KEY); + basicDatasetValidation(location); + } + + @Test + public void testGrib2S3Full() throws IOException { + String location = String.format("cdms3://s3.us-east-1.amazonaws.com/%s#delimiter=/", GRIB2_BUCKET_KEY); + basicDatasetValidation(location); + } + + @Test + public void testGrib2S3Short() throws IOException { + String location = String.format("cdms3:%s#delimiter=/", GRIB2_BUCKET_KEY); + basicDatasetValidation(location); + } + + @Test + public void testGrib2SFullNoDelimiter() throws IOException { + String location = String.format("cdms3://s3.us-east-1.amazonaws.com/%s", GRIB2_BUCKET_KEY); + basicDatasetValidation(location); + } + + @Test + public void testGrib2S3ShortNoDelimiter() throws IOException { + String location = String.format("cdms3:%s", GRIB2_BUCKET_KEY); + basicDatasetValidation(location); + } + + @Test + public void compareGrib1() throws IOException { + String location = String.format("cdms3://s3.us-east-1.amazonaws.com/%s#delimiter=/", GRIB1_BUCKET_KEY); + compareWithLocal(location, GRIB1_LOCAL); + } + + @Test + public void compareGrib2() throws IOException { + String location = String.format("cdms3://s3.us-east-1.amazonaws.com/%s#delimiter=/", GRIB2_BUCKET_KEY); + compareWithLocal(location, GRIB2_LOCAL); + } + + private static void compareWithLocal(String remoteLocation, String localLocation) throws IOException { + try (NetcdfDataset remoteNetcdfDataset = NetcdfDatasets.openDataset(remoteLocation); + NetcdfDataset localNetcdfDataset = NetcdfDatasets.openDataset(localLocation)) { + Formatter f = new Formatter(); + CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true); + if (!compare.compare(localNetcdfDataset, remoteNetcdfDataset, null)) { + System.out.printf("Compare %s%n%s%n", localLocation, f); + fail(); + } + } + } + + private static void basicDatasetValidation(String location) throws IOException { + try (NetcdfDataset netcdfDataset = NetcdfDatasets.openDataset(location)) { + assertThat(netcdfDataset.getLastModified()).isGreaterThan(0); + assertThat(netcdfDataset.getVariables()).isNotEmpty(); + + Variable variable = netcdfDataset.getVariables().get(1); + Array data = variable.read(); + assertThat(data).isNotNull(); + } + } +} diff --git a/grib/src/test/java/ucar/nc2/grib/collection/TestGribCdmIndex.java b/grib/src/test/java/ucar/nc2/grib/collection/TestGribCdmIndex.java new file mode 100644 index 0000000000..e092f94dad --- /dev/null +++ b/grib/src/test/java/ucar/nc2/grib/collection/TestGribCdmIndex.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + +package ucar.nc2.grib.collection; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Test; +import thredds.inventory.MFile; +import thredds.inventory.MFiles; + +public class TestGribCdmIndex { + + @Test + public void testMakeIndexFile() { + MFile directory = MFiles.create("/path/to/"); + MFile idxMFile = GribCdmIndex.makeIndexFile("collection name", directory); + assertThat(idxMFile.getPath()).startsWith(directory.getPath()); + assertThat(idxMFile.getName()).endsWith(GribCdmIndex.NCX_SUFFIX); + + + directory = MFiles.create("cdms3:bucket?path/to/#delimiter=/"); + idxMFile = GribCdmIndex.makeIndexFile("collection name", directory); + assertThat(idxMFile.getPath()).startsWith("cdms3:bucket?path/to/"); + assertThat(idxMFile.getPath()).endsWith("#delimiter=/"); + } + + @Test + public void testMakeNameFromIndexFilename() { + String indexFilename = "/path/to/collection_name" + GribCdmIndex.NCX_SUFFIX; + String expectedName = "collection_name"; + String actualName = GribCdmIndex.makeNameFromIndexFilename(indexFilename); + assertThat(actualName).isEqualTo(expectedName); + + indexFilename = "cdms3:bucket?path/to/collection_name" + GribCdmIndex.NCX_SUFFIX + "#delimiter=/"; + actualName = GribCdmIndex.makeNameFromIndexFilename(indexFilename); + assertThat(actualName).isEqualTo(expectedName); + } + +}