Skip to content

Commit aab6ded

Browse files
lesserwhirlslesserwhirls
committed
Use remote GRIB index files,if they exist
Remote index files will be used, if they exist. If a remote index file needs to be updated, the operation will fail. Will revisit in the future if we want to support writing/updating remote index files directly from netCDF-Java. We could write new index files locally in the case where a remote index file needs updated, but that needs further consideration.
1 parent 3c0c0ec commit aab6ded

File tree

9 files changed

+184
-35
lines changed

9 files changed

+184
-35
lines changed

‎cdm/core/src/main/java/thredds/inventory/MFiles.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.slf4j.Logger;
1313
import org.slf4j.LoggerFactory;
1414
import thredds.filesystem.MFileOS;
15+
import thredds.filesystem.MFileOS7;
1516
import ucar.nc2.internal.ncml.NcmlReader;
1617

1718
/**
@@ -66,4 +67,14 @@ public static MFile createIfExists(String location) {
6667
final MFile mFile = create(location);
6768
return mFile.exists() ? mFile : null;
6869
}
70+
71+
/**
72+
* Checks if MFile represents a file stored on a local filesystem
73+
*
74+
* @param mfile MFile to check
75+
* @return true if MFile is a local filesystem
76+
*/
77+
public static boolean isLocal(MFile mfile) {
78+
return mfile instanceof MFileOS || mfile instanceof MFileOS7;
79+
}
6980
}

‎grib/src/main/java/ucar/nc2/grib/GribIndexCache.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,17 @@ public static MFile getFileOrCache(String fileLocation) {
5151
* @return existing file if you can find it, else null
5252
*/
5353
public static MFile getExistingFileOrCache(String fileLocation) {
54+
MFile idxMFile = MFiles.create(fileLocation);
55+
if (!MFiles.isLocal(idxMFile)) {
56+
// for remote file systems, check to see if the index file exists and, if so, use it
57+
// note: the remote index file may require updating, which isn't support at this point,
58+
// so opening the remote GRIB file may ultimately fail.
59+
if (idxMFile.exists()) {
60+
return idxMFile;
61+
}
62+
}
63+
// if the GRIB index file is local OR the remote index does not exist, check
64+
// the DiskCache.
5465
File result = getDiskCache2().getExistingFileOrCache(fileLocation);
5566
if (result == null && Grib.debugGbxIndexOnly && fileLocation.endsWith(".gbx9.ncx4")) { // might create only from
5667
// gbx9 for debugging

‎grib/src/main/java/ucar/nc2/grib/collection/Grib1CollectionWriter.java‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -91,23 +91,26 @@ public Set<Long> getCoordinateRuntimes() {
9191
*/
9292

9393
// indexFile is in the cache
94-
boolean writeIndex(String name, MFile idxFile, CoordinateRuntime masterRuntime, List<Group> groups, List<MFile> files,
95-
GribCollectionImmutable.Type type, CalendarDateRange dateRange) throws IOException {
94+
boolean writeIndex(String name, MFile idxMFile, CoordinateRuntime masterRuntime, List<Group> groups,
95+
List<MFile> files, GribCollectionImmutable.Type type, CalendarDateRange dateRange) throws IOException {
96+
if (!MFiles.isLocal(idxMFile)) {
97+
throw new IllegalArgumentException("Only local file systems are supported for index creation");
98+
}
9699
Grib1Record first = null; // take global metadata from here
97100
boolean deleteOnClose = false;
98101

99-
if (idxFile.exists()) {
100-
RandomAccessFile.eject(idxFile.getPath());
101-
if (idxFile instanceof thredds.filesystem.MFileOS) {
102-
File f = new File(idxFile.getPath());
102+
if (idxMFile.exists()) {
103+
RandomAccessFile.eject(idxMFile.getPath());
104+
if (idxMFile instanceof thredds.filesystem.MFileOS) {
105+
File f = new File(idxMFile.getPath());
103106
if (!f.delete()) {
104-
logger.warn(" gc1 cant delete index file {}", idxFile.getPath());
107+
logger.warn(" gc1 cant delete index file {}", idxMFile.getPath());
105108
}
106109
}
107110
}
108-
logger.debug(" createIndex for {}", idxFile.getPath());
111+
logger.debug(" createIndex for {}", idxMFile.getPath());
109112

110-
try (RandomAccessFile raf = new RandomAccessFile(idxFile.getPath(), "rw")) {
113+
try (RandomAccessFile raf = new RandomAccessFile(idxMFile.getPath(), "rw")) {
111114
raf.order(RandomAccessFile.BIG_ENDIAN);
112115

113116
//// header message
@@ -234,10 +237,10 @@ boolean writeIndex(String name, MFile idxFile, CoordinateRuntime masterRuntime,
234237
} finally {
235238

236239
// remove it on failure
237-
if (deleteOnClose && idxFile instanceof thredds.filesystem.MFileOS) {
238-
File f = new File(idxFile.getPath());
240+
if (deleteOnClose && idxMFile instanceof thredds.filesystem.MFileOS) {
241+
File f = new File(idxMFile.getPath());
239242
if (!f.delete())
240-
logger.error(" gc1 cant deleteOnClose index file {}", idxFile.getPath());
243+
logger.error(" gc1 cant deleteOnClose index file {}", idxMFile.getPath());
241244
}
242245
}
243246
}

‎grib/src/main/java/ucar/nc2/grib/collection/Grib2CollectionWriter.java‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -92,23 +92,26 @@ public List<Coordinate> getCoordinates() {
9292
* GribCollectionIndex (sizeIndex bytes)
9393
*/
9494

95-
boolean writeIndex(String name, MFile idxFile, CoordinateRuntime masterRuntime, List<Group> groups, List<MFile> files,
96-
GribCollectionImmutable.Type type, CalendarDateRange dateRange) throws IOException {
95+
boolean writeIndex(String name, MFile idxMFile, CoordinateRuntime masterRuntime, List<Group> groups,
96+
List<MFile> files, GribCollectionImmutable.Type type, CalendarDateRange dateRange) throws IOException {
97+
if (!MFiles.isLocal(idxMFile)) {
98+
throw new IllegalArgumentException("Only local file systems are supported for index creation");
99+
}
97100
Grib2Record first = null; // take global metadata from here
98101
boolean deleteOnClose = false;
99102

100-
if (idxFile.exists()) {
101-
RandomAccessFile.eject(idxFile.getPath());
102-
if (idxFile instanceof thredds.filesystem.MFileOS) {
103-
File f = new File(idxFile.getPath());
103+
if (idxMFile.exists()) {
104+
RandomAccessFile.eject(idxMFile.getPath());
105+
if (idxMFile instanceof thredds.filesystem.MFileOS) {
106+
File f = new File(idxMFile.getPath());
104107
if (!f.delete()) {
105-
logger.error("gc2 cant delete index file {}", idxFile.getPath());
108+
logger.error("gc2 cant delete index file {}", idxMFile.getPath());
106109
}
107110
}
108111
}
109-
logger.debug(" createIndex for {}", idxFile.getPath());
112+
logger.debug(" createIndex for {}", idxMFile.getPath());
110113

111-
try (RandomAccessFile raf = new RandomAccessFile(idxFile.getPath(), "rw")) {
114+
try (RandomAccessFile raf = new RandomAccessFile(idxMFile.getPath(), "rw")) {
112115
//// header message
113116
raf.order(RandomAccessFile.BIG_ENDIAN);
114117
raf.write(MAGIC_START.getBytes(StandardCharsets.UTF_8));
@@ -230,10 +233,10 @@ boolean writeIndex(String name, MFile idxFile, CoordinateRuntime masterRuntime,
230233

231234
} finally {
232235
// remove it on failure
233-
if (deleteOnClose && idxFile instanceof thredds.filesystem.MFileOS) {
234-
File f = new File(idxFile.getPath());
236+
if (deleteOnClose && idxMFile instanceof thredds.filesystem.MFileOS) {
237+
File f = new File(idxMFile.getPath());
235238
if (!f.delete())
236-
logger.error(" gc2 cant deleteOnClose index file {}", idxFile.getPath());
239+
logger.error(" gc2 cant deleteOnClose index file {}", idxMFile.getPath());
237240
}
238241
}
239242

‎grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,9 @@
88
import com.google.protobuf.ByteString;
99
import java.io.InputStream;
1010
import java.nio.charset.StandardCharsets;
11-
import thredds.filesystem.MFileOS;
12-
import thredds.filesystem.MFileOS7;
1311
import thredds.inventory.CollectionUpdateType;
1412
import thredds.inventory.MFile;
13+
import thredds.inventory.MFiles;
1514
import ucar.nc2.NetcdfFiles;
1615
import ucar.nc2.grib.GribIndex;
1716
import ucar.nc2.grib.GribIndexCache;
@@ -173,7 +172,7 @@ public boolean makeIndex(String filename, RandomAccessFile dataRaf) throws IOExc
173172
String idxPathTmp = GribUtils.makeIndexFileName(idxPath, ".tmp");
174173
MFile idxMFileTmp = GribIndexCache.getFileOrCache(idxPathTmp);
175174

176-
if (!(idxMFile instanceof MFileOS || idxMFile instanceof MFileOS7)) {
175+
if (!MFiles.isLocal(idxMFile)) {
177176
throw new IllegalArgumentException("Only local file systems are supported for index creation");
178177
}
179178

‎grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,9 @@
88
import com.google.protobuf.ByteString;
99
import java.io.InputStream;
1010
import java.nio.charset.StandardCharsets;
11-
import thredds.filesystem.MFileOS;
12-
import thredds.filesystem.MFileOS7;
1311
import thredds.inventory.CollectionUpdateType;
1412
import thredds.inventory.MFile;
13+
import thredds.inventory.MFiles;
1514
import ucar.nc2.NetcdfFiles;
1615
import ucar.nc2.grib.GribIndex;
1716
import ucar.nc2.grib.GribIndexCache;
@@ -208,7 +207,7 @@ public boolean makeIndex(String filename, RandomAccessFile dataRaf) throws IOExc
208207
String idxPathTmp = GribUtils.makeIndexFileName(idxPath, ".tmp");
209208
MFile idxMFileTmp = GribIndexCache.getFileOrCache(idxPathTmp);
210209

211-
if (!(idxMFile instanceof MFileOS || idxMFile instanceof MFileOS7)) {
210+
if (!MFiles.isLocal(idxMFile)) {
212211
throw new IllegalArgumentException("Only local file systems are supported for index creation");
213212
}
214213

grib/src/test/java/ucar/nc2/grib/TestSingleGribFileS3.java renamed to grib/src/test/java/ucar/nc2/grib/TestSingleGribFileS3LocalIndex.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
import ucar.nc2.util.CompareNetcdf2;
1919
import ucar.unidata.util.test.TestDir;
2020

21-
public class TestSingleGribFileS3 {
21+
public class TestSingleGribFileS3LocalIndex {
2222

2323
private static final String GRIB1_BUCKET_KEY = "thredds-test-data?test-grib-without-index/radar_national.grib1";
2424
private static final String GRIB2_BUCKET_KEY = "thredds-test-data?test-grib-without-index/cosmo-eu.grib2";

‎grib/src/test/java/ucar/nc2/grib/TestSingleGribFileS3RemoteIndex.java‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/*
2+
* Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata
3+
* See LICENSE for license information.
4+
*/
5+
6+
package ucar.nc2.grib;
7+
8+
import static com.google.common.truth.Truth.assertThat;
9+
import static org.junit.Assert.assertThrows;
10+
import static org.junit.Assert.fail;
11+
12+
import java.io.IOException;
13+
import java.util.Formatter;
14+
import org.junit.Test;
15+
import ucar.ma2.Array;
16+
import ucar.nc2.Variable;
17+
import ucar.nc2.dataset.NetcdfDataset;
18+
import ucar.nc2.dataset.NetcdfDatasets;
19+
import ucar.nc2.util.CompareNetcdf2;
20+
import ucar.unidata.util.test.TestDir;
21+
22+
/**
23+
* Tests reading single grib files from S3 when index files already exists in the bucket
24+
*/
25+
public class TestSingleGribFileS3RemoteIndex {
26+
27+
private static final String GRIB1_BUCKET_KEY = "thredds-test-data?test-grib-index/radar_national.grib1";
28+
private static final String GRIB2_BUCKET_KEY = "thredds-test-data?test-grib-index/cosmo-eu.grib2";
29+
private static final String GRIB1_LOCAL = TestDir.localTestDataDir + "radar_national.grib1";
30+
private static final String GRIB2_LOCAL = TestDir.localTestDataDir + "cosmo-eu.grib2";
31+
32+
@Test
33+
public void testGrib1S3Full() throws IOException {
34+
String location = String.format("cdms3://s3.us-east-1.amazonaws.com/%s#delimiter=/", GRIB1_BUCKET_KEY);
35+
// .ncx4 object is older than .gbx9 object, so an update is required (which isn't supported)
36+
assertThrows(IOException.class, () -> basicDatasetValidation(location));
37+
}
38+
39+
@Test
40+
public void testGrib1S3Short() throws IOException {
41+
String location = String.format("cdms3:%s#delimiter=/", GRIB1_BUCKET_KEY);
42+
// .ncx4 object is older than .gbx9 object, so an update is required (which isn't supported)
43+
assertThrows(IOException.class, () -> basicDatasetValidation(location));
44+
}
45+
46+
@Test
47+
public void testGrib1SFullNoDelimiter() throws IOException {
48+
String location = String.format("cdms3://s3.us-east-1.amazonaws.com/%s", GRIB1_BUCKET_KEY);
49+
// .ncx4 object is older than .gbx9 object, so an update is required (which isn't supported)
50+
assertThrows(IOException.class, () -> basicDatasetValidation(location));
51+
}
52+
53+
@Test
54+
public void testGrib1S3ShortNoDelimiter() throws IOException {
55+
String location = String.format("cdms3:%s", GRIB1_BUCKET_KEY);
56+
// .ncx4 object is older than .gbx9 object, so an update is required (which isn't supported)
57+
assertThrows(IOException.class, () -> basicDatasetValidation(location));
58+
}
59+
60+
@Test
61+
public void testGrib2S3Full() throws IOException {
62+
// .ncx4 is missing, so will be written locally and used successfully
63+
String location = String.format("cdms3://s3.us-east-1.amazonaws.com/%s#delimiter=/", GRIB2_BUCKET_KEY);
64+
basicDatasetValidation(location);
65+
}
66+
67+
@Test
68+
public void testGrib2S3Short() throws IOException {
69+
// .ncx4 is missing, so will be written locally and used successfully
70+
String location = String.format("cdms3:%s#delimiter=/", GRIB2_BUCKET_KEY);
71+
basicDatasetValidation(location);
72+
}
73+
74+
@Test
75+
public void testGrib2SFullNoDelimiter() throws IOException {
76+
// .ncx4 is missing, so will be written locally and used successfully
77+
String location = String.format("cdms3://s3.us-east-1.amazonaws.com/%s", GRIB2_BUCKET_KEY);
78+
basicDatasetValidation(location);
79+
}
80+
81+
@Test
82+
public void testGrib2S3ShortNoDelimiter() throws IOException {
83+
// .ncx4 is missing, so will be written locally and used successfully
84+
String location = String.format("cdms3:%s", GRIB2_BUCKET_KEY);
85+
basicDatasetValidation(location);
86+
}
87+
88+
@Test
89+
public void compareGrib1() throws IOException {
90+
// .ncx4 object is older than .gbx9 object, so an update is required (which isn't supported)
91+
String location = String.format("cdms3://s3.us-east-1.amazonaws.com/%s#delimiter=/", GRIB1_BUCKET_KEY);
92+
assertThrows(IOException.class, () -> compareWithLocal(location, GRIB1_LOCAL));
93+
}
94+
95+
@Test
96+
public void compareGrib2() throws IOException {
97+
// .ncx4 is missing, so will be written locally and used successfully
98+
String location = String.format("cdms3://s3.us-east-1.amazonaws.com/%s#delimiter=/", GRIB2_BUCKET_KEY);
99+
compareWithLocal(location, GRIB2_LOCAL);
100+
}
101+
102+
private static void compareWithLocal(String remoteLocation, String localLocation) throws IOException {
103+
try (NetcdfDataset remoteNetcdfDataset = NetcdfDatasets.openDataset(remoteLocation);
104+
NetcdfDataset localNetcdfDataset = NetcdfDatasets.openDataset(localLocation)) {
105+
Formatter f = new Formatter();
106+
CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true);
107+
if (!compare.compare(localNetcdfDataset, remoteNetcdfDataset, null)) {
108+
System.out.printf("Compare %s%n%s%n", localLocation, f);
109+
fail();
110+
}
111+
}
112+
}
113+
114+
private static void basicDatasetValidation(String location) throws IOException {
115+
try (NetcdfDataset netcdfDataset = NetcdfDatasets.openDataset(location)) {
116+
assertThat(netcdfDataset.getLastModified()).isGreaterThan(0);
117+
assertThat(netcdfDataset.getVariables()).isNotEmpty();
118+
119+
Variable variable = netcdfDataset.getVariables().get(1);
120+
Array data = variable.read();
121+
assertThat(data).isNotNull();
122+
}
123+
}
124+
}

‎uicdm/src/main/java/ucar/nc2/ui/MFileTable.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
11
/*
2-
* Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
2+
* Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
33
* See LICENSE for license information.
44
*/
55

66
package ucar.nc2.ui;
77

8-
import thredds.filesystem.MFileOS;
9-
import thredds.filesystem.MFileOS7;
108
import thredds.inventory.MFile;
9+
import thredds.inventory.MFiles;
1110
import ucar.nc2.time.CalendarDateFormatter;
1211
import ucar.ui.widget.BAMutil;
1312
import ucar.ui.widget.IndependentWindow;
@@ -102,7 +101,7 @@ public void save() {
102101
}
103102

104103
public void setFiles(MFile mdir, Collection<MFile> files) {
105-
if (!(mdir instanceof MFileOS || mdir instanceof MFileOS7)) {
104+
if (!MFiles.isLocal(mdir)) {
106105
throw new IllegalArgumentException("Only local file systems are supported for index creation");
107106
}
108107
setFiles(new File(mdir.getPath()), files);

0 commit comments

Comments
 (0)
⚡