From 3b5eff77e98451e3f8ee1e8902ba98f3a0010a99 Mon Sep 17 00:00:00 2001 From: Sean Arms <67096+lesserwhirls@users.noreply.github.com> Date: Sat, 13 Jun 2026 06:21:22 -0600 Subject: [PATCH] Extend limited reader in H5headerNew to handle filtered data Test file added to test dataset collection and is checked with other HDF5 files by integration tests. Note: old API handled this case without issue, so no changes needed there. Fixes unidata/netcdf-java#1556. --- .../nc2/internal/iosp/hdf5/H5headerNew.java | 36 ++++++++++++++++++- .../internal/iosp/hdf5/H5tiledLayoutBB.java | 29 ++++++++++++--- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5headerNew.java b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5headerNew.java index ec0a345211..df580be698 100644 --- a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5headerNew.java +++ b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5headerNew.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -25,6 +25,7 @@ import ucar.ma2.ArrayStructure; import ucar.ma2.ArrayStructureBB; import ucar.ma2.DataType; +import ucar.ma2.Index; import ucar.ma2.IndexIterator; import ucar.ma2.InvalidRangeException; import ucar.ma2.Section; @@ -59,6 +60,7 @@ import ucar.nc2.write.NetcdfFileFormat; import ucar.nc2.iosp.IospHelper; import ucar.nc2.iosp.Layout; +import ucar.nc2.iosp.LayoutBB; import ucar.nc2.iosp.LayoutRegular; import ucar.nc2.iosp.hdf5.DataBTree; import ucar.nc2.iosp.hdf5.H5headerIF; @@ -1957,6 +1959,29 @@ Object getFillValueNonDefault() { Array readArray() throws IOException { int[] shape = mds.dimLength; DataType dataType = typeInfo.dataType; + + if (useFillValue) { + Object pa = IospHelper.makePrimitiveArray((int) Index.computeSize(shape), dataType, getFillValue()); + if (dataType == DataType.CHAR) + pa = IospHelper.convertByteToChar((byte[]) pa); + return Array.factory(dataType, shape, pa); + } + + // filtered, must read and decode by chunk + if (mfp != null) { + ByteOrder bo = (typeInfo.endian == 0) ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN; + LayoutBB layoutBB; + try { + layoutBB = new H5tiledLayoutBB(this, shape, dataType.getSize(), new Section(shape), getRandomAccessFile(), + mfp.getFilters(), bo); + } catch (InvalidRangeException e) { + // should not happen because we passed in the full shape + throw new IllegalStateException(); + } + Object data = IospHelper.readDataFill(layoutBB, dataType, getFillValue()); + return Array.factory(dataType, shape, data); + } + Layout layout; try { if (isChunked) { @@ -1977,6 +2002,15 @@ Array readArray() throws IOException { String readString() throws IOException { int[] shape = new int[] {mdt.byteSize}; DataType dataType = typeInfo.dataType; + + if (useFillValue) { + Object pa = IospHelper.makePrimitiveArray((int) Index.computeSize(shape), dataType, getFillValue()); + if (dataType == DataType.CHAR) + pa = IospHelper.convertByteToChar((byte[]) pa); + Array dataArray = Array.factory(dataType, shape, pa); + return (dataArray instanceof ArrayChar.D1) ? ((ArrayChar) dataArray).getString() : ""; + } + Layout layout; try { if (isChunked) { diff --git a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5tiledLayoutBB.java b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5tiledLayoutBB.java index e025e9bccf..7a9672aa7d 100644 --- a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5tiledLayoutBB.java +++ b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5tiledLayoutBB.java @@ -1,7 +1,8 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ + package ucar.nc2.internal.iosp.hdf5; import com.google.common.primitives.Ints; @@ -71,9 +72,27 @@ public class H5tiledLayoutBB implements LayoutBB { */ public H5tiledLayoutBB(Variable v2, Section wantSection, RandomAccessFile raf, H5objects.Filter[] filterProps, ByteOrder byteOrder) throws InvalidRangeException, IOException { - wantSection = Section.fill(wantSection, v2.getShape()); + this((H5headerNew.Vinfo) v2.getSPobject(), v2.getShape(), v2.getElementSize(), wantSection, raf, filterProps, + byteOrder); + } + + /** + * This constructor can be used when the Variable is not yet built. + * + * @param vinfo the data object + * @param varShape the variable's shape + * @param elemSize the variable's element size in bytes + * @param wantSection the wanted section of data, contains a List of Range objects. must be complete + * @param raf the RandomAccessFile + * @param filterProps set of filter properties from which filter object will be created + * @throws InvalidRangeException if section invalid for this variable + * @throws IOException on io error + */ + H5tiledLayoutBB(H5headerNew.Vinfo vinfo, int[] varShape, int elemSize, Section wantSection, RandomAccessFile raf, + H5objects.Filter[] filterProps, ByteOrder byteOrder) throws InvalidRangeException, IOException { + + wantSection = Section.fill(wantSection, varShape); - H5headerNew.Vinfo vinfo = (H5headerNew.Vinfo) v2.getSPobject(); assert vinfo.isChunked; assert vinfo.btree != null; @@ -82,7 +101,7 @@ public H5tiledLayoutBB(Variable v2, Section wantSection, RandomAccessFile raf, H for (int i = 0; i < filterProps.length; i++) { // add var info to filter props Map props = filterProps[i].getProperties(); - props.put(Filters.Keys.ELEM_SIZE, v2.getElementSize()); + props.put(Filters.Keys.ELEM_SIZE, elemSize); // try to get filter by name or id, throw if not recognized filter try { filters[i] = Filters.getFilter(props); @@ -95,7 +114,7 @@ public H5tiledLayoutBB(Variable v2, Section wantSection, RandomAccessFile raf, H // we have to translate the want section into the same rank as the storageSize, in order to be able to call // Section.intersect(). It appears that storageSize (actually msl.chunkSize) may have an extra dimension, relative // to the Variable. - DataType dtype = v2.getDataType(); + DataType dtype = vinfo.getNCDataType(); if ((dtype == DataType.CHAR) && (wantSection.getRank() < vinfo.storageSize.length)) { this.want = Section.builder().appendRanges(wantSection.getRanges()).appendRange(1).build(); } else {