chore(): Added benchmark tests for parquet and hdf5

src/main/java/org/apache/sysds/runtime/io/ReaderHDF5.java

@@ -81,10 +81,17 @@ public class ReaderHDF5 extends MatrixReader {
 	protected static final int HDF5_READ_PARALLEL_MIN_BYTES =
 		getHdf5ReadInt("sysds.hdf5.read.parallel.min.bytes", DEFAULT_HDF5_READ_PARALLEL_MIN_BYTES);
 
+	private static final String HDF5_READ_SPARSE_LAYOUT =
+		System.getProperty("sysds.hdf5.read.sparse.layout", "dense");
+
 	public ReaderHDF5(FileFormatPropertiesHDF5 props) {
 		_props = props;
 	}
 
+	protected static boolean useSparseCOORead() {
+		return "coo".equalsIgnoreCase(HDF5_READ_SPARSE_LAYOUT);
+	}
+
 	@Override
 	public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz)
 		throws IOException, DMLRuntimeException {
@@ -219,14 +226,18 @@ public static long readMatrixFromHDF5(H5ByteReader byteReader, String datasetNam
 				LOG.trace("[HDF5] Forcing dense output for dataset=" + datasetName);
 		}
 		H5RootObject rootObject = H5.H5Fopen(byteReader);
-		H5ContiguousDataset contiguousDataset = H5.H5Dopen(rootObject, datasetName);
-
-		int ncol = (int) rootObject.getCol();
-		LOG.trace("[HDF5] readMatrix dataset=" + datasetName + " dims=" + rootObject.getRow() + "x"
-			+ rootObject.getCol() + " loop=[" + rl + "," + ru + ") dest=" + dest.getNumRows() + "x"
-			+ dest.getNumColumns());
 
 		try {
+			H5ContiguousDataset contiguousDataset = H5.H5Dopen(rootObject, datasetName);
+
+			if(useSparseCOORead())
+				return readSparseCOOFromHDF5(contiguousDataset, dest, clen);
+
+			int ncol = (int) rootObject.getCol();
+			LOG.trace("[HDF5] readMatrix dataset=" + datasetName + " dims=" + rootObject.getRow() + "x"
+				+ rootObject.getCol() + " loop=[" + rl + "," + ru + ") dest=" + dest.getNumRows() + "x"
+				+ dest.getNumColumns());
+
 			double[] row = null;
 			double[] blockBuffer = null;
 			int[] ixBuffer = null;
@@ -360,6 +371,47 @@ public static long readMatrixFromHDF5(H5ByteReader byteReader, String datasetNam
 		return lnnz;
 	}
 
+	private static long readSparseCOOFromHDF5(H5ContiguousDataset dataset, MatrixBlock dest, long expectedClen) {
+		double[] meta = new double[3];
+		dataset.readRowDoubles(0, meta, 0);
+
+		long originalRows = (long) meta[0];
+		long originalCols = (long) meta[1];
+		long nnz = (long) meta[2];
+
+		if(originalRows != dest.getNumRows() || originalCols != dest.getNumColumns())
+			throw new DMLRuntimeException("Sparse COO HDF5 metadata mismatch: "
+				+ originalRows + "x" + originalCols + " vs "
+				+ dest.getNumRows() + "x" + dest.getNumColumns() + ".");
+
+		if(expectedClen >= 0 && originalCols != expectedClen)
+			throw new DMLRuntimeException("Sparse COO HDF5 column metadata mismatch: "
+				+ originalCols + " vs " + expectedClen + ".");
+
+		if(nnz > Integer.MAX_VALUE)
+			throw new DMLRuntimeException("Sparse COO HDF5 nnz exceeds supported row index range: " + nnz);
+
+		if(!dest.isInSparseFormat())
+			throw new DMLRuntimeException("Sparse COO HDF5 read requires sparse output.");
+
+		SparseBlock sb = dest.getSparseBlock();
+		double[] triple = new double[3];
+
+		for(int i = 1; i <= (int) nnz; i++) {
+			dataset.readRowDoubles(i, triple, 0);
+
+			int row = (int) triple[0];
+			int col = (int) triple[1];
+			double val = triple[2];
+
+			// TODO Generalize COO reconstruction by pre-counting nonzeros per row and preallocating sparse rows.
+			sb.allocate(row, 1);
+			sb.append(row, col, val);
+		}
+
+		return nnz;
+	}
+
 	public static MatrixBlock computeHDF5Size(List<Path> files, FileSystem fs, String datasetName, long estnnz)
 		throws IOException, DMLRuntimeException
 	{

src/main/java/org/apache/sysds/runtime/io/ReaderHDF5Parallel.java

@@ -54,7 +54,7 @@
 
 public class ReaderHDF5Parallel extends ReaderHDF5 {
 
-	final private int _numThreads;
+	private final int _numThreads;
 	protected JobConf _job;
 
 	public ReaderHDF5Parallel(FileFormatPropertiesHDF5 props) {
@@ -65,6 +65,9 @@ public ReaderHDF5Parallel(FileFormatPropertiesHDF5 props) {
 	@Override
 	public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz)
 		throws IOException, DMLRuntimeException {
+		// COO stores sparse triples, not dense matrix rows; reuse the COO-aware sequential reader.
+		if(useSparseCOORead())
+			return new ReaderHDF5(_props).readMatrixFromHDFS(fname, rlen, clen, blen, estnnz);
 
 		// prepare file access
 		_job = new JobConf(ConfigurationManager.getCachedJobConf());
@@ -99,7 +102,7 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl
 			ArrayList<ReadHDF5Task> tasks = new ArrayList<>();
 			rlen = src.getNumRows();
 			int blklen = (int) Math.ceil((double) rlen / numParts);
-			for(int i = 0; i < _numThreads & i * blklen < rlen; i++) {
+			for(int i = 0; i < _numThreads && i * blklen < rlen; i++) {
 				int rl = i * blklen;
 				int ru = (int) Math.min((i + 1) * blklen, rlen);
 				Path newPath = HDFSTool.isDirectory(fs, path) ? 

src/main/java/org/apache/sysds/runtime/io/WriterHDF5.java

@@ -39,12 +39,43 @@
 
 public class WriterHDF5 extends MatrixWriter {
 
+	private static final int DEFAULT_HDF5_WRITE_BATCH_ROWS = 1024;
+	private static final int DEFAULT_HDF5_WRITE_BATCH_BYTES = 1024 * 1024;
+
+	private static final int HDF5_WRITE_BATCH_ROWS =
+		getHdf5WriteInt("sysds.hdf5.write.batch.rows", DEFAULT_HDF5_WRITE_BATCH_ROWS);
+
+	private static final int HDF5_WRITE_BATCH_BYTES =
+		getHdf5WriteInt("sysds.hdf5.write.batch.bytes", DEFAULT_HDF5_WRITE_BATCH_BYTES);
+
+	private static final String HDF5_WRITE_SPARSE_LAYOUT =
+		System.getProperty("sysds.hdf5.write.sparse.layout", "dense");
+
+	private static int getHdf5WriteInt(String key, int defaultValue) {
+		String value = System.getProperty(key);
+		if(value == null)
+			return defaultValue;
+
+		try {
+			int parsed = Integer.parseInt(value.trim());
+			return parsed > 0 ? parsed : defaultValue;
+		}
+		catch(NumberFormatException ex) {
+			return defaultValue;
+		}
+	}
+
 	protected static FileFormatPropertiesHDF5 _props = null;
 
 	public WriterHDF5(FileFormatPropertiesHDF5 _props) {
 		WriterHDF5._props = _props;
 	}
 
+	private static boolean useSparseCOO(MatrixBlock src) {
+		return src.isInSparseFormat()
+			&& "coo".equalsIgnoreCase(HDF5_WRITE_SPARSE_LAYOUT);
+	}
+
 	@Override
 	public void writeMatrixToHDFS(MatrixBlock src, String fname, long rlen, long clen, int blen, long nnz, boolean diag)
 		throws IOException, DMLRuntimeException
@@ -65,8 +96,10 @@ public void writeMatrixToHDFS(MatrixBlock src, String fname, long rlen, long cle
 		//if the file already exists on HDFS, remove it.
 		HDFSTool.deleteFileIfExistOnHDFS(fname);
 
-		//core write (sequential/parallel)
-		writeHDF5MatrixToHDFS(path, job, fs, src);
+		if(useSparseCOO(src))
+			writeSparseCOOMatrixToFile(path, fs, src, rlen, clen, nnz);
+		else
+			writeHDF5MatrixToHDFS(path, job, fs, src);
 
 		IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, path);
 	}
@@ -88,48 +121,149 @@ protected static void writeHDF5MatrixToFile(Path path, JobConf job, FileSystem f
 		throws IOException 
 	{
 		int clen = src.getNumColumns();
-		BufferedOutputStream bos = new BufferedOutputStream(fs.create(path, true));
 		String datasetName = _props.getDatasetName();
-		H5RootObject rootObject = H5.H5Screate(bos, src.getNumRows(), src.getNumColumns());
-		H5.H5Dcreate(rootObject, src.getNumRows(), src.getNumColumns(), datasetName);
+
+		try(BufferedOutputStream bos = new BufferedOutputStream(fs.create(path, true))) {
+			H5RootObject rootObject = H5.H5Screate(bos, src.getNumRows(), src.getNumColumns());
+			H5.H5Dcreate(rootObject, src.getNumRows(), src.getNumColumns(), datasetName);
 
-		//write headers
-		if(rl == 0) {
-			H5.H5WriteHeaders(rootObject);
+			if(rl == 0)
+				H5.H5WriteHeaders(rootObject);
+
+			int batchRows = getWriteBatchRows(clen);
+			if(src.isInSparseFormat())
+				writeSparseBatched(rootObject, src, rl, rlen, clen, batchRows);
+			else
+				writeDenseBatched(rootObject, src, rl, rlen, clen, batchRows);
 		}
+	}
 
-		try {
-			// Write the data to the datasets.
-			double[] row = new double[clen];
-			if( src.isInSparseFormat() ) {
-				SparseBlock sb = src.getSparseBlock();
-				for(int i = rl; i < rlen; i++) {
-					Arrays.fill(row, 0);
-					if( !sb.isEmpty(i) ) {
-						int apos = sb.pos(i);
-						int alen = sb.size(i);
-						double[] avals = sb.values(i);
-						int[] aix = sb.indexes(i);
-						for(int j = apos; j < apos+alen; j++)
-							row[aix[j]] = avals[j];
-					}
-					H5.H5Dwrite(rootObject, row);
-				}
+	private static int getWriteBatchRows(int clen) {
+		long rowBytes = (long) clen * Double.BYTES;
+
+		int rowsByBytes = rowBytes > 0 ? (int) Math.max(1, HDF5_WRITE_BATCH_BYTES / rowBytes) : 1;
+
+		int rows = Math.max(1, Math.min(HDF5_WRITE_BATCH_ROWS, rowsByBytes));
+		rows = roundDownPowerOfTwo(rows);
+		long cells = (long) rows * clen;
+
+		if(cells > Integer.MAX_VALUE)
+			throw new DMLRuntimeException("HDF5 write batch too large: " + rows + " x " + clen);
+
+		return rows;
+	}
+
+	private static int roundDownPowerOfTwo(int value) {
+		int ret = 1;
+		while(ret <= value / 2)
+			ret *= 2;
+		return ret;
+	}
+
+	private static void writeDenseBatched(H5RootObject rootObject, MatrixBlock src, int rl, int ru, int clen, int batchRows) {
+
+		DenseBlock db = src.getDenseBlock();
+		double[] batch = new double[batchRows * clen];
+
+		for(int rowStart = rl; rowStart < ru; rowStart += batchRows) {
+			int rows = Math.min(batchRows, ru - rowStart);
+
+			for(int r = 0; r < rows; r++) {
+				int srcRow = rowStart + r;
+				int off = r * clen;
+
+				for(int c = 0; c < clen; c++)
+					batch[off + c] = db.get(srcRow, c);
 			}
-			else {
-				DenseBlock db = src.getDenseBlock();
-				for(int i = rl; i < rlen; i++) {
-					for(int j = 0; j < clen;j++) {
-						double lvalue = db!=null ? db.get(i, j) : 0;
-						row[j] = lvalue;
-					}
-					H5.H5Dwrite(rootObject, row);
-				}
+
+			if(rows == batchRows)
+				H5.H5Dwrite(rootObject, batch);
+			else
+				H5.H5Dwrite(rootObject, Arrays.copyOf(batch, rows * clen));
+		}
+	}
+
+	private static void writeSparseBatched(H5RootObject rootObject, MatrixBlock src, int rl, int ru, int clen, int batchRows) {
+		SparseBlock sb = src.getSparseBlock();
+		double[] batch = new double[batchRows * clen];
+
+		for(int rowStart = rl; rowStart < ru; rowStart += batchRows) {
+			int rows = Math.min(batchRows, ru - rowStart);
+			Arrays.fill(batch, 0, rows * clen, 0.0);
+
+			for(int r = 0; r < rows; r++) {
+				int srcRow = rowStart + r;
+
+				if(sb == null || sb.isEmpty(srcRow))
+					continue;
+
+				int apos = sb.pos(srcRow);
+				int alen = sb.size(srcRow);
+				int[] aix = sb.indexes(srcRow);
+				double[] avals = sb.values(srcRow);
+
+				int off = r * clen;
+				for(int k = apos; k < apos + alen; k++)
+					batch[off + aix[k]] = avals[k];
 			}
+
+			if(rows == batchRows)
+				H5.H5Dwrite(rootObject, batch);
+			else
+				H5.H5Dwrite(rootObject, Arrays.copyOf(batch, rows * clen));
 		}
-		finally {
-			IOUtilFunctions.closeSilently(bos);
+	}
+
+	private static void writeSparseCOOMatrixToFile(Path path, FileSystem fs, MatrixBlock src, long rlen, long clen, long nnz) throws IOException {
+		String datasetName = _props.getDatasetName();
+
+		long cooRows = nnz + 1;
+		long cooCols = 3;
+
+		try(BufferedOutputStream bos = new BufferedOutputStream(fs.create(path, true))) {
+			H5RootObject rootObject = H5.H5Screate(bos, cooRows, cooCols);
+			H5.H5Dcreate(rootObject, cooRows, cooCols, datasetName);
+			H5.H5WriteHeaders(rootObject);
+
+			H5.H5Dwrite(rootObject, new double[] {
+				(double) rlen,
+				(double) clen,
+				(double) nnz
+			});
+
+			writeSparseCOOEntries(rootObject, src);
+		}
+	}
+
+	private static void writeSparseCOOEntries(H5RootObject rootObject, MatrixBlock src) {
+		SparseBlock sb = src.getSparseBlock();
+		int batchRows = getWriteBatchRows(3);
+		double[] batch = new double[batchRows * 3];
+
+		int pos = 0;
+		for(int i = 0; i < src.getNumRows(); i++) {
+			if(sb == null || sb.isEmpty(i))
+				continue;
+
+			int apos = sb.pos(i);
+			int alen = sb.size(i);
+			int[] aix = sb.indexes(i);
+			double[] avals = sb.values(i);
+
+			for(int k = apos; k < apos + alen; k++) {
+				batch[pos++] = i;
+				batch[pos++] = aix[k];
+				batch[pos++] = avals[k];
+
+				if(pos == batch.length) {
+					H5.H5Dwrite(rootObject, batch);
+					pos = 0;
+				}
+			}
 		}
+
+		if(pos > 0)
+			H5.H5Dwrite(rootObject, Arrays.copyOf(batch, pos));
 	}
 
 	@Override

src/main/java/org/apache/sysds/runtime/io/hdf5/H5.java

@@ -219,8 +219,8 @@ public static void H5WriteHeaders(H5RootObject rootObject) {
 	public static void H5Dwrite(H5RootObject rootObject, double[] data) {
 		try {
 			H5BufferBuilder bb = new H5BufferBuilder();
-			for(Double d : data) {
-				bb.writeDouble(d);
+			for(int i = 0; i < data.length; i++) {
+				bb.writeDouble(data[i]);
 			}
 			rootObject.getBufferedOutputStream().write(bb.noOrderBuild().array());
 		}

src/test/java/org/apache/sysds/performance/Main.java

@@ -31,6 +31,7 @@
 import org.apache.sysds.performance.generators.GenMatrices;
 import org.apache.sysds.performance.generators.IGenerate;
 import org.apache.sysds.performance.generators.MatrixFile;
+import org.apache.sysds.performance.io.HDF5IOBenchmark;
 import org.apache.sysds.performance.matrix.MatrixAppend;
 import org.apache.sysds.performance.matrix.MatrixBinaryCellPerf;
 import org.apache.sysds.performance.matrix.MatrixMultiplicationPerf;
@@ -143,6 +144,12 @@ private static void exec(int prog, String[] args) throws Exception {
 			case 1009:
 				MatrixMultiplicationPerf.main(args);
 				break;
+			case 1010:
+				ParquetIOBenchmark.main(args);
+				break;
+			case 1011:
+				HDF5IOBenchmark.main(args);
+				break;
 			default:
 				break;
 		}