From 68684f6fa6233f3e35f2a2e5e4d441a02177710b Mon Sep 17 00:00:00 2001 From: t99-i Date: Sun, 19 Jul 2026 16:16:07 +0200 Subject: [PATCH 1/2] [MINOR] fixed incorrect column name indexing (off-by-one error) in CSV header writing --- .../instructions/spark/utils/FrameRDDConverterUtils.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/utils/FrameRDDConverterUtils.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/utils/FrameRDDConverterUtils.java index 9371d43094c..3a35e43a98c 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/utils/FrameRDDConverterUtils.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/utils/FrameRDDConverterUtils.java @@ -663,9 +663,9 @@ public Iterator call(Tuple2 arg0) //handle header information and frame meta data if( ix==1 ) { if( _props.hasHeader() ) { - for(int j = 1; j <= blk.getNumColumns(); j++) { - sb.append(blk.getColumnNames()[j] - + ((j Date: Sat, 25 Jul 2026 23:41:59 +0200 Subject: [PATCH 2/2] [SYSTEMDS-3857] - added corresponding test which should fail with the old implementation --- .../io/csv/FrameCSVHeaderNamesTest.java | 104 ++++++++++++++++++ .../functions/frame/FrameCSVHeaderNames.dml | 23 ++++ 2 files changed, 127 insertions(+) create mode 100644 src/test/java/org/apache/sysds/test/functions/io/csv/FrameCSVHeaderNamesTest.java create mode 100644 src/test/scripts/functions/frame/FrameCSVHeaderNames.dml diff --git a/src/test/java/org/apache/sysds/test/functions/io/csv/FrameCSVHeaderNamesTest.java b/src/test/java/org/apache/sysds/test/functions/io/csv/FrameCSVHeaderNamesTest.java new file mode 100644 index 00000000000..5995af1bc2a --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/io/csv/FrameCSVHeaderNamesTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.io.csv; + +import java.util.Collections; + +import org.apache.sysds.api.DMLScript; +import org.apache.sysds.common.Types; +import org.apache.sysds.common.Types.ExecType; +import org.apache.sysds.common.Types.FileFormat; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.io.FileFormatPropertiesCSV; +import org.apache.sysds.runtime.io.FrameReader; +import org.apache.sysds.runtime.io.FrameReaderFactory; +import org.apache.sysds.runtime.io.FrameWriter; +import org.apache.sysds.runtime.io.FrameWriterFactory; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +public class FrameCSVHeaderNamesTest extends AutomatedTestBase { + private static final String TEST_NAME = "FrameCSVHeaderNames"; + private static final String TEST_DIR = "functions/frame/"; + private static final String TEST_CLASS_DIR = + TEST_DIR + FrameCSVHeaderNamesTest.class.getSimpleName() + "/"; + + private static final int ROWS = 42; + private static final String[] COLUMN_NAMES = new String[] {"customer_id", "signup_date", "score"}; + + @Override + public void setUp() { + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"B"})); + } + + @Test + public void testFrameCSVHeaderNamesCP() { + runCSVHeaderNamesTest(ExecType.CP); + } + + @Test + public void testFrameCSVHeaderNamesSpark() { + runCSVHeaderNamesTest(ExecType.SPARK); + } + + private void runCSVHeaderNamesTest(ExecType et) { + Types.ExecMode platformOld = setExecMode(et); + boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; + setOutputBuffering(true); + try { + getAndLoadTestConfiguration(TEST_NAME); + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + programArgs = new String[] { + "-args", input("A"), String.valueOf(ROWS), String.valueOf(COLUMN_NAMES.length), output("B") + }; + + Types.ValueType[] schema = Collections.nCopies( + COLUMN_NAMES.length, Types.ValueType.FP64).toArray(new Types.ValueType[0]); + FrameBlock inputFrame = new FrameBlock(schema); + inputFrame.setColumnNames(COLUMN_NAMES); + + double[][] data = getRandomMatrix(ROWS, schema.length, -10, 10, 0.8, 7); + TestUtils.initFrameData(inputFrame, data, schema, ROWS); + + FrameWriter writer = FrameWriterFactory.createFrameWriter( + FileFormat.CSV, new FileFormatPropertiesCSV(true, ",", false)); + writer.writeFrameToHDFS(inputFrame, input("A"), ROWS, schema.length); + + runTest(true, false, null, -1); + + FrameReader reader = FrameReaderFactory.createFrameReader( + FileFormat.CSV, new FileFormatPropertiesCSV(true, ",", false)); + FrameBlock outputFrame = reader.readFrameFromHDFS(output("B"), ROWS, schema.length); + Assert.assertArrayEquals("Column names were not preserved across CSV read/write.", + COLUMN_NAMES, outputFrame.getColumnNames()); + } + catch(Exception ex) { + throw new RuntimeException(ex); + } + finally { + rtplatform = platformOld; + DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; + } + } +} diff --git a/src/test/scripts/functions/frame/FrameCSVHeaderNames.dml b/src/test/scripts/functions/frame/FrameCSVHeaderNames.dml new file mode 100644 index 00000000000..35f9ead3267 --- /dev/null +++ b/src/test/scripts/functions/frame/FrameCSVHeaderNames.dml @@ -0,0 +1,23 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = read($1, rows=$2, cols=$3, data_type="frame", format="csv", header=TRUE); +write(X, $4, format="csv", header=TRUE);