From 00662c1df0fb643732b65011504dab59cfd59bc3 Mon Sep 17 00:00:00 2001 From: Saleem Malik Date: Tue, 21 Jul 2026 12:50:49 +0530 Subject: [PATCH 1/6] keep an escaped value that equals the null string --- .../org/apache/commons/csv/CSVParser.java | 4 +++- .../java/org/apache/commons/csv/Lexer.java | 3 +++ .../java/org/apache/commons/csv/Token.java | 4 ++++ .../org/apache/commons/csv/CSVParserTest.java | 16 ++++++++++++++++ .../apache/commons/csv/CSVPrinterTest.java | 19 +++++++++++++++---- 5 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/apache/commons/csv/CSVParser.java b/src/main/java/org/apache/commons/csv/CSVParser.java index 6d2a3b994..334bc73ad 100644 --- a/src/main/java/org/apache/commons/csv/CSVParser.java +++ b/src/main/java/org/apache/commons/csv/CSVParser.java @@ -819,8 +819,10 @@ private String handleNull(final String input) { final String nullString = format.getNullString(); final boolean strictQuoteMode = isStrictQuoteMode(); if (input.equals(nullString)) { + // An escaped value is the null string itself, not the null marker: printing null writes the null string + // verbatim, so "\\N" for nullString "\N" can only have come from a value that really is "\N". // nullString = NULL(String), distinguish between "NULL" and NULL in ALL_NON_NULL or NON_NUMERIC quote mode - return strictQuoteMode && isQuoted ? input : null; + return reusableToken.isEscaped || strictQuoteMode && isQuoted ? input : null; } // don't set nullString, distinguish between "" and ,, (absent values) in All_NON_NULL or NON_NUMERIC quote mode return strictQuoteMode && nullString == null && input.isEmpty() && !isQuoted ? null : input; diff --git a/src/main/java/org/apache/commons/csv/Lexer.java b/src/main/java/org/apache/commons/csv/Lexer.java index c78d1c7b1..12f1aa060 100644 --- a/src/main/java/org/apache/commons/csv/Lexer.java +++ b/src/main/java/org/apache/commons/csv/Lexer.java @@ -76,12 +76,15 @@ final class Lexer implements Closeable { private void appendNextEscapedCharacterToToken(final Token token) throws IOException { if (isEscapeDelimiter()) { token.content.append(delimiter); + token.isEscaped = true; } else { final int unescaped = readEscape(); if (unescaped == EOF) { // unexpected char after escape + // The escape character is kept verbatim, so nothing was translated. token.content.append((char) escape).append((char) reader.getLastChar()); } else { token.content.append((char) unescaped); + token.isEscaped = true; } } } diff --git a/src/main/java/org/apache/commons/csv/Token.java b/src/main/java/org/apache/commons/csv/Token.java index 42c4e3bdc..bde790a65 100644 --- a/src/main/java/org/apache/commons/csv/Token.java +++ b/src/main/java/org/apache/commons/csv/Token.java @@ -61,11 +61,15 @@ enum Type { boolean isQuoted; + /** True when an escape sequence in the input was translated while building {@link #content}. */ + boolean isEscaped; + void reset() { content.setLength(0); type = INVALID; isReady = false; isQuoted = false; + isEscaped = false; } /** diff --git a/src/test/java/org/apache/commons/csv/CSVParserTest.java b/src/test/java/org/apache/commons/csv/CSVParserTest.java index 5300a63a0..be8e8c10e 100644 --- a/src/test/java/org/apache/commons/csv/CSVParserTest.java +++ b/src/test/java/org/apache/commons/csv/CSVParserTest.java @@ -662,6 +662,22 @@ void testEndOfFileBehaviorExcel() throws Exception { } } + @Test + void testEscapedNullStringIsAValue() throws Exception { + // "\N" is the MySQL and PostgreSQL null marker, "\\N" is the value "\N", which is what the printer writes for it. + for (final CSVFormat format : new CSVFormat[] { CSVFormat.MYSQL, CSVFormat.POSTGRESQL_TEXT, CSVFormat.ORACLE }) { + final StringWriter writer = new StringWriter(); + try (CSVPrinter printer = new CSVPrinter(writer, format)) { + printer.printRecord("\\N", null); + } + try (CSVParser parser = CSVParser.parse(writer.toString(), format)) { + final CSVRecord record = parser.nextRecord(); + assertEquals("\\N", record.get(0), format.toString()); + assertNull(record.get(1), format.toString()); + } + } + } + @Test void testExcelFormat1() throws IOException { final String code = "value1,value2,value3,value4\r\na,b,c,d\r\n x,,,\r\n\r\n\"\"\"hello\"\"\",\" \"\"world\"\"\",\"abc\ndef\",\r\n"; diff --git a/src/test/java/org/apache/commons/csv/CSVPrinterTest.java b/src/test/java/org/apache/commons/csv/CSVPrinterTest.java index 1c25821ea..0d25abf61 100644 --- a/src/test/java/org/apache/commons/csv/CSVPrinterTest.java +++ b/src/test/java/org/apache/commons/csv/CSVPrinterTest.java @@ -158,13 +158,17 @@ private void doRandom(final CSVFormat format, final int iter) throws Exception { } /** - * Converts an input CSV array into expected output values, including NULLs. NULL strings are converted to null values because the parser will convert - * these strings to null. + * Converts an input CSV array into expected output values, including NULLs. A value equal to the null string is converted to null only when the printer + * writes that value unchanged; when the printer escapes or quotes it, the parser reads it back as the value it is. */ - private T[] expectNulls(final T[] original, final CSVFormat csvFormat) { + private T[] expectNulls(final T[] original, final CSVFormat csvFormat) throws IOException { final T[] fixed = original.clone(); + final String nullString = csvFormat.getNullString(); + if (nullString == null || !printsVerbatim(csvFormat, nullString)) { + return fixed; + } for (int i = 0; i < fixed.length; i++) { - if (Objects.equals(csvFormat.getNullString(), fixed[i])) { + if (Objects.equals(nullString, fixed[i])) { fixed[i] = null; } } @@ -196,6 +200,13 @@ private String printNullRecord(final CSVFormat format) throws IOException { return sw.toString(); } + /** Tests whether the format prints the given value unchanged, in which case the parser cannot tell it from the null string. */ + private boolean printsVerbatim(final CSVFormat csvFormat, final String value) throws IOException { + final StringBuilder sb = new StringBuilder(); + csvFormat.print(value, sb, true); + return value.contentEquals(sb); + } + private CSVPrinter printWithHeaderComments(final StringWriter sw, final Date now, final CSVFormat baseFormat) throws IOException { // Use withHeaderComments first to test CSV-145 // @formatter:off From 803db6dbc80415c833f702af13cd74bcc88d9a5e Mon Sep 17 00:00:00 2001 From: Saleem Malik Date: Wed, 22 Jul 2026 10:59:17 +0530 Subject: [PATCH 2/6] Use a parameterized test for the escaped null string case --- .../org/apache/commons/csv/CSVParserTest.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/test/java/org/apache/commons/csv/CSVParserTest.java b/src/test/java/org/apache/commons/csv/CSVParserTest.java index be8e8c10e..a559c6e72 100644 --- a/src/test/java/org/apache/commons/csv/CSVParserTest.java +++ b/src/test/java/org/apache/commons/csv/CSVParserTest.java @@ -662,19 +662,19 @@ void testEndOfFileBehaviorExcel() throws Exception { } } - @Test - void testEscapedNullStringIsAValue() throws Exception { + @ParameterizedTest + @EnumSource(value = CSVFormat.Predefined.class, names = { "MySQL", "PostgreSQLCsv", "PostgreSQLText", "Oracle" }) + void testEscapedNullStringIsAValue(final CSVFormat.Predefined predefined) throws Exception { // "\N" is the MySQL and PostgreSQL null marker, "\\N" is the value "\N", which is what the printer writes for it. - for (final CSVFormat format : new CSVFormat[] { CSVFormat.MYSQL, CSVFormat.POSTGRESQL_TEXT, CSVFormat.ORACLE }) { - final StringWriter writer = new StringWriter(); - try (CSVPrinter printer = new CSVPrinter(writer, format)) { - printer.printRecord("\\N", null); - } - try (CSVParser parser = CSVParser.parse(writer.toString(), format)) { - final CSVRecord record = parser.nextRecord(); - assertEquals("\\N", record.get(0), format.toString()); - assertNull(record.get(1), format.toString()); - } + final CSVFormat format = predefined.getFormat(); + final StringWriter writer = new StringWriter(); + try (CSVPrinter printer = new CSVPrinter(writer, format)) { + printer.printRecord("\\N", null); + } + try (CSVParser parser = CSVParser.parse(writer.toString(), format)) { + final CSVRecord record = parser.nextRecord(); + assertEquals("\\N", record.get(0)); + assertNull(record.get(1)); } } From 06f8b0f3e1b76f70a237548d55ec94460af4dc62 Mon Sep 17 00:00:00 2001 From: Gary Gregory Date: Thu, 23 Jul 2026 07:51:27 -0400 Subject: [PATCH 3/6] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/test/java/org/apache/commons/csv/CSVParserTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/apache/commons/csv/CSVParserTest.java b/src/test/java/org/apache/commons/csv/CSVParserTest.java index a559c6e72..b2a2fb3e1 100644 --- a/src/test/java/org/apache/commons/csv/CSVParserTest.java +++ b/src/test/java/org/apache/commons/csv/CSVParserTest.java @@ -665,7 +665,7 @@ void testEndOfFileBehaviorExcel() throws Exception { @ParameterizedTest @EnumSource(value = CSVFormat.Predefined.class, names = { "MySQL", "PostgreSQLCsv", "PostgreSQLText", "Oracle" }) void testEscapedNullStringIsAValue(final CSVFormat.Predefined predefined) throws Exception { - // "\N" is the MySQL and PostgreSQL null marker, "\\N" is the value "\N", which is what the printer writes for it. + // For formats whose null string is "\\N" (e.g., MySQL, PostgreSQL Text, Oracle), a literal value "\\N" must be written as "\\\\N" so it is not read back as null. final CSVFormat format = predefined.getFormat(); final StringWriter writer = new StringWriter(); try (CSVPrinter printer = new CSVPrinter(writer, format)) { From 752e8296052babe5648eae36ca8abd9be48d9f67 Mon Sep 17 00:00:00 2001 From: Gary Gregory Date: Thu, 23 Jul 2026 07:56:56 -0400 Subject: [PATCH 4/6] Copilot doesn't know about our Checkstyle rules Updated comment for clarity on handling null strings in CSV formats. --- src/test/java/org/apache/commons/csv/CSVParserTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/apache/commons/csv/CSVParserTest.java b/src/test/java/org/apache/commons/csv/CSVParserTest.java index b2a2fb3e1..9b9337c71 100644 --- a/src/test/java/org/apache/commons/csv/CSVParserTest.java +++ b/src/test/java/org/apache/commons/csv/CSVParserTest.java @@ -665,7 +665,8 @@ void testEndOfFileBehaviorExcel() throws Exception { @ParameterizedTest @EnumSource(value = CSVFormat.Predefined.class, names = { "MySQL", "PostgreSQLCsv", "PostgreSQLText", "Oracle" }) void testEscapedNullStringIsAValue(final CSVFormat.Predefined predefined) throws Exception { - // For formats whose null string is "\\N" (e.g., MySQL, PostgreSQL Text, Oracle), a literal value "\\N" must be written as "\\\\N" so it is not read back as null. + // For formats whose null string is "\\N" (e.g., MySQL, PostgreSQL Text, Oracle), + // a literal value "\\N" must be written as "\\\\N" so it is not read back as null. final CSVFormat format = predefined.getFormat(); final StringWriter writer = new StringWriter(); try (CSVPrinter printer = new CSVPrinter(writer, format)) { From 1c9b194f1bc148465d378236371a0e08dc55d78a Mon Sep 17 00:00:00 2001 From: Gary Gregory Date: Thu, 23 Jul 2026 08:07:45 -0400 Subject: [PATCH 5/6] Clarify comment on escaped null string in tests Updated comment for clarity on null string handling. --- src/test/java/org/apache/commons/csv/CSVParserTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/apache/commons/csv/CSVParserTest.java b/src/test/java/org/apache/commons/csv/CSVParserTest.java index 9b9337c71..0a3391874 100644 --- a/src/test/java/org/apache/commons/csv/CSVParserTest.java +++ b/src/test/java/org/apache/commons/csv/CSVParserTest.java @@ -665,7 +665,7 @@ void testEndOfFileBehaviorExcel() throws Exception { @ParameterizedTest @EnumSource(value = CSVFormat.Predefined.class, names = { "MySQL", "PostgreSQLCsv", "PostgreSQLText", "Oracle" }) void testEscapedNullStringIsAValue(final CSVFormat.Predefined predefined) throws Exception { - // For formats whose null string is "\\N" (e.g., MySQL, PostgreSQL Text, Oracle), + // For formats whose null string is "\\N" (e.g., MySQL, PostgreSQL Text, Oracle), // a literal value "\\N" must be written as "\\\\N" so it is not read back as null. final CSVFormat format = predefined.getFormat(); final StringWriter writer = new StringWriter(); From 885f9b7d174b1def0cdbfbd701c0194198dbba56 Mon Sep 17 00:00:00 2001 From: Saleem Malik Date: Wed, 29 Jul 2026 11:13:51 +0530 Subject: [PATCH 6/6] Name the escaped-null-string test value and refine comments Signed-off-by: Saleem Malik --- src/main/java/org/apache/commons/csv/CSVParser.java | 7 ++++--- src/test/java/org/apache/commons/csv/CSVParserTest.java | 9 +++++---- src/test/java/org/apache/commons/csv/CSVPrinterTest.java | 5 +++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/apache/commons/csv/CSVParser.java b/src/main/java/org/apache/commons/csv/CSVParser.java index 334bc73ad..4275b937a 100644 --- a/src/main/java/org/apache/commons/csv/CSVParser.java +++ b/src/main/java/org/apache/commons/csv/CSVParser.java @@ -819,10 +819,11 @@ private String handleNull(final String input) { final String nullString = format.getNullString(); final boolean strictQuoteMode = isStrictQuoteMode(); if (input.equals(nullString)) { - // An escaped value is the null string itself, not the null marker: printing null writes the null string - // verbatim, so "\\N" for nullString "\N" can only have come from a value that really is "\N". + // A token that needed an escape translation cannot be the null marker: printing null emits the null + // string without escaping it (it may be quoted, but never escaped), so an escaped "\N" for nullString + // "\N" can only have come from a field whose value really is "\N". // nullString = NULL(String), distinguish between "NULL" and NULL in ALL_NON_NULL or NON_NUMERIC quote mode - return reusableToken.isEscaped || strictQuoteMode && isQuoted ? input : null; + return reusableToken.isEscaped || (strictQuoteMode && isQuoted) ? input : null; } // don't set nullString, distinguish between "" and ,, (absent values) in All_NON_NULL or NON_NUMERIC quote mode return strictQuoteMode && nullString == null && input.isEmpty() && !isQuoted ? null : input; diff --git a/src/test/java/org/apache/commons/csv/CSVParserTest.java b/src/test/java/org/apache/commons/csv/CSVParserTest.java index 0a3391874..50a6ae03d 100644 --- a/src/test/java/org/apache/commons/csv/CSVParserTest.java +++ b/src/test/java/org/apache/commons/csv/CSVParserTest.java @@ -665,16 +665,17 @@ void testEndOfFileBehaviorExcel() throws Exception { @ParameterizedTest @EnumSource(value = CSVFormat.Predefined.class, names = { "MySQL", "PostgreSQLCsv", "PostgreSQLText", "Oracle" }) void testEscapedNullStringIsAValue(final CSVFormat.Predefined predefined) throws Exception { - // For formats whose null string is "\\N" (e.g., MySQL, PostgreSQL Text, Oracle), - // a literal value "\\N" must be written as "\\\\N" so it is not read back as null. + // "\N" is the null string for MySQL, PostgreSQL Text and Oracle; PostgreSQL CSV uses an empty null + // string. In every case a field whose value equals "\N" must round trip as that value, not as null. + final String valueEqualToNullString = "\\N"; final CSVFormat format = predefined.getFormat(); final StringWriter writer = new StringWriter(); try (CSVPrinter printer = new CSVPrinter(writer, format)) { - printer.printRecord("\\N", null); + printer.printRecord(valueEqualToNullString, null); } try (CSVParser parser = CSVParser.parse(writer.toString(), format)) { final CSVRecord record = parser.nextRecord(); - assertEquals("\\N", record.get(0)); + assertEquals(valueEqualToNullString, record.get(0)); assertNull(record.get(1)); } } diff --git a/src/test/java/org/apache/commons/csv/CSVPrinterTest.java b/src/test/java/org/apache/commons/csv/CSVPrinterTest.java index 0d25abf61..f1f910bb5 100644 --- a/src/test/java/org/apache/commons/csv/CSVPrinterTest.java +++ b/src/test/java/org/apache/commons/csv/CSVPrinterTest.java @@ -158,8 +158,9 @@ private void doRandom(final CSVFormat format, final int iter) throws Exception { } /** - * Converts an input CSV array into expected output values, including NULLs. A value equal to the null string is converted to null only when the printer - * writes that value unchanged; when the printer escapes or quotes it, the parser reads it back as the value it is. + * Converts an input CSV array into expected output values, including NULLs. A value equal to the null string is expected back as null only when the + * printer writes it verbatim (neither escaped nor quoted); once it is escaped or quoted the parser can tell it apart from a real null and reads it back + * as the value it is. */ private T[] expectNulls(final T[] original, final CSVFormat csvFormat) throws IOException { final T[] fixed = original.clone();