From a598e61e8eba7a0a79437ec250c004569ae75385 Mon Sep 17 00:00:00 2001 From: Optio Agent Date: Fri, 24 Jul 2026 00:17:12 +0000 Subject: [PATCH] Fix ADIF import silently dropping every QSO from a headerless log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two ADIF importers (LogFileImport, used by the web-logger HTTP upload, and ImportSharedLogs, used by the Android share/VIEW intent) both derived the record body by splitting the file on and returning "" when no marker was found. Per the ADI file format a Header is optional: it is present only when the file does NOT begin with '<'. A file that begins with '<' is headerless and consists entirely of data records. So a valid headerless ADIF log — a common export shape, and what a WSJT-X wsjtx_log.adi whose header was written with the v2.2.2 bug looks like — imported as zero QSOs, with no error surfaced to the user (getErrorCount() == 0). Silent data loss on import. Root cause fix: a shared, spec-correct AdifFormat.stripHeader() returns the whole content for a headerless file (first non-blank char, after an optional UTF-8 BOM, is '<'), the text after the first for a headed file, and "" only when a header is present but unterminated or the input is null. Both getLogBody() methods now delegate to it. Tests: new AdifFormat.stripHeader cases (headed/headerless/BOM/whitespace/ case-insensitive /first--only/unterminated/null) and an end-to-end LogFileImport headerless-import test (replacing the stale test that pinned the old drop-everything behaviour). All pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../java/com/k1af/ft8af/log/AdifFormat.java | 59 +++++++++++++++++++ .../com/k1af/ft8af/log/ImportSharedLogs.java | 10 ++-- .../com/k1af/ft8af/log/LogFileImport.java | 12 ++-- .../com/k1af/ft8af/log/AdifFormatTest.java | 57 ++++++++++++++++++ .../com/k1af/ft8af/log/LogFileImportTest.java | 29 +++++++-- 5 files changed, 150 insertions(+), 17 deletions(-) diff --git a/ft8af/app/src/main/java/com/k1af/ft8af/log/AdifFormat.java b/ft8af/app/src/main/java/com/k1af/ft8af/log/AdifFormat.java index b7bdf5ada..4659f7ce2 100644 --- a/ft8af/app/src/main/java/com/k1af/ft8af/log/AdifFormat.java +++ b/ft8af/app/src/main/java/com/k1af/ft8af/log/AdifFormat.java @@ -2,6 +2,8 @@ import java.nio.charset.StandardCharsets; import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Small helpers for writing ADIF fields safely. @@ -160,4 +162,61 @@ public static String sliceByUtf8Length(String raw, int byteLen) { } return raw.substring(0, i); } + + /** The ADIF header terminator {@code }, matched case-insensitively. */ + private static final Pattern EOH = Pattern.compile("<[Ee][Oo][Hh]>"); + + /** + * The data-record body of a full {@code .adi} file — everything after the optional + * header — ready to be split on {@code }. + * + *

Per the ADI file format a Header is optional: it is present only when the + * file does not begin with {@code '<'}. When present it is arbitrary text + * terminated by an {@code } tag, and the data records follow that tag. A file that + * begins with {@code '<'} is headerless and consists entirely of data records. + * + *

Both importers previously split on {@code } and returned {@code ""} whenever no + * marker was found. That silently dropped every QSO of a valid headerless ADIF + * log on import (0 records, and 0 errors surfaced to the user) — a common export shape, + * and exactly what a WSJT-X {@code wsjtx_log.adi} whose header was written with the + * v2.2.2 {@code } bug looks like once opened. This helper returns: + *

    + *
  • the whole content when the file is headerless (first non-blank char is {@code '<'});
    • + *
  • the text after the first {@code } when a header is present and terminated;
    • + *
  • {@code ""} only when a header is present but unterminated (no {@code }) or the + * input is null — there is genuinely no parseable record body.
    • + *
+ * + * @param fileContext the full {@code .adi} file text (null → {@code ""}) + * @return the record-body text to split on {@code } + */ + public static String stripHeader(String fileContext) { + if (fileContext == null) { + return ""; + } + if (beginsWithField(fileContext)) { + // Headerless file: the whole content is data records. + return fileContext; + } + // Header present (does not begin with '<'); records start after its terminator. + Matcher m = EOH.matcher(fileContext); + return m.find() ? fileContext.substring(m.end()) : ""; + } + + /** + * True when the first non-whitespace character of {@code content} (after an optional + * leading UTF-8 BOM) is {@code '<'} — i.e. the file opens with an ADIF field and so, + * per the spec, carries no header. + */ + private static boolean beginsWithField(String content) { + int i = 0; + int n = content.length(); + if (n > 0 && content.charAt(0) == '\uFEFF') { // skip a UTF-8 BOM if present + i = 1; + } + while (i < n && Character.isWhitespace(content.charAt(i))) { + i++; + } + return i < n && content.charAt(i) == '<'; + } } diff --git a/ft8af/app/src/main/java/com/k1af/ft8af/log/ImportSharedLogs.java b/ft8af/app/src/main/java/com/k1af/ft8af/log/ImportSharedLogs.java index 0148c3ebb..b666b146a 100644 --- a/ft8af/app/src/main/java/com/k1af/ft8af/log/ImportSharedLogs.java +++ b/ft8af/app/src/main/java/com/k1af/ft8af/log/ImportSharedLogs.java @@ -65,12 +65,10 @@ private boolean loadData(OnShareLogEvents onShareLogEvents) { } public String getLogBody() { - String[] temp = fileContext.split("[<][Ee][Oo][Hh][>]"); - if (temp.length > 1) { - return temp[temp.length - 1]; - } else { - return ""; - } + // ADIF's Header is optional: a file that begins with '<' is headerless and is + // entirely records. AdifFormat.stripHeader handles both cases; the previous + // "return '' when no " dropped every QSO of a valid headerless log. + return AdifFormat.stripHeader(fileContext); } /** diff --git a/ft8af/app/src/main/java/com/k1af/ft8af/log/LogFileImport.java b/ft8af/app/src/main/java/com/k1af/ft8af/log/LogFileImport.java index dce1649bc..5afdcfef9 100644 --- a/ft8af/app/src/main/java/com/k1af/ft8af/log/LogFileImport.java +++ b/ft8af/app/src/main/java/com/k1af/ft8af/log/LogFileImport.java @@ -17,7 +17,7 @@ * Log file import. * The constructor requires a log file name; the file here is posted from NanoHTTPd's session. * getFileContext returns the entire file content. - * getLogBody returns all raw record content from the log file, i.e., all data after the <eoh> tag. + * getLogBody returns all raw record content from the log file: the data after the <eoh> tag, or the whole file when it is a headerless ADIF log (see AdifFormat.stripHeader). * getLogRecords returns a list of all parsed records stored as HashMaps, where the Key is the field name (uppercase) and the value is the actual value. * * @author BGY70Z @@ -92,12 +92,10 @@ public String getFileContext() { } public String getLogBody() { - String[] temp = fileContext.split("[<][Ee][Oo][Hh][>]"); - if (temp.length > 1) { - return temp[temp.length - 1]; - } else { - return ""; - } + // ADIF's Header is optional: a file that begins with '<' is headerless and is + // entirely records. AdifFormat.stripHeader handles both cases; the previous + // "return '' when no " dropped every QSO of a valid headerless log. + return AdifFormat.stripHeader(fileContext); } /** diff --git a/ft8af/app/src/test/java/com/k1af/ft8af/log/AdifFormatTest.java b/ft8af/app/src/test/java/com/k1af/ft8af/log/AdifFormatTest.java index c75f05beb..cb4c4f62a 100644 --- a/ft8af/app/src/test/java/com/k1af/ft8af/log/AdifFormatTest.java +++ b/ft8af/app/src/test/java/com/k1af/ft8af/log/AdifFormatTest.java @@ -222,4 +222,61 @@ public void sliceByUtf8Length_roundTripsUtf8LengthForAstralChars() { assertThat(AdifFormat.sliceByUtf8Length(emoji, AdifFormat.utf8Length(emoji))) .isEqualTo(emoji); } + + // ---- stripHeader: the ADIF header/record split (optional-header handling) ---- + + @Test + public void stripHeader_headedFile_returnsBodyAfterEoh() { + // A header (does not begin with '<') is terminated by ; records follow it. + String body = AdifFormat.stripHeader( + "Generated by FT8AF3.1.0K1ABC"); + assertThat(body).isEqualTo("K1ABC"); + assertThat(body).doesNotContain("adif_ver"); + } + + @Test + public void stripHeader_headlessFile_returnsWholeContent() { + // Regression: the file begins with '<' so per the ADI spec it has NO header and is + // entirely records. The old importers split on , found none, and returned "" \u2014 + // dropping every QSO. The whole content must survive. + String content = "K1ABCFN42\nW1AW"; + assertThat(AdifFormat.stripHeader(content)).isEqualTo(content); + } + + @Test + public void stripHeader_headlessFile_skipsLeadingWhitespaceAndBom() { + // A leading BOM or newline before the first field must not be mistaken for a header. + String content = "K1ABC"; + assertThat(AdifFormat.stripHeader("\n " + content)).isEqualTo("\n " + content); + assertThat(AdifFormat.stripHeader("\uFEFF" + content)).isEqualTo("\uFEFF" + content); + } + + @Test + public void stripHeader_eohMarkerIsCaseInsensitive() { + assertThat(AdifFormat.stripHeader("hdrK1ABC")) + .isEqualTo("K1ABC"); + assertThat(AdifFormat.stripHeader("hdrK1ABC")) + .isEqualTo("K1ABC"); + } + + @Test + public void stripHeader_splitsAtFirstEohOnly() { + // Only the first terminates the header; a later literal one (however unlikely) + // stays in the body rather than truncating it. + assertThat(AdifFormat.stripHeader("hdrab")) + .isEqualTo("ab"); + } + + @Test + public void stripHeader_headPresentButNoEoh_returnsEmpty() { + // Header present (does not begin with '<') but unterminated: no parseable body. + assertThat(AdifFormat.stripHeader("header only, no eoh marker")).isEmpty(); + } + + @Test + public void stripHeader_nullOrEmpty_returnsEmpty() { + assertThat(AdifFormat.stripHeader(null)).isEmpty(); + assertThat(AdifFormat.stripHeader("")).isEmpty(); + assertThat(AdifFormat.stripHeader(" ")).isEmpty(); + } } diff --git a/ft8af/app/src/test/java/com/k1af/ft8af/log/LogFileImportTest.java b/ft8af/app/src/test/java/com/k1af/ft8af/log/LogFileImportTest.java index 03b060b4a..28ac72b79 100644 --- a/ft8af/app/src/test/java/com/k1af/ft8af/log/LogFileImportTest.java +++ b/ft8af/app/src/test/java/com/k1af/ft8af/log/LogFileImportTest.java @@ -193,13 +193,34 @@ public void malformedAdif_errorLinesHtmlEscapesAngleBrackets() throws IOExceptio } @Test - public void getLogBody_returnsEmptyWhenNoEoh() throws IOException { - File f = tmp.newFile("noeoh.adi"); + public void headerlessAdif_importsRecords() throws IOException { + // Per the ADI spec a file that begins with '<' has no header and is entirely + // data records (WSJT-X's v2.2.2 -header bug, and many loggers/hand exports, + // produce headerless .adi files). The old code split on , found none, and + // returned "" — silently dropping every QSO with no error surfaced. + File f = tmp.newFile("headerless.adi"); try (FileOutputStream out = new FileOutputStream(f)) { - out.write("K1ABC".getBytes(StandardCharsets.UTF_8)); + out.write("K1ABCFN42\nW1AW" + .getBytes(StandardCharsets.UTF_8)); + } + LogFileImport imp = new LogFileImport(task, f.getAbsolutePath()); + assertThat(imp.getLogBody()).contains("K1ABC"); + ArrayList> records = imp.getLogRecords(); + assertThat(records).hasSize(2); + assertThat(records.get(0).get("CALL")).isEqualTo("K1ABC"); + assertThat(records.get(0).get("GRIDSQUARE")).isEqualTo("FN42"); + assertThat(records.get(1).get("CALL")).isEqualTo("W1AW"); + } + + @Test + public void headerPresentButUnterminated_yieldsNoRecords() throws IOException { + // A file that does NOT begin with '<' has a header; without an terminator + // it is malformed and carries no parseable record body. + File f = tmp.newFile("badheader.adi"); + try (FileOutputStream out = new FileOutputStream(f)) { + out.write("header only, no eoh marker".getBytes(StandardCharsets.UTF_8)); } LogFileImport imp = new LogFileImport(task, f.getAbsolutePath()); - // No marker → split produces a single element → body is "". assertThat(imp.getLogBody()).isEmpty(); assertThat(imp.getLogRecords()).isEmpty(); }