feat(csv): probe for a dialect instead of checking for one

src/odr/internal/csv/csv_file.cpp

@@ -1,13 +1,20 @@
 #include <odr/internal/csv/csv_file.hpp>
 
+#include <odr/exceptions.hpp>
+
 #include <odr/internal/csv/csv_util.hpp>
 
+#include <utility>
+
 namespace odr::internal::csv {
 
 CsvFile::CsvFile(std::shared_ptr<text::TextFile> file)
     : m_file{std::move(file)} {
-  // TODO use text file?
-  check_csv_file(*m_file->file()->stream());
+  const Probe probe = csv::probe(*m_file->file(), m_file->encoding());
+  if (!probe.is_csv) {
+    throw NoCsvFile();
+  }
+  m_dialect = probe.dialect;
 }
 
 std::shared_ptr<abstract::File> CsvFile::file() const noexcept {
@@ -31,4 +38,6 @@ bool CsvFile::is_decodable() const noexcept { return false; }
 
 TextEncoding CsvFile::encoding() const noexcept { return m_file->encoding(); }
 
+Dialect CsvFile::dialect() const noexcept { return m_dialect; }
+
 } // namespace odr::internal::csv

src/odr/internal/csv/csv_file.hpp

@@ -2,6 +2,7 @@
 
 #include <odr/file.hpp>
 
+#include <odr/internal/csv/csv_util.hpp>
 #include <odr/internal/text/text_file.hpp>
 
 #include <memory>
@@ -22,8 +23,12 @@ class CsvFile final : public abstract::TextFile {
 
   [[nodiscard]] TextEncoding encoding() const noexcept override;
 
+  /// The dialect detection resolved.
+  [[nodiscard]] Dialect dialect() const noexcept;
+
 private:
   std::shared_ptr<text::TextFile> m_file;
+  Dialect m_dialect;
 };
 
 } // namespace odr::internal::csv

src/odr/internal/csv/csv_util.cpp

@@ -1,23 +1,99 @@
 #include <odr/internal/csv/csv_util.hpp>
 
-#include <cstddef>
+#include <odr/odr.hpp>
+
+#include <odr/internal/abstract/file.hpp>
+#include <odr/internal/encoding/detect.hpp>
+#include <odr/internal/encoding/transcode.hpp>
+
+#include <algorithm>
+#include <array>
 #include <istream>
+#include <map>
+#include <memory>
 #include <optional>
-#include <stdexcept>
-#include <string>
 #include <utility>
-#include <vector>
 
 namespace odr::internal {
 
 namespace {
 
-constexpr char csv_delimiter = ',';
-constexpr char csv_quote = '"';
+using csv::Dialect;
+using csv::Probe;
+using csv::RecordReader;
+
+/// The separators worth trying, in the order they win ties.
+constexpr std::array candidate_separators{',', ';', '\t', '|'};
+
+/// Share of records that must agree on a field count — not all of them, a
+/// stray ragged row is normal and the parser tolerates it.
+constexpr double required_agreement = 0.9;
+
+/// Excel opens a file it exported for a `;` locale with `sep=;`.
+std::optional<char> separator_directive(const std::string_view text) {
+  constexpr std::string_view prefix = "sep=";
+  if (!text.starts_with(prefix) || text.size() < prefix.size() + 2) {
+    return std::nullopt;
+  }
+  const char separator = text[prefix.size()];
+  const char after = text[prefix.size() + 1];
+  if (after != '\n' && after != '\r') {
+    return std::nullopt;
+  }
+  return separator;
+}
+
+/// The field counts of every record @p dialect finds in @p text.
+struct Scan final {
+  std::vector<std::uint32_t> counts;
+  bool unterminated{false};
+};
+
+Scan scan(const std::string_view text, const Dialect dialect) {
+  Scan result;
+  RecordReader reader(text, dialect);
+  std::vector<std::string> fields;
+  while (reader.read(fields)) {
+    result.counts.push_back(static_cast<std::uint32_t>(fields.size()));
+  }
+  result.unterminated = reader.unterminated();
+  return result;
+}
+
+/// How well @p dialect explains @p text: the field count most records carry,
+/// and the share of records carrying it.
+struct Score final {
+  std::uint32_t columns{0};
+  double agreement{0.0};
+};
+
+Score score(Scan scan, const bool complete) {
+  // a sample cut mid-record says nothing about the record it cut
+  if (!complete && !scan.counts.empty()) {
+    scan.counts.pop_back();
+  }
+  if (scan.counts.empty()) {
+    return {};
+  }
+
+  std::map<std::uint32_t, std::size_t> histogram;
+  for (const std::uint32_t count : scan.counts) {
+    ++histogram[count];
+  }
+  const auto modal = std::ranges::max_element(
+      histogram, {}, [](const auto &entry) { return entry.second; });
+
+  return {modal->first, static_cast<double>(modal->second) /
+                            static_cast<double>(scan.counts.size())};
+}
 
 } // namespace
 
-bool csv::read_record(std::istream &in, std::vector<std::string> &fields) {
+csv::RecordReader::RecordReader(const std::string_view text,
+                                const Dialect dialect) noexcept
+    : m_text{text}, m_dialect{dialect} {}
+
+bool csv::RecordReader::read(std::vector<std::string> &fields) {
   fields.clear();
   std::string field;
   bool quoted = false;
@@ -28,51 +104,49 @@ bool csv::read_record(std::istream &in, std::vector<std::string> &fields) {
     field.clear();
   };
 
-  char c = 0;
-  while (in.get(c)) {
+  for (; m_position < m_text.size(); ++m_position) {
+    const char c = m_text[m_position];
+
     if (quoted) {
-      if (c != csv_quote) {
+      if (c != m_dialect.quote) {
         field.push_back(c);
         continue;
       }
       // a doubled quote is an escaped one and stays inside the field
-      if (in.peek() == csv_quote) {
-        in.get(c);
-        field.push_back(csv_quote);
+      if (m_position + 1 < m_text.size() &&
+          m_text[m_position + 1] == m_dialect.quote) {
+        ++m_position;
+        field.push_back(m_dialect.quote);
         continue;
       }
       quoted = false;
       continue;
     }
 
-    switch (c) {
-    case csv_quote:
+    if (c == m_dialect.quote) {
       quoted = true;
       started = true;
-      break;
-    case csv_delimiter:
+    } else if (c == m_dialect.separator) {
       end_field();
       started = true;
-      break;
-    case '\r':
-      break; // CRLF, and a lone CR
-    case '\n':
+    } else if (c == '\r') {
+      // CRLF, and a lone CR
+    } else if (c == '\n') {
       if (!started) {
-        break; // empty line
+        continue; // empty line
       }
       end_field();
+      ++m_position;
       return true;
-    default:
+    } else {
       field.push_back(c);
       started = true;
-      break;
     }
   }
 
-  // reaching EOF inside a quoted field means the quote was never closed; the
-  // partial field is not a record
+  // the caller decides what an unterminated field means
   if (quoted) {
-    throw std::runtime_error("csv quoted field is not terminated");
+    m_unterminated = true;
   }
 
   if (!started) {
@@ -82,26 +156,65 @@ bool csv::read_record(std::istream &in, std::vector<std::string> &fields) {
   return true;
 }
 
-void csv::check_csv_file(std::istream &in) {
-  std::optional<std::size_t> columns;
+bool csv::RecordReader::unterminated() const noexcept { return m_unterminated; }
 
-  std::vector<std::string> fields;
-  while (read_record(in, fields)) {
-    if (!columns.has_value()) {
-      columns = fields.size();
+csv::Probe csv::probe(const std::string_view text, const bool complete,
+                      const char quote) {
+  Probe result;
+  result.dialect.quote = quote;
+
+  if (const std::optional<char> declared = separator_directive(text);
+      declared.has_value()) {
+    result.dialect.separator = *declared;
+    result.separator_directive = true;
+
+    const std::size_t line_end = text.find_first_of("\r\n");
+    const std::size_t body = text.find_first_not_of("\r\n", line_end);
+    const Score scored =
+        score(scan(body == std::string_view::npos ? "" : text.substr(body),
+                   result.dialect),
+              complete);
+    result.columns = scored.columns;
+    result.is_csv = scored.columns >= 1;
+    return result;
+  }
+
+  Score best;
+  for (const char separator : candidate_separators) {
+    const Dialect dialect{.separator = separator, .quote = quote};
+    const Scan scanned = scan(text, dialect);
+
+    // a dangling quote in a file we have all of is good evidence of not-csv
+    if (complete && scanned.unterminated) {
+      continue;
+    }
+
+    const Score scored = score(scanned, complete);
+    if (scored.columns < 2) {
       continue;
     }
-    if (fields.size() != *columns) {
-      throw std::runtime_error("csv row has " + std::to_string(fields.size()) +
-                               " fields, expected " + std::to_string(*columns));
+    if (scored.agreement > best.agreement ||
+        (scored.agreement == best.agreement && scored.columns > best.columns)) {
+      best = scored;
+      result.dialect = dialect;
     }
   }
 
-  // one column is every line of prose ever written, so it is no evidence of a
-  // csv; the caller is probing an otherwise unrecognised text file
-  if (!columns.has_value() || *columns <= 1) {
-    throw std::runtime_error("no csv file");
+  result.columns = best.columns;
+  result.is_csv = best.columns >= 2 && best.agreement >= required_agreement;
+  return result;
+}
+
+csv::Probe csv::probe(const abstract::File &file, const TextEncoding encoding,
+                      const char quote) {
+  if (!text_encoding_is_decodable(encoding)) {
+    return {};
   }
+
+  const std::unique_ptr<std::istream> in = file.stream();
+  const std::string bytes = encoding::read_probe(*in);
+  const bool complete = bytes.size() < encoding::default_probe_size;
+  return probe(encoding::to_utf8(bytes, encoding), complete, quote);
 }
 
 } // namespace odr::internal

src/odr/internal/csv/csv_util.hpp

@@ -1,22 +1,73 @@
 #pragma once
 
-#include <iosfwd>
+#include <odr/file.hpp>
+
+#include <cstddef>
+#include <cstdint>
 #include <string>
+#include <string_view>
 #include <vector>
 
+namespace odr::internal::abstract {
+class File;
+}
+
 namespace odr::internal::csv {
 
-/// Reads the next record into @p fields; `false` once the input is exhausted.
+/// The lexical shape of a csv file.
+struct Dialect final {
+  char separator{','};
+  char quote{'"'};
+};
+
+/// Reads RFC 4180 records out of already-decoded UTF-8 text.
 ///
-/// RFC 4180: `,` separates, `"` quotes, `""` is a literal quote inside a quoted
-/// field, and a quoted field may span lines. An empty line yields no record,
-/// which is what keeps a trailing newline from reading as a one-field row.
+/// Rejects nothing: text ending inside a quoted field still yields its record
+/// and sets @ref unterminated. Refusing a file is detection's job.
+class RecordReader final {
+public:
+  RecordReader(std::string_view text, Dialect dialect) noexcept;
+
+  /// Reads the next record into @p fields; `false` once the text is exhausted.
+  /// An empty line yields no record, so a trailing newline is not a row.
+  bool read(std::vector<std::string> &fields);
+
+  /// Whether the text ran out inside a quoted field.
+  [[nodiscard]] bool unterminated() const noexcept;
+
+private:
+  std::string_view m_text;
+  Dialect m_dialect;
+  std::size_t m_position{0};
+  bool m_unterminated{false};
+};
+
+/// What a probe made of a file's opening bytes.
+struct Probe final {
+  Dialect dialect;
+  /// The field count most records carry.
+  std::uint32_t columns{0};
+  /// Whether the opening line is Excel's `sep=` directive, which names the
+  /// separator and is not data.
+  bool separator_directive{false};
+  /// Whether this looks like a csv at all — see @ref probe.
+  bool is_csv{false};
+};
+
+/// Scores @p text — a file's opening bytes decoded to UTF-8 — as a csv and
+/// resolves its dialect. @p complete says whether that is the whole file, since
+/// a sample cut mid-record says nothing about the record it cut.
 ///
-/// @throws std::runtime_error if the input ends inside a quoted field.
-bool read_record(std::istream &in, std::vector<std::string> &fields);
+/// The verdict is a detection heuristic, not a validity rule: two columns
+/// minimum, because one column is every line of prose ever written, and no
+/// dangling quote in a complete file. A declared csv goes straight to
+/// @ref RecordReader and is never asked to pass this.
+[[nodiscard]] Probe probe(std::string_view text, bool complete,
+                          char quote = '"');
 
-/// Throws unless @p in parses as a csv whose records all carry the same number
-/// of fields, and more than one of them.
-void check_csv_file(std::istream &in);
+/// Reads @p file's opening bytes, decodes them and scores them. Not a csv when
+/// @p encoding cannot be decoded.
+[[nodiscard]] Probe probe(const abstract::File &file, TextEncoding encoding,
+                          char quote = '"');
 
 } // namespace odr::internal::csv