diff --git a/src/odr/file.cpp b/src/odr/file.cpp index d30944cb5..554c4434c 100644 --- a/src/odr/file.cpp +++ b/src/odr/file.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -183,6 +184,11 @@ bool DecodedFile::is_text_file() const { nullptr; } +bool DecodedFile::is_csv_file() const { + return std::dynamic_pointer_cast(m_impl) != + nullptr; +} + bool DecodedFile::is_image_file() const { return std::dynamic_pointer_cast(m_impl) != nullptr; @@ -217,6 +223,15 @@ TextFile DecodedFile::as_text_file() const { throw NoTextFile(); } +CsvFile DecodedFile::as_csv_file() const { + if (const std::shared_ptr csv_file = + std::dynamic_pointer_cast(m_impl); + csv_file != nullptr) { + return CsvFile(csv_file); + } + throw NoCsvFile(); +} + ImageFile DecodedFile::as_image_file() const { if (const std::shared_ptr image_file = std::dynamic_pointer_cast(m_impl); @@ -290,6 +305,26 @@ std::string TextFile::text() const { return internal::encoding::to_utf8(bytes, encoding); } +CsvFile CsvFile::from_file(const File &file, const CsvOptions &options, + const Logger &logger) { + ODR_VERBOSE(logger, "open as csv with options"); + return CsvFile( + std::make_shared(file.impl(), options)); +} + +CsvFile::CsvFile(std::shared_ptr impl) + : DecodedFile(impl), m_impl{std::move(impl)} {} + +CsvOptions CsvFile::options() const { return m_impl->options(); } + +CsvFile CsvFile::with_options(const CsvOptions &options) const { + return CsvFile(m_impl->with_options(options)); +} + +std::shared_ptr CsvFile::impl() const { + return m_impl; +} + ImageFile::ImageFile(std::shared_ptr impl) : DecodedFile(impl), m_impl{std::move(impl)} {} diff --git a/src/odr/file.hpp b/src/odr/file.hpp index 9dc2c230c..b5065b4c7 100644 --- a/src/odr/file.hpp +++ b/src/odr/file.hpp @@ -12,6 +12,7 @@ namespace odr::internal::abstract { class File; class DecodedFile; class TextFile; +class CsvFile; class ImageFile; class ArchiveFile; class DocumentFile; @@ -21,6 +22,7 @@ class FontFile; namespace odr { class TextFile; +class CsvFile; class ImageFile; class ArchiveFile; class DocumentFile; @@ -368,6 +370,7 @@ class DecodedFile { [[nodiscard]] FileTypeCapabilities capabilities() const; [[nodiscard]] bool is_text_file() const; + [[nodiscard]] bool is_csv_file() const; [[nodiscard]] bool is_image_file() const; [[nodiscard]] bool is_archive_file() const; [[nodiscard]] bool is_document_file() const; @@ -375,6 +378,7 @@ class DecodedFile { [[nodiscard]] bool is_font_file() const; [[nodiscard]] TextFile as_text_file() const; + [[nodiscard]] CsvFile as_csv_file() const; [[nodiscard]] ImageFile as_image_file() const; [[nodiscard]] ArchiveFile as_archive_file() const; [[nodiscard]] DocumentFile as_document_file() const; @@ -408,6 +412,41 @@ class TextFile final : public DecodedFile { std::shared_ptr m_impl; }; +/// @brief How to read a csv file. +/// +/// An unset field is detected from the file's opening bytes; a set one is taken +/// as given. @ref CsvFile::options returns these with every field resolved, so +/// a caller can show what was detected and offer to change it. +struct CsvOptions final { + std::optional encoding{}; + std::optional separator{}; + std::optional quote{}; +}; + +/// @brief Represents a csv file. +class CsvFile final : public DecodedFile { +public: + /// @brief Decodes @p file as a csv, with @p options. + /// @throws NoCsvFile if no separator was given and the file does not look + /// like one. + [[nodiscard]] static CsvFile from_file(const File &file, + const CsvOptions &options, + const Logger &logger = Logger::null()); + + explicit CsvFile(std::shared_ptr); + + /// @brief The options in use, every field resolved. + [[nodiscard]] CsvOptions options() const; + + /// @brief The same file read with @p options. + [[nodiscard]] CsvFile with_options(const CsvOptions &options) const; + + [[nodiscard]] std::shared_ptr impl() const; + +private: + std::shared_ptr m_impl; +}; + /// @brief Represents an image file. class ImageFile final : public DecodedFile { public: diff --git a/src/odr/internal/abstract/file.hpp b/src/odr/internal/abstract/file.hpp index 05de574ae..0dbe81bd9 100644 --- a/src/odr/internal/abstract/file.hpp +++ b/src/odr/internal/abstract/file.hpp @@ -67,6 +67,16 @@ class TextFile : public DecodedFile { [[nodiscard]] virtual TextEncoding encoding() const noexcept = 0; }; +class CsvFile : public TextFile { +public: + /// The options in use, every field resolved. + [[nodiscard]] virtual CsvOptions options() const = 0; + + /// The same file read with @p options. + [[nodiscard]] virtual std::shared_ptr + with_options(const CsvOptions &options) const = 0; +}; + class ImageFile : public DecodedFile { public: [[nodiscard]] FileCategory file_category() const noexcept final { diff --git a/src/odr/internal/csv/csv_file.cpp b/src/odr/internal/csv/csv_file.cpp index 38e63af69..6fb7d6b7e 100644 --- a/src/odr/internal/csv/csv_file.cpp +++ b/src/odr/internal/csv/csv_file.cpp @@ -4,10 +4,25 @@ #include +#include #include namespace odr::internal::csv { +namespace { + +/// An incoherent dialect is a caller mistake, not bad input. +void check(const Dialect dialect) { + if (dialect.separator == dialect.quote) { + throw std::invalid_argument("csv separator equals its quote"); + } + if (dialect.separator == '\n' || dialect.separator == '\r') { + throw std::invalid_argument("csv separator is a line break"); + } +} + +} // namespace + CsvFile::CsvFile(std::shared_ptr file) : m_file{std::move(file)} { const Probe probe = csv::probe(*m_file->file(), m_file->encoding()); @@ -15,6 +30,32 @@ CsvFile::CsvFile(std::shared_ptr file) throw NoCsvFile(); } m_dialect = probe.dialect; + m_separator_directive = probe.separator_directive; +} + +CsvFile::CsvFile(std::shared_ptr file, + const CsvOptions &options) { + m_file = + options.encoding.has_value() + ? std::make_shared(std::move(file), *options.encoding) + : std::make_shared(std::move(file)); + + m_dialect.quote = options.quote.value_or(m_dialect.quote); + + if (options.separator.has_value()) { + m_dialect.separator = *options.separator; + check(m_dialect); + return; + } + + const Probe probe = + csv::probe(*m_file->file(), m_file->encoding(), m_dialect.quote); + if (!probe.is_csv) { + throw NoCsvFile(); + } + m_dialect.separator = probe.dialect.separator; + m_separator_directive = probe.separator_directive; + check(m_dialect); } std::shared_ptr CsvFile::file() const noexcept { @@ -38,6 +79,21 @@ bool CsvFile::is_decodable() const noexcept { return false; } TextEncoding CsvFile::encoding() const noexcept { return m_file->encoding(); } +CsvOptions CsvFile::options() const { + return {.encoding = encoding(), + .separator = m_dialect.separator, + .quote = m_dialect.quote}; +} + +std::shared_ptr +CsvFile::with_options(const CsvOptions &options) const { + return std::make_shared(m_file->file(), options); +} + Dialect CsvFile::dialect() const noexcept { return m_dialect; } +bool CsvFile::separator_directive() const noexcept { + return m_separator_directive; +} + } // namespace odr::internal::csv diff --git a/src/odr/internal/csv/csv_file.hpp b/src/odr/internal/csv/csv_file.hpp index c72847497..a8525e230 100644 --- a/src/odr/internal/csv/csv_file.hpp +++ b/src/odr/internal/csv/csv_file.hpp @@ -9,10 +9,17 @@ namespace odr::internal::csv { -class CsvFile final : public abstract::TextFile { +class CsvFile final : public abstract::CsvFile { public: + /// Detects everything; the path `open_strategy` probes with. + /// @throws NoCsvFile if it does not look like a csv. explicit CsvFile(std::shared_ptr file); + /// Detects only what @p options leaves unset. + /// @throws NoCsvFile if no separator was given and detection finds none. + /// @throws std::invalid_argument if @p options is not coherent. + CsvFile(std::shared_ptr file, const CsvOptions &options); + [[nodiscard]] std::shared_ptr file() const noexcept override; [[nodiscard]] FileType file_type() const noexcept override; @@ -23,12 +30,19 @@ class CsvFile final : public abstract::TextFile { [[nodiscard]] TextEncoding encoding() const noexcept override; - /// The dialect detection resolved. + [[nodiscard]] CsvOptions options() const override; + [[nodiscard]] std::shared_ptr + with_options(const CsvOptions &options) const override; + + /// The dialect in use. [[nodiscard]] Dialect dialect() const noexcept; + /// Whether the opening line is Excel's `sep=` directive rather than data. + [[nodiscard]] bool separator_directive() const noexcept; private: std::shared_ptr m_file; Dialect m_dialect; + bool m_separator_directive{false}; }; } // namespace odr::internal::csv diff --git a/test/src/internal/csv/csv_file_test.cpp b/test/src/internal/csv/csv_file_test.cpp index 04b4c5e5b..1aadb5997 100644 --- a/test/src/internal/csv/csv_file_test.cpp +++ b/test/src/internal/csv/csv_file_test.cpp @@ -146,3 +146,78 @@ TEST(RecordReader, an_unterminated_quote_still_yields_its_field) { EXPECT_EQ(fields, (std::vector{"a", "b"})); EXPECT_TRUE(reader.unterminated()); } + +TEST(CsvOptions, detection_fills_in_what_was_not_given) { + const CsvFile file = + CsvFile::from_file(File::from_memory("a;b\n1;2\n"), CsvOptions{}); + + const CsvOptions options = file.options(); + EXPECT_EQ(options.separator, ';'); + EXPECT_EQ(options.quote, '"'); + EXPECT_EQ(options.encoding, TextEncoding::utf8); +} + +/// Detection is a guess; a caller who knows better does not have to argue with +/// it, and nothing is refused for disagreeing. +TEST(CsvOptions, a_given_separator_is_taken_as_given) { + const std::string content = "a|b\n1|2\n"; + + EXPECT_EQ( + CsvFile::from_file(File::from_memory(content), {}).options().separator, + '|'); + EXPECT_EQ(CsvFile::from_file(File::from_memory(content), {.separator = ','}) + .options() + .separator, + ','); +} + +/// One column is no evidence of a csv, but it is a perfectly good csv once +/// someone says so. +TEST(CsvOptions, a_declared_separator_makes_anything_readable) { + EXPECT_THROW((void)CsvFile::from_file(File::from_memory("a\nb\nc\n"), {}), + NoCsvFile); + EXPECT_NO_THROW((void)CsvFile::from_file(File::from_memory("a\nb\nc\n"), + {.separator = ','})); + // and so is prose, and an empty file + EXPECT_NO_THROW((void)CsvFile::from_file( + File::from_memory("lorem ipsum dolor\nsit amet\n"), {.separator = ','})); + EXPECT_NO_THROW( + (void)CsvFile::from_file(File::from_memory(""), {.separator = ','})); +} + +TEST(CsvOptions, an_incoherent_dialect_is_a_caller_mistake) { + EXPECT_THROW((void)CsvFile::from_file(File::from_memory("a,b\n"), + {.separator = '"', .quote = '"'}), + std::invalid_argument); + EXPECT_THROW( + (void)CsvFile::from_file(File::from_memory("a,b\n"), {.separator = '\n'}), + std::invalid_argument); +} + +TEST(CsvOptions, a_given_encoding_skips_detection) { + // latin-1 bytes that are not valid utf-8; detection would not name them + const File file = File::from_memory("caf\xe9,x\nb,y\n"); + + EXPECT_EQ(CsvFile::from_file(file, {.encoding = TextEncoding::iso_8859_1}) + .options() + .encoding, + TextEncoding::iso_8859_1); +} + +TEST(CsvOptions, with_options_derives_another_handle) { + const CsvFile file = + CsvFile::from_file(File::from_memory("a;b\n1;2\n"), CsvOptions{}); + const CsvFile other = file.with_options({.separator = ','}); + + EXPECT_EQ(file.options().separator, ';'); + EXPECT_EQ(other.options().separator, ','); +} + +TEST(CsvOptions, a_decoded_csv_is_reachable_as_one) { + const File file( + TestData::test_file_path("odr-public/csv/file_example_ODS_5000.csv")); + const DecodedFile decoded(file, FileType::comma_separated_values); + + EXPECT_TRUE(decoded.is_csv_file()); + EXPECT_EQ(decoded.as_csv_file().options().separator, ','); +}