feat(encoding): name and decode text encodings

CMakeLists.txt

@@ -100,6 +100,10 @@ set(ODR_SOURCE_FILES
         "src/odr/table_dimension.cpp"
         "src/odr/table_position.cpp"
 
+        "src/odr/internal/encoding/detect.cpp"
+        "src/odr/internal/encoding/encoding_data.cpp"
+        "src/odr/internal/encoding/text_encoding_table.cpp"
+        "src/odr/internal/encoding/transcode.cpp"
         "src/odr/internal/file_type_table.cpp"
         "${CMAKE_CURRENT_BINARY_DIR}/src/odr/internal/git_info.cpp"
         "src/odr/internal/magic.cpp"
@@ -233,7 +237,6 @@ set(ODR_SOURCE_FILES
         "src/odr/internal/svm/svm_to_svg.cpp"
 
         "src/odr/internal/text/text_file.cpp"
-        "src/odr/internal/text/text_util.cpp"
 
         "src/odr/internal/util/byte_util.cpp"
         "src/odr/internal/util/byte_stream_util.cpp"

apple/src/ODRFile.mm

@@ -4,6 +4,7 @@
 #import "ODRPrivate.h"
 
 #include <odr/file.hpp>
+#include <odr/odr.hpp>
 
 #include <istream>
 #include <optional>
@@ -516,9 +517,12 @@ @implementation ODRTextFile
 - (nullable NSString *)charset {
   return guarded_value(
       [&]() -> NSString * {
-        const std::optional<std::string> charset =
-            self.handle.as_text_file().charset();
-        return charset.has_value() ? to_nsstring(*charset) : nil;
+        const odr::TextEncoding encoding =
+            self.handle.as_text_file().encoding();
+        return encoding == odr::TextEncoding::unknown
+                   ? nil
+                   : to_nsstring(
+                         std::string(odr::text_encoding_to_string(encoding)));
       },
       nil);
 }

jni/src/jni_file.cpp

@@ -5,6 +5,7 @@
 #include <odr/document.hpp>
 #include <odr/file.hpp>
 #include <odr/filesystem.hpp>
+#include <odr/odr.hpp>
 
 #include <sstream>
 
@@ -288,8 +289,12 @@ extern "C" JNIEXPORT jstring JNICALL
 Java_app_opendocument_core_TextFile_charsetNative(JNIEnv *env, jobject,
                                                   jlong handle) {
   return guarded(env, [&] {
-    return odr_jni::make_string_opt(env,
-                                    decoded(handle).as_text_file().charset());
+    const odr::TextEncoding encoding =
+        decoded(handle).as_text_file().encoding();
+    return odr_jni::make_string_opt(
+        env, encoding == odr::TextEncoding::unknown
+                 ? std::optional<std::string>{}
+                 : std::string(odr::text_encoding_to_string(encoding)));
   });
 }
 

python/src/bind_file.cpp

@@ -5,6 +5,7 @@
 #include <odr/file.hpp>
 #include <odr/filesystem.hpp>
 #include <odr/logger.hpp>
+#include <odr/odr.hpp>
 
 #include <pybind11/stl.h>
 
@@ -215,7 +216,14 @@ void odr_python::bind_file(py::module_ &m) {
       .def("as_font_file", &odr::DecodedFile::as_font_file);
 
   py::class_<odr::TextFile, odr::DecodedFile>(m, "TextFile")
-      .def("charset", &odr::TextFile::charset)
+      .def("charset",
+           [](const odr::TextFile &file) -> std::optional<std::string> {
+             const odr::TextEncoding encoding = file.encoding();
+             if (encoding == odr::TextEncoding::unknown) {
+               return {};
+             }
+             return std::string(odr::text_encoding_to_string(encoding));
+           })
       .def("text", &odr::TextFile::text);
 
   py::class_<odr::ImageFile, odr::DecodedFile>(m, "ImageFile")

src/odr/exceptions.cpp

@@ -22,6 +22,10 @@ UnsupportedFileType::UnsupportedFileType(const FileType file_type)
     : Exception("unsupported file type: " + file_type_to_string(file_type)),
       file_type{file_type} {}
 
+UnsupportedTextEncoding::UnsupportedTextEncoding(
+    const TextEncoding text_encoding)
+    : Exception("unsupported text encoding"), text_encoding{text_encoding} {}
+
 FileReadError::FileReadError() : Exception("file read error") {}
 
 FileWriteError::FileWriteError(const std::string &path)

src/odr/exceptions.hpp

@@ -5,6 +5,7 @@
 
 namespace odr {
 enum class FileType;
+enum class TextEncoding;
 
 /// @brief Base of every exception type this library declares. The decoders also
 /// throw plain `std::runtime_error` for malformed input with no dedicated type,
@@ -37,6 +38,13 @@ struct UnsupportedFileType final : Exception {
   explicit UnsupportedFileType(FileType file_type);
 };
 
+/// @brief Unsupported text encoding exception
+struct UnsupportedTextEncoding final : Exception {
+  TextEncoding text_encoding;
+
+  explicit UnsupportedTextEncoding(TextEncoding text_encoding);
+};
+
 /// @brief File read error
 struct FileReadError final : Exception {
   FileReadError();
@@ -88,7 +96,9 @@ struct NoJsonFile final : Exception {
 };
 
 /// @brief Unknown charset exception
-struct UnknownCharset final : Exception {
+/// @deprecated Nothing throws this any more: a file whose encoding cannot be
+/// named is still text, and reports @ref TextEncoding::unknown.
+struct [[deprecated("nothing throws this")]] UnknownCharset final : Exception {
   UnknownCharset();
 };
 

src/odr/file.cpp

@@ -7,6 +7,7 @@
 
 #include <odr/internal/abstract/file.hpp>
 #include <odr/internal/common/file.hpp>
+#include <odr/internal/encoding/transcode.hpp>
 #include <odr/internal/magic.hpp>
 #include <odr/internal/open_strategy.hpp>
 #include <odr/internal/util/file_util.hpp>
@@ -264,8 +265,14 @@ FontFile DecodedFile::as_font_file() const {
 TextFile::TextFile(std::shared_ptr<internal::abstract::TextFile> impl)
     : DecodedFile(impl), m_impl{std::move(impl)} {}
 
+TextEncoding TextFile::encoding() const { return m_impl->encoding(); }
+
 std::optional<std::string> TextFile::charset() const {
-  return {}; // TODO
+  const TextEncoding encoding = this->encoding();
+  if (encoding == TextEncoding::unknown) {
+    return {};
+  }
+  return std::string(text_encoding_to_string(encoding));
 }
 
 std::unique_ptr<std::istream> TextFile::stream() const {
@@ -274,7 +281,13 @@ std::unique_ptr<std::istream> TextFile::stream() const {
 
 std::string TextFile::text() const {
   const auto stream = m_impl->file()->stream();
-  return internal::util::stream::read(*stream);
+  std::string bytes = internal::util::stream::read(*stream);
+
+  const TextEncoding encoding = this->encoding();
+  if (!text_encoding_is_decodable(encoding)) {
+    return bytes;
+  }
+  return internal::encoding::to_utf8(bytes, encoding);
 }
 
 ImageFile::ImageFile(std::shared_ptr<internal::abstract::ImageFile> impl)

src/odr/file.hpp

@@ -208,6 +208,60 @@ enum class DocumentType {
   drawing,
 };
 
+/// @brief Collection of text encodings.
+///
+/// Only some can be decoded — see @ref text_encoding_is_decodable; the rest are
+/// named so a caller can say what a file is and hand it to something that can.
+enum class TextEncoding {
+  unknown,
+
+  // unicode
+  utf8,
+  utf16le,
+  utf16be,
+  utf32le,
+  utf32be,
+
+  // single byte
+  ibm866,
+  iso_8859_1,
+  iso_8859_2,
+  iso_8859_3,
+  iso_8859_4,
+  iso_8859_5,
+  iso_8859_6,
+  iso_8859_7,
+  iso_8859_8,
+  iso_8859_10,
+  iso_8859_13,
+  iso_8859_14,
+  iso_8859_15,
+  iso_8859_16,
+  koi8_r,
+  koi8_u,
+  macintosh,
+  windows_874,
+  windows_1250,
+  windows_1251,
+  windows_1252,
+  windows_1253,
+  windows_1254,
+  windows_1255,
+  windows_1256,
+  windows_1257,
+  windows_1258,
+  x_mac_cyrillic,
+
+  // multi byte; named but not decoded
+  big5,
+  euc_jp,
+  euc_kr,
+  gb18030,
+  iso_2022_jp,
+  iso_2022_kr,
+  shift_jis,
+};
+
 /// @brief Meta information about a file.
 ///
 /// The document fields are only meaningful when @ref document_type is set;
@@ -336,9 +390,18 @@ class TextFile final : public DecodedFile {
 public:
   explicit TextFile(std::shared_ptr<internal::abstract::TextFile>);
 
-  [[nodiscard]] std::optional<std::string> charset() const;
+  /// @brief The encoding the file's bytes were detected as, or decoded with.
+  [[nodiscard]] TextEncoding encoding() const;
+
+  /// @deprecated See @ref encoding. Returns the encoding's canonical name, and
+  /// `nullopt` for @ref TextEncoding::unknown.
+  [[deprecated("use encoding()")]] [[nodiscard]] std::optional<std::string>
+  charset() const;
 
+  /// @brief The file's bytes as they are.
   [[nodiscard]] std::unique_ptr<std::istream> stream() const;
+  /// @brief The file's text, decoded to UTF-8 where @ref encoding is
+  /// decodable, and the raw bytes where it is not.
   [[nodiscard]] std::string text() const;
 
 private:

src/odr/internal/abstract/file.hpp

@@ -62,6 +62,9 @@ class TextFile : public DecodedFile {
   [[nodiscard]] FileCategory file_category() const noexcept final {
     return FileCategory::text;
   }
+
+  /// The encoding the bytes were detected as, or decoded with.
+  [[nodiscard]] virtual TextEncoding encoding() const noexcept = 0;
 };
 
 class ImageFile : public DecodedFile {

src/odr/internal/csv/csv_file.cpp

@@ -29,4 +29,6 @@ FileMeta CsvFile::file_meta() const noexcept {
 
 bool CsvFile::is_decodable() const noexcept { return false; }
 
+TextEncoding CsvFile::encoding() const noexcept { return m_file->encoding(); }
+
 } // namespace odr::internal::csv

src/odr/internal/csv/csv_file.hpp

@@ -20,9 +20,10 @@ class CsvFile final : public abstract::TextFile {
 
   [[nodiscard]] bool is_decodable() const noexcept override;
 
+  [[nodiscard]] TextEncoding encoding() const noexcept override;
+
 private:
   std::shared_ptr<text::TextFile> m_file;
-  std::string m_charset;
 };
 
 } // namespace odr::internal::csv

src/odr/internal/encoding/detect.cpp

@@ -0,0 +1,67 @@
+#include <odr/internal/encoding/detect.hpp>
+
+#include <odr/internal/encoding/text_encoding_table.hpp>
+
+#include <istream>
+#include <string_view>
+
+#include <uchardet/uchardet.h>
+
+namespace odr::internal {
+
+namespace {
+
+using namespace std::string_view_literals;
+
+/// The encoding a byte-order mark names, `unknown` when there is none.
+TextEncoding encoding_by_bom(const std::string_view probe) {
+  const auto starts_with = [&](const std::string_view bom) {
+    return probe.substr(0, bom.size()) == bom;
+  };
+
+  // utf-32le opens with the utf-16le mark, so it has to be tested first
+  if (starts_with("\xff\xfe\x00\x00"sv)) {
+    return TextEncoding::utf32le;
+  }
+  if (starts_with("\x00\x00\xfe\xff"sv)) {
+    return TextEncoding::utf32be;
+  }
+  if (starts_with("\xef\xbb\xbf"sv)) {
+    return TextEncoding::utf8;
+  }
+  if (starts_with("\xff\xfe"sv)) {
+    return TextEncoding::utf16le;
+  }
+  if (starts_with("\xfe\xff"sv)) {
+    return TextEncoding::utf16be;
+  }
+  return TextEncoding::unknown;
+}
+
+} // namespace
+
+std::string encoding::read_probe(std::istream &in,
+                                 const std::size_t max_bytes) {
+  std::string result(max_bytes, '\0');
+  in.read(result.data(), static_cast<std::streamsize>(max_bytes));
+  result.resize(static_cast<std::size_t>(in.gcount()));
+  return result;
+}
+
+TextEncoding encoding::detect(const std::string_view probe) {
+  if (const TextEncoding by_bom = encoding_by_bom(probe);
+      by_bom != TextEncoding::unknown) {
+    return by_bom;
+  }
+
+  auto *const detector = uchardet_new();
+  uchardet_handle_data(detector, probe.data(), probe.size());
+  uchardet_data_end(detector);
+  const std::string name = uchardet_get_charset(detector);
+  uchardet_delete(detector);
+
+  const auto *row = text_encoding_table::find_by_name(name);
+  return row == nullptr ? TextEncoding::unknown : row->encoding;
+}
+
+} // namespace odr::internal

src/odr/internal/encoding/detect.hpp

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <odr/file.hpp>
+
+#include <cstddef>
+#include <iosfwd>
+#include <string>
+#include <string_view>
+
+namespace odr::internal::encoding {
+
+/// Enough for uchardet, and bounded so classifying a large file costs no pass
+/// over it.
+constexpr std::size_t default_probe_size = std::size_t{64} * 1024;
+
+/// Reads at most @p max_bytes from @p in, from wherever it stands.
+[[nodiscard]] std::string
+read_probe(std::istream &in, std::size_t max_bytes = default_probe_size);
+
+/// The encoding of @p probe, @ref TextEncoding::unknown if it cannot be named.
+/// A byte-order mark decides alone, else uchardet guesses — from the probe
+/// only, so an encoding invisible in an ASCII prefix is missed.
+[[nodiscard]] TextEncoding detect(std::string_view probe);
+
+} // namespace odr::internal::encoding