From cbc058ae29a218599cac51f34b144f081c29f7e9 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 9 Aug 2026 09:26:30 +0200 Subject: [PATCH] feat(encoding): name and decode text encodings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `TextFile::charset()` was a stub returning nothing, and the charset uchardet guessed was used nowhere — every non-UTF-8 text file rendered as mojibake because `html/text_file.cpp` piped raw bytes into a document declaring UTF-8. Detection also drained the whole stream, so classifying one unrecognised large file cost a full pass over it. A new `internal/encoding` package carries a public `TextEncoding` enum with a name table behind it, mirroring `file_type_table`: canonical name first, aliases after, matching that ignores case and `-`/`_`/space so `windows-1252`, `WINDOWS_1252` and `cp1252` land together. uchardet returns a name; this is what maps it home. Decoding covers UTF-8/16/32 and the WHATWG single-byte set, generated from Python's codecs by `tools/encoding/generate_encoding_data.py`. Malformed input becomes U+FFFD rather than an error: a decoder that throws half way through leaves the caller with nothing, and a wrong guess is the expected failure. The multi-byte legacy encodings are named but not decoded, which is enough to render them as text with their own charset in the html header and let the browser do the work. Detection now reads a bounded probe. Nothing rejects: text is the fallback for bytes nothing else claims, so a viewer handed a random binary shows junk rather than an error, and bytes we cannot name are `TextEncoding::unknown` rather than a throw. `TextFile::text()` now returns what its name promises — the file's text, decoded to UTF-8 — where before it returned raw bytes and a Latin-1 file came back as mojibake. `stream()` is unchanged for callers that want the bytes. `TextFile::charset()` stays, deprecated and now actually working, in terms of the new `TextFile::encoding()`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QSgWdLTSLCWDeFvbVwZDVU --- CMakeLists.txt | 5 +- apple/src/ODRFile.mm | 10 +- jni/src/jni_file.cpp | 9 +- python/src/bind_file.cpp | 10 +- src/odr/exceptions.cpp | 4 + src/odr/exceptions.hpp | 12 +- src/odr/file.cpp | 17 +- src/odr/file.hpp | 65 +- src/odr/internal/abstract/file.hpp | 3 + src/odr/internal/csv/csv_file.cpp | 2 + src/odr/internal/csv/csv_file.hpp | 3 +- src/odr/internal/encoding/detect.cpp | 67 ++ src/odr/internal/encoding/detect.hpp | 25 + src/odr/internal/encoding/encoding_data.cpp | 991 ++++++++++++++++++ src/odr/internal/encoding/encoding_data.hpp | 46 + .../internal/encoding/text_encoding_table.cpp | 159 +++ .../internal/encoding/text_encoding_table.hpp | 28 + src/odr/internal/encoding/transcode.cpp | 230 ++++ src/odr/internal/encoding/transcode.hpp | 18 + src/odr/internal/html/text_file.cpp | 32 +- src/odr/internal/json/json_file.cpp | 2 + src/odr/internal/json/json_file.hpp | 3 +- src/odr/internal/text/text_file.cpp | 32 +- src/odr/internal/text/text_file.hpp | 10 +- src/odr/internal/text/text_util.cpp | 35 - src/odr/internal/text/text_util.hpp | 11 - src/odr/odr.cpp | 37 + src/odr/odr.hpp | 20 + test/CMakeLists.txt | 1 + .../internal/encoding/text_encoding_test.cpp | 150 +++ test/src/internal/text/text_file_test.cpp | 59 +- tools/encoding/generate_encoding_data.py | 130 +++ 32 files changed, 2150 insertions(+), 76 deletions(-) create mode 100644 src/odr/internal/encoding/detect.cpp create mode 100644 src/odr/internal/encoding/detect.hpp create mode 100644 src/odr/internal/encoding/encoding_data.cpp create mode 100644 src/odr/internal/encoding/encoding_data.hpp create mode 100644 src/odr/internal/encoding/text_encoding_table.cpp create mode 100644 src/odr/internal/encoding/text_encoding_table.hpp create mode 100644 src/odr/internal/encoding/transcode.cpp create mode 100644 src/odr/internal/encoding/transcode.hpp delete mode 100644 src/odr/internal/text/text_util.cpp delete mode 100644 src/odr/internal/text/text_util.hpp create mode 100644 test/src/internal/encoding/text_encoding_test.cpp create mode 100644 tools/encoding/generate_encoding_data.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e4299d96..d37d8ee5e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,6 +100,10 @@ set(ODR_SOURCE_FILES "src/odr/table_dimension.cpp" "src/odr/table_position.cpp" + "src/odr/internal/encoding/detect.cpp" + "src/odr/internal/encoding/encoding_data.cpp" + "src/odr/internal/encoding/text_encoding_table.cpp" + "src/odr/internal/encoding/transcode.cpp" "src/odr/internal/file_type_table.cpp" "${CMAKE_CURRENT_BINARY_DIR}/src/odr/internal/git_info.cpp" "src/odr/internal/magic.cpp" @@ -233,7 +237,6 @@ set(ODR_SOURCE_FILES "src/odr/internal/svm/svm_to_svg.cpp" "src/odr/internal/text/text_file.cpp" - "src/odr/internal/text/text_util.cpp" "src/odr/internal/util/byte_util.cpp" "src/odr/internal/util/byte_stream_util.cpp" diff --git a/apple/src/ODRFile.mm b/apple/src/ODRFile.mm index f7167a1ec..d1dad972d 100644 --- a/apple/src/ODRFile.mm +++ b/apple/src/ODRFile.mm @@ -4,6 +4,7 @@ #import "ODRPrivate.h" #include +#include #include #include @@ -516,9 +517,12 @@ @implementation ODRTextFile - (nullable NSString *)charset { return guarded_value( [&]() -> NSString * { - const std::optional charset = - self.handle.as_text_file().charset(); - return charset.has_value() ? to_nsstring(*charset) : nil; + const odr::TextEncoding encoding = + self.handle.as_text_file().encoding(); + return encoding == odr::TextEncoding::unknown + ? nil + : to_nsstring( + std::string(odr::text_encoding_to_string(encoding))); }, nil); } diff --git a/jni/src/jni_file.cpp b/jni/src/jni_file.cpp index 9331d8f3c..b55218ccf 100644 --- a/jni/src/jni_file.cpp +++ b/jni/src/jni_file.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -288,8 +289,12 @@ extern "C" JNIEXPORT jstring JNICALL Java_app_opendocument_core_TextFile_charsetNative(JNIEnv *env, jobject, jlong handle) { return guarded(env, [&] { - return odr_jni::make_string_opt(env, - decoded(handle).as_text_file().charset()); + const odr::TextEncoding encoding = + decoded(handle).as_text_file().encoding(); + return odr_jni::make_string_opt( + env, encoding == odr::TextEncoding::unknown + ? std::optional{} + : std::string(odr::text_encoding_to_string(encoding))); }); } diff --git a/python/src/bind_file.cpp b/python/src/bind_file.cpp index c9159fb9a..b0eaf79d6 100644 --- a/python/src/bind_file.cpp +++ b/python/src/bind_file.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -215,7 +216,14 @@ void odr_python::bind_file(py::module_ &m) { .def("as_font_file", &odr::DecodedFile::as_font_file); py::class_(m, "TextFile") - .def("charset", &odr::TextFile::charset) + .def("charset", + [](const odr::TextFile &file) -> std::optional { + const odr::TextEncoding encoding = file.encoding(); + if (encoding == odr::TextEncoding::unknown) { + return {}; + } + return std::string(odr::text_encoding_to_string(encoding)); + }) .def("text", &odr::TextFile::text); py::class_(m, "ImageFile") diff --git a/src/odr/exceptions.cpp b/src/odr/exceptions.cpp index 1ff5b674d..065ca8e05 100644 --- a/src/odr/exceptions.cpp +++ b/src/odr/exceptions.cpp @@ -22,6 +22,10 @@ UnsupportedFileType::UnsupportedFileType(const FileType file_type) : Exception("unsupported file type: " + file_type_to_string(file_type)), file_type{file_type} {} +UnsupportedTextEncoding::UnsupportedTextEncoding( + const TextEncoding text_encoding) + : Exception("unsupported text encoding"), text_encoding{text_encoding} {} + FileReadError::FileReadError() : Exception("file read error") {} FileWriteError::FileWriteError(const std::string &path) diff --git a/src/odr/exceptions.hpp b/src/odr/exceptions.hpp index 4e0d2fdf8..dac37e653 100644 --- a/src/odr/exceptions.hpp +++ b/src/odr/exceptions.hpp @@ -5,6 +5,7 @@ namespace odr { enum class FileType; +enum class TextEncoding; /// @brief Base of every exception type this library declares. The decoders also /// throw plain `std::runtime_error` for malformed input with no dedicated type, @@ -37,6 +38,13 @@ struct UnsupportedFileType final : Exception { explicit UnsupportedFileType(FileType file_type); }; +/// @brief Unsupported text encoding exception +struct UnsupportedTextEncoding final : Exception { + TextEncoding text_encoding; + + explicit UnsupportedTextEncoding(TextEncoding text_encoding); +}; + /// @brief File read error struct FileReadError final : Exception { FileReadError(); @@ -88,7 +96,9 @@ struct NoJsonFile final : Exception { }; /// @brief Unknown charset exception -struct UnknownCharset final : Exception { +/// @deprecated Nothing throws this any more: a file whose encoding cannot be +/// named is still text, and reports @ref TextEncoding::unknown. +struct [[deprecated("nothing throws this")]] UnknownCharset final : Exception { UnknownCharset(); }; diff --git a/src/odr/file.cpp b/src/odr/file.cpp index d14d3c482..d30944cb5 100644 --- a/src/odr/file.cpp +++ b/src/odr/file.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -264,8 +265,14 @@ FontFile DecodedFile::as_font_file() const { TextFile::TextFile(std::shared_ptr impl) : DecodedFile(impl), m_impl{std::move(impl)} {} +TextEncoding TextFile::encoding() const { return m_impl->encoding(); } + std::optional TextFile::charset() const { - return {}; // TODO + const TextEncoding encoding = this->encoding(); + if (encoding == TextEncoding::unknown) { + return {}; + } + return std::string(text_encoding_to_string(encoding)); } std::unique_ptr TextFile::stream() const { @@ -274,7 +281,13 @@ std::unique_ptr TextFile::stream() const { std::string TextFile::text() const { const auto stream = m_impl->file()->stream(); - return internal::util::stream::read(*stream); + std::string bytes = internal::util::stream::read(*stream); + + const TextEncoding encoding = this->encoding(); + if (!text_encoding_is_decodable(encoding)) { + return bytes; + } + return internal::encoding::to_utf8(bytes, encoding); } ImageFile::ImageFile(std::shared_ptr impl) diff --git a/src/odr/file.hpp b/src/odr/file.hpp index 0b2f872fd..9dc2c230c 100644 --- a/src/odr/file.hpp +++ b/src/odr/file.hpp @@ -208,6 +208,60 @@ enum class DocumentType { drawing, }; +/// @brief Collection of text encodings. +/// +/// Only some can be decoded — see @ref text_encoding_is_decodable; the rest are +/// named so a caller can say what a file is and hand it to something that can. +enum class TextEncoding { + unknown, + + // unicode + utf8, + utf16le, + utf16be, + utf32le, + utf32be, + + // single byte + ibm866, + iso_8859_1, + iso_8859_2, + iso_8859_3, + iso_8859_4, + iso_8859_5, + iso_8859_6, + iso_8859_7, + iso_8859_8, + iso_8859_10, + iso_8859_13, + iso_8859_14, + iso_8859_15, + iso_8859_16, + koi8_r, + koi8_u, + macintosh, + windows_874, + windows_1250, + windows_1251, + windows_1252, + windows_1253, + windows_1254, + windows_1255, + windows_1256, + windows_1257, + windows_1258, + x_mac_cyrillic, + + // multi byte; named but not decoded + big5, + euc_jp, + euc_kr, + gb18030, + iso_2022_jp, + iso_2022_kr, + shift_jis, +}; + /// @brief Meta information about a file. /// /// The document fields are only meaningful when @ref document_type is set; @@ -336,9 +390,18 @@ class TextFile final : public DecodedFile { public: explicit TextFile(std::shared_ptr); - [[nodiscard]] std::optional charset() const; + /// @brief The encoding the file's bytes were detected as, or decoded with. + [[nodiscard]] TextEncoding encoding() const; + + /// @deprecated See @ref encoding. Returns the encoding's canonical name, and + /// `nullopt` for @ref TextEncoding::unknown. + [[deprecated("use encoding()")]] [[nodiscard]] std::optional + charset() const; + /// @brief The file's bytes as they are. [[nodiscard]] std::unique_ptr stream() const; + /// @brief The file's text, decoded to UTF-8 where @ref encoding is + /// decodable, and the raw bytes where it is not. [[nodiscard]] std::string text() const; private: diff --git a/src/odr/internal/abstract/file.hpp b/src/odr/internal/abstract/file.hpp index 94a9ea519..05de574ae 100644 --- a/src/odr/internal/abstract/file.hpp +++ b/src/odr/internal/abstract/file.hpp @@ -62,6 +62,9 @@ class TextFile : public DecodedFile { [[nodiscard]] FileCategory file_category() const noexcept final { return FileCategory::text; } + + /// The encoding the bytes were detected as, or decoded with. + [[nodiscard]] virtual TextEncoding encoding() const noexcept = 0; }; class ImageFile : public DecodedFile { diff --git a/src/odr/internal/csv/csv_file.cpp b/src/odr/internal/csv/csv_file.cpp index a129c3924..1c5751905 100644 --- a/src/odr/internal/csv/csv_file.cpp +++ b/src/odr/internal/csv/csv_file.cpp @@ -29,4 +29,6 @@ FileMeta CsvFile::file_meta() const noexcept { bool CsvFile::is_decodable() const noexcept { return false; } +TextEncoding CsvFile::encoding() const noexcept { return m_file->encoding(); } + } // namespace odr::internal::csv diff --git a/src/odr/internal/csv/csv_file.hpp b/src/odr/internal/csv/csv_file.hpp index 50a0c84fe..2f4df0c7f 100644 --- a/src/odr/internal/csv/csv_file.hpp +++ b/src/odr/internal/csv/csv_file.hpp @@ -20,9 +20,10 @@ class CsvFile final : public abstract::TextFile { [[nodiscard]] bool is_decodable() const noexcept override; + [[nodiscard]] TextEncoding encoding() const noexcept override; + private: std::shared_ptr m_file; - std::string m_charset; }; } // namespace odr::internal::csv diff --git a/src/odr/internal/encoding/detect.cpp b/src/odr/internal/encoding/detect.cpp new file mode 100644 index 000000000..52edc32e5 --- /dev/null +++ b/src/odr/internal/encoding/detect.cpp @@ -0,0 +1,67 @@ +#include + +#include + +#include +#include + +#include + +namespace odr::internal { + +namespace { + +using namespace std::string_view_literals; + +/// The encoding a byte-order mark names, `unknown` when there is none. +TextEncoding encoding_by_bom(const std::string_view probe) { + const auto starts_with = [&](const std::string_view bom) { + return probe.substr(0, bom.size()) == bom; + }; + + // utf-32le opens with the utf-16le mark, so it has to be tested first + if (starts_with("\xff\xfe\x00\x00"sv)) { + return TextEncoding::utf32le; + } + if (starts_with("\x00\x00\xfe\xff"sv)) { + return TextEncoding::utf32be; + } + if (starts_with("\xef\xbb\xbf"sv)) { + return TextEncoding::utf8; + } + if (starts_with("\xff\xfe"sv)) { + return TextEncoding::utf16le; + } + if (starts_with("\xfe\xff"sv)) { + return TextEncoding::utf16be; + } + return TextEncoding::unknown; +} + +} // namespace + +std::string encoding::read_probe(std::istream &in, + const std::size_t max_bytes) { + std::string result(max_bytes, '\0'); + in.read(result.data(), static_cast(max_bytes)); + result.resize(static_cast(in.gcount())); + return result; +} + +TextEncoding encoding::detect(const std::string_view probe) { + if (const TextEncoding by_bom = encoding_by_bom(probe); + by_bom != TextEncoding::unknown) { + return by_bom; + } + + auto *const detector = uchardet_new(); + uchardet_handle_data(detector, probe.data(), probe.size()); + uchardet_data_end(detector); + const std::string name = uchardet_get_charset(detector); + uchardet_delete(detector); + + const auto *row = text_encoding_table::find_by_name(name); + return row == nullptr ? TextEncoding::unknown : row->encoding; +} + +} // namespace odr::internal diff --git a/src/odr/internal/encoding/detect.hpp b/src/odr/internal/encoding/detect.hpp new file mode 100644 index 000000000..51e451360 --- /dev/null +++ b/src/odr/internal/encoding/detect.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace odr::internal::encoding { + +/// Enough for uchardet, and bounded so classifying a large file costs no pass +/// over it. +constexpr std::size_t default_probe_size = std::size_t{64} * 1024; + +/// Reads at most @p max_bytes from @p in, from wherever it stands. +[[nodiscard]] std::string +read_probe(std::istream &in, std::size_t max_bytes = default_probe_size); + +/// The encoding of @p probe, @ref TextEncoding::unknown if it cannot be named. +/// A byte-order mark decides alone, else uchardet guesses — from the probe +/// only, so an encoding invisible in an ASCII prefix is missed. +[[nodiscard]] TextEncoding detect(std::string_view probe); + +} // namespace odr::internal::encoding diff --git a/src/odr/internal/encoding/encoding_data.cpp b/src/odr/internal/encoding/encoding_data.cpp new file mode 100644 index 000000000..e6d02d685 --- /dev/null +++ b/src/odr/internal/encoding/encoding_data.cpp @@ -0,0 +1,991 @@ +// Auto-generated by tools/encoding/generate_encoding_data.py -- DO NOT EDIT. +// Regenerate with: python3 tools/encoding/generate_encoding_data.py +// Data: the Python standard library's codecs. + +// clang-format off + +#include + +namespace odr::internal::encoding::encoding_data { + +const SingleByteTable ibm866{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556, + 0x2555, 0x2563, 0x2551, 0x2557, 0x255d, 0x255c, 0x255b, 0x2510, + 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x255e, 0x255f, + 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x2567, + 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256b, + 0x256a, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, + 0x0401, 0x0451, 0x0404, 0x0454, 0x0407, 0x0457, 0x040e, 0x045e, + 0x00b0, 0x2219, 0x00b7, 0x221a, 0x2116, 0x00a4, 0x25a0, 0x00a0, +}; + +const SingleByteTable iso_8859_1{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff, +}; + +const SingleByteTable iso_8859_2{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0104, 0x02d8, 0x0141, 0x00a4, 0x013d, 0x015a, 0x00a7, + 0x00a8, 0x0160, 0x015e, 0x0164, 0x0179, 0x00ad, 0x017d, 0x017b, + 0x00b0, 0x0105, 0x02db, 0x0142, 0x00b4, 0x013e, 0x015b, 0x02c7, + 0x00b8, 0x0161, 0x015f, 0x0165, 0x017a, 0x02dd, 0x017e, 0x017c, + 0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7, + 0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e, + 0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7, + 0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df, + 0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7, + 0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f, + 0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7, + 0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9, +}; + +const SingleByteTable iso_8859_3{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0126, 0x02d8, 0x00a3, 0x00a4, 0xfffd, 0x0124, 0x00a7, + 0x00a8, 0x0130, 0x015e, 0x011e, 0x0134, 0x00ad, 0xfffd, 0x017b, + 0x00b0, 0x0127, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x0125, 0x00b7, + 0x00b8, 0x0131, 0x015f, 0x011f, 0x0135, 0x00bd, 0xfffd, 0x017c, + 0x00c0, 0x00c1, 0x00c2, 0xfffd, 0x00c4, 0x010a, 0x0108, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0xfffd, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x0120, 0x00d6, 0x00d7, + 0x011c, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x016c, 0x015c, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0xfffd, 0x00e4, 0x010b, 0x0109, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0xfffd, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x0121, 0x00f6, 0x00f7, + 0x011d, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x016d, 0x015d, 0x02d9, +}; + +const SingleByteTable iso_8859_4{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0104, 0x0138, 0x0156, 0x00a4, 0x0128, 0x013b, 0x00a7, + 0x00a8, 0x0160, 0x0112, 0x0122, 0x0166, 0x00ad, 0x017d, 0x00af, + 0x00b0, 0x0105, 0x02db, 0x0157, 0x00b4, 0x0129, 0x013c, 0x02c7, + 0x00b8, 0x0161, 0x0113, 0x0123, 0x0167, 0x014a, 0x017e, 0x014b, + 0x0100, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x012e, + 0x010c, 0x00c9, 0x0118, 0x00cb, 0x0116, 0x00cd, 0x00ce, 0x012a, + 0x0110, 0x0145, 0x014c, 0x0136, 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x0172, 0x00da, 0x00db, 0x00dc, 0x0168, 0x016a, 0x00df, + 0x0101, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x012f, + 0x010d, 0x00e9, 0x0119, 0x00eb, 0x0117, 0x00ed, 0x00ee, 0x012b, + 0x0111, 0x0146, 0x014d, 0x0137, 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x0173, 0x00fa, 0x00fb, 0x00fc, 0x0169, 0x016b, 0x02d9, +}; + +const SingleByteTable iso_8859_5{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0401, 0x0402, 0x0403, 0x0404, 0x0405, 0x0406, 0x0407, + 0x0408, 0x0409, 0x040a, 0x040b, 0x040c, 0x00ad, 0x040e, 0x040f, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, + 0x2116, 0x0451, 0x0452, 0x0453, 0x0454, 0x0455, 0x0456, 0x0457, + 0x0458, 0x0459, 0x045a, 0x045b, 0x045c, 0x00a7, 0x045e, 0x045f, +}; + +const SingleByteTable iso_8859_6{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0xfffd, 0xfffd, 0xfffd, 0x00a4, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0x060c, 0x00ad, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0x061b, 0xfffd, 0xfffd, 0xfffd, 0x061f, + 0xfffd, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, + 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f, + 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x0637, + 0x0638, 0x0639, 0x063a, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647, + 0x0648, 0x0649, 0x064a, 0x064b, 0x064c, 0x064d, 0x064e, 0x064f, + 0x0650, 0x0651, 0x0652, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, +}; + +const SingleByteTable iso_8859_7{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x2018, 0x2019, 0x00a3, 0x20ac, 0x20af, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x037a, 0x00ab, 0x00ac, 0x00ad, 0xfffd, 0x2015, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x0384, 0x0385, 0x0386, 0x00b7, + 0x0388, 0x0389, 0x038a, 0x00bb, 0x038c, 0x00bd, 0x038e, 0x038f, + 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, + 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f, + 0x03a0, 0x03a1, 0xfffd, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7, + 0x03a8, 0x03a9, 0x03aa, 0x03ab, 0x03ac, 0x03ad, 0x03ae, 0x03af, + 0x03b0, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, + 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, + 0x03c0, 0x03c1, 0x03c2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, + 0x03c8, 0x03c9, 0x03ca, 0x03cb, 0x03cc, 0x03cd, 0x03ce, 0xfffd, +}; + +const SingleByteTable iso_8859_8{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0xfffd, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00d7, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00f7, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0x2017, + 0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0x05d7, + 0x05d8, 0x05d9, 0x05da, 0x05db, 0x05dc, 0x05dd, 0x05de, 0x05df, + 0x05e0, 0x05e1, 0x05e2, 0x05e3, 0x05e4, 0x05e5, 0x05e6, 0x05e7, + 0x05e8, 0x05e9, 0x05ea, 0xfffd, 0xfffd, 0x200e, 0x200f, 0xfffd, +}; + +const SingleByteTable iso_8859_10{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0104, 0x0112, 0x0122, 0x012a, 0x0128, 0x0136, 0x00a7, + 0x013b, 0x0110, 0x0160, 0x0166, 0x017d, 0x00ad, 0x016a, 0x014a, + 0x00b0, 0x0105, 0x0113, 0x0123, 0x012b, 0x0129, 0x0137, 0x00b7, + 0x013c, 0x0111, 0x0161, 0x0167, 0x017e, 0x2015, 0x016b, 0x014b, + 0x0100, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x012e, + 0x010c, 0x00c9, 0x0118, 0x00cb, 0x0116, 0x00cd, 0x00ce, 0x00cf, + 0x00d0, 0x0145, 0x014c, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x0168, + 0x00d8, 0x0172, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df, + 0x0101, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x012f, + 0x010d, 0x00e9, 0x0119, 0x00eb, 0x0117, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x0146, 0x014d, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x0169, + 0x00f8, 0x0173, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x0138, +}; + +const SingleByteTable iso_8859_13{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x201d, 0x00a2, 0x00a3, 0x00a4, 0x201e, 0x00a6, 0x00a7, + 0x00d8, 0x00a9, 0x0156, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00c6, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x201c, 0x00b5, 0x00b6, 0x00b7, + 0x00f8, 0x00b9, 0x0157, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00e6, + 0x0104, 0x012e, 0x0100, 0x0106, 0x00c4, 0x00c5, 0x0118, 0x0112, + 0x010c, 0x00c9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012a, 0x013b, + 0x0160, 0x0143, 0x0145, 0x00d3, 0x014c, 0x00d5, 0x00d6, 0x00d7, + 0x0172, 0x0141, 0x015a, 0x016a, 0x00dc, 0x017b, 0x017d, 0x00df, + 0x0105, 0x012f, 0x0101, 0x0107, 0x00e4, 0x00e5, 0x0119, 0x0113, + 0x010d, 0x00e9, 0x017a, 0x0117, 0x0123, 0x0137, 0x012b, 0x013c, + 0x0161, 0x0144, 0x0146, 0x00f3, 0x014d, 0x00f5, 0x00f6, 0x00f7, + 0x0173, 0x0142, 0x015b, 0x016b, 0x00fc, 0x017c, 0x017e, 0x2019, +}; + +const SingleByteTable iso_8859_14{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x1e02, 0x1e03, 0x00a3, 0x010a, 0x010b, 0x1e0a, 0x00a7, + 0x1e80, 0x00a9, 0x1e82, 0x1e0b, 0x1ef2, 0x00ad, 0x00ae, 0x0178, + 0x1e1e, 0x1e1f, 0x0120, 0x0121, 0x1e40, 0x1e41, 0x00b6, 0x1e56, + 0x1e81, 0x1e57, 0x1e83, 0x1e60, 0x1ef3, 0x1e84, 0x1e85, 0x1e61, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x0174, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x1e6a, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x0176, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x0175, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x1e6b, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x0177, 0x00ff, +}; + +const SingleByteTable iso_8859_15{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x20ac, 0x00a5, 0x0160, 0x00a7, + 0x0161, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x017d, 0x00b5, 0x00b6, 0x00b7, + 0x017e, 0x00b9, 0x00ba, 0x00bb, 0x0152, 0x0153, 0x0178, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff, +}; + +const SingleByteTable iso_8859_16{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0104, 0x0105, 0x0141, 0x20ac, 0x201e, 0x0160, 0x00a7, + 0x0161, 0x00a9, 0x0218, 0x00ab, 0x0179, 0x00ad, 0x017a, 0x017b, + 0x00b0, 0x00b1, 0x010c, 0x0142, 0x017d, 0x201d, 0x00b6, 0x00b7, + 0x017e, 0x010d, 0x0219, 0x00bb, 0x0152, 0x0153, 0x0178, 0x017c, + 0x00c0, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0106, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x0110, 0x0143, 0x00d2, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x015a, + 0x0170, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x0118, 0x021a, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x0107, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x0111, 0x0144, 0x00f2, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x015b, + 0x0171, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x0119, 0x021b, 0x00ff, +}; + +const SingleByteTable koi8_r{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x2500, 0x2502, 0x250c, 0x2510, 0x2514, 0x2518, 0x251c, 0x2524, + 0x252c, 0x2534, 0x253c, 0x2580, 0x2584, 0x2588, 0x258c, 0x2590, + 0x2591, 0x2592, 0x2593, 0x2320, 0x25a0, 0x2219, 0x221a, 0x2248, + 0x2264, 0x2265, 0x00a0, 0x2321, 0x00b0, 0x00b2, 0x00b7, 0x00f7, + 0x2550, 0x2551, 0x2552, 0x0451, 0x2553, 0x2554, 0x2555, 0x2556, + 0x2557, 0x2558, 0x2559, 0x255a, 0x255b, 0x255c, 0x255d, 0x255e, + 0x255f, 0x2560, 0x2561, 0x0401, 0x2562, 0x2563, 0x2564, 0x2565, + 0x2566, 0x2567, 0x2568, 0x2569, 0x256a, 0x256b, 0x256c, 0x00a9, + 0x044e, 0x0430, 0x0431, 0x0446, 0x0434, 0x0435, 0x0444, 0x0433, + 0x0445, 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, + 0x043f, 0x044f, 0x0440, 0x0441, 0x0442, 0x0443, 0x0436, 0x0432, + 0x044c, 0x044b, 0x0437, 0x0448, 0x044d, 0x0449, 0x0447, 0x044a, + 0x042e, 0x0410, 0x0411, 0x0426, 0x0414, 0x0415, 0x0424, 0x0413, + 0x0425, 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, + 0x041f, 0x042f, 0x0420, 0x0421, 0x0422, 0x0423, 0x0416, 0x0412, + 0x042c, 0x042b, 0x0417, 0x0428, 0x042d, 0x0429, 0x0427, 0x042a, +}; + +const SingleByteTable koi8_u{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x2500, 0x2502, 0x250c, 0x2510, 0x2514, 0x2518, 0x251c, 0x2524, + 0x252c, 0x2534, 0x253c, 0x2580, 0x2584, 0x2588, 0x258c, 0x2590, + 0x2591, 0x2592, 0x2593, 0x2320, 0x25a0, 0x2219, 0x221a, 0x2248, + 0x2264, 0x2265, 0x00a0, 0x2321, 0x00b0, 0x00b2, 0x00b7, 0x00f7, + 0x2550, 0x2551, 0x2552, 0x0451, 0x0454, 0x2554, 0x0456, 0x0457, + 0x2557, 0x2558, 0x2559, 0x255a, 0x255b, 0x0491, 0x255d, 0x255e, + 0x255f, 0x2560, 0x2561, 0x0401, 0x0404, 0x2563, 0x0406, 0x0407, + 0x2566, 0x2567, 0x2568, 0x2569, 0x256a, 0x0490, 0x256c, 0x00a9, + 0x044e, 0x0430, 0x0431, 0x0446, 0x0434, 0x0435, 0x0444, 0x0433, + 0x0445, 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, + 0x043f, 0x044f, 0x0440, 0x0441, 0x0442, 0x0443, 0x0436, 0x0432, + 0x044c, 0x044b, 0x0437, 0x0448, 0x044d, 0x0449, 0x0447, 0x044a, + 0x042e, 0x0410, 0x0411, 0x0426, 0x0414, 0x0415, 0x0424, 0x0413, + 0x0425, 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, + 0x041f, 0x042f, 0x0420, 0x0421, 0x0422, 0x0423, 0x0416, 0x0412, + 0x042c, 0x042b, 0x0417, 0x0428, 0x042d, 0x0429, 0x0427, 0x042a, +}; + +const SingleByteTable macintosh{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x00c4, 0x00c5, 0x00c7, 0x00c9, 0x00d1, 0x00d6, 0x00dc, 0x00e1, + 0x00e0, 0x00e2, 0x00e4, 0x00e3, 0x00e5, 0x00e7, 0x00e9, 0x00e8, + 0x00ea, 0x00eb, 0x00ed, 0x00ec, 0x00ee, 0x00ef, 0x00f1, 0x00f3, + 0x00f2, 0x00f4, 0x00f6, 0x00f5, 0x00fa, 0x00f9, 0x00fb, 0x00fc, + 0x2020, 0x00b0, 0x00a2, 0x00a3, 0x00a7, 0x2022, 0x00b6, 0x00df, + 0x00ae, 0x00a9, 0x2122, 0x00b4, 0x00a8, 0x2260, 0x00c6, 0x00d8, + 0x221e, 0x00b1, 0x2264, 0x2265, 0x00a5, 0x00b5, 0x2202, 0x2211, + 0x220f, 0x03c0, 0x222b, 0x00aa, 0x00ba, 0x03a9, 0x00e6, 0x00f8, + 0x00bf, 0x00a1, 0x00ac, 0x221a, 0x0192, 0x2248, 0x2206, 0x00ab, + 0x00bb, 0x2026, 0x00a0, 0x00c0, 0x00c3, 0x00d5, 0x0152, 0x0153, + 0x2013, 0x2014, 0x201c, 0x201d, 0x2018, 0x2019, 0x00f7, 0x25ca, + 0x00ff, 0x0178, 0x2044, 0x20ac, 0x2039, 0x203a, 0xfb01, 0xfb02, + 0x2021, 0x00b7, 0x201a, 0x201e, 0x2030, 0x00c2, 0x00ca, 0x00c1, + 0x00cb, 0x00c8, 0x00cd, 0x00ce, 0x00cf, 0x00cc, 0x00d3, 0x00d4, + 0xf8ff, 0x00d2, 0x00da, 0x00db, 0x00d9, 0x0131, 0x02c6, 0x02dc, + 0x00af, 0x02d8, 0x02d9, 0x02da, 0x00b8, 0x02dd, 0x02db, 0x02c7, +}; + +const SingleByteTable windows_874{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0x2026, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0x00a0, 0x0e01, 0x0e02, 0x0e03, 0x0e04, 0x0e05, 0x0e06, 0x0e07, + 0x0e08, 0x0e09, 0x0e0a, 0x0e0b, 0x0e0c, 0x0e0d, 0x0e0e, 0x0e0f, + 0x0e10, 0x0e11, 0x0e12, 0x0e13, 0x0e14, 0x0e15, 0x0e16, 0x0e17, + 0x0e18, 0x0e19, 0x0e1a, 0x0e1b, 0x0e1c, 0x0e1d, 0x0e1e, 0x0e1f, + 0x0e20, 0x0e21, 0x0e22, 0x0e23, 0x0e24, 0x0e25, 0x0e26, 0x0e27, + 0x0e28, 0x0e29, 0x0e2a, 0x0e2b, 0x0e2c, 0x0e2d, 0x0e2e, 0x0e2f, + 0x0e30, 0x0e31, 0x0e32, 0x0e33, 0x0e34, 0x0e35, 0x0e36, 0x0e37, + 0x0e38, 0x0e39, 0x0e3a, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0x0e3f, + 0x0e40, 0x0e41, 0x0e42, 0x0e43, 0x0e44, 0x0e45, 0x0e46, 0x0e47, + 0x0e48, 0x0e49, 0x0e4a, 0x0e4b, 0x0e4c, 0x0e4d, 0x0e4e, 0x0e4f, + 0x0e50, 0x0e51, 0x0e52, 0x0e53, 0x0e54, 0x0e55, 0x0e56, 0x0e57, + 0x0e58, 0x0e59, 0x0e5a, 0x0e5b, 0xfffd, 0xfffd, 0xfffd, 0xfffd, +}; + +const SingleByteTable windows_1250{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0x201a, 0xfffd, 0x201e, 0x2026, 0x2020, 0x2021, + 0xfffd, 0x2030, 0x0160, 0x2039, 0x015a, 0x0164, 0x017d, 0x0179, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0xfffd, 0x2122, 0x0161, 0x203a, 0x015b, 0x0165, 0x017e, 0x017a, + 0x00a0, 0x02c7, 0x02d8, 0x0141, 0x00a4, 0x0104, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x015e, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x017b, + 0x00b0, 0x00b1, 0x02db, 0x0142, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x0105, 0x015f, 0x00bb, 0x013d, 0x02dd, 0x013e, 0x017c, + 0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7, + 0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e, + 0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7, + 0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df, + 0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7, + 0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f, + 0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7, + 0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9, +}; + +const SingleByteTable windows_1251{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0402, 0x0403, 0x201a, 0x0453, 0x201e, 0x2026, 0x2020, 0x2021, + 0x20ac, 0x2030, 0x0409, 0x2039, 0x040a, 0x040c, 0x040b, 0x040f, + 0x0452, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0xfffd, 0x2122, 0x0459, 0x203a, 0x045a, 0x045c, 0x045b, 0x045f, + 0x00a0, 0x040e, 0x045e, 0x0408, 0x00a4, 0x0490, 0x00a6, 0x00a7, + 0x0401, 0x00a9, 0x0404, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x0407, + 0x00b0, 0x00b1, 0x0406, 0x0456, 0x0491, 0x00b5, 0x00b6, 0x00b7, + 0x0451, 0x2116, 0x0454, 0x00bb, 0x0458, 0x0405, 0x0455, 0x0457, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, +}; + +const SingleByteTable windows_1252{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, + 0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0xfffd, 0x017d, 0xfffd, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0xfffd, 0x017e, 0x0178, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff, +}; + +const SingleByteTable windows_1253{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, + 0xfffd, 0x2030, 0xfffd, 0x2039, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0xfffd, 0x2122, 0xfffd, 0x203a, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0x00a0, 0x0385, 0x0386, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0xfffd, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x2015, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x0384, 0x00b5, 0x00b6, 0x00b7, + 0x0388, 0x0389, 0x038a, 0x00bb, 0x038c, 0x00bd, 0x038e, 0x038f, + 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, + 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f, + 0x03a0, 0x03a1, 0xfffd, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7, + 0x03a8, 0x03a9, 0x03aa, 0x03ab, 0x03ac, 0x03ad, 0x03ae, 0x03af, + 0x03b0, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, + 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, + 0x03c0, 0x03c1, 0x03c2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, + 0x03c8, 0x03c9, 0x03ca, 0x03cb, 0x03cc, 0x03cd, 0x03ce, 0xfffd, +}; + +const SingleByteTable windows_1254{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, + 0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0xfffd, 0xfffd, 0x0178, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x011e, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x0130, 0x015e, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x011f, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x0131, 0x015f, 0x00ff, +}; + +const SingleByteTable windows_1255{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, + 0x02c6, 0x2030, 0xfffd, 0x2039, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0x02dc, 0x2122, 0xfffd, 0x203a, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x20aa, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00d7, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00f7, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x05b0, 0x05b1, 0x05b2, 0x05b3, 0x05b4, 0x05b5, 0x05b6, 0x05b7, + 0x05b8, 0x05b9, 0xfffd, 0x05bb, 0x05bc, 0x05bd, 0x05be, 0x05bf, + 0x05c0, 0x05c1, 0x05c2, 0x05c3, 0x05f0, 0x05f1, 0x05f2, 0x05f3, + 0x05f4, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0x05d7, + 0x05d8, 0x05d9, 0x05da, 0x05db, 0x05dc, 0x05dd, 0x05de, 0x05df, + 0x05e0, 0x05e1, 0x05e2, 0x05e3, 0x05e4, 0x05e5, 0x05e6, 0x05e7, + 0x05e8, 0x05e9, 0x05ea, 0xfffd, 0xfffd, 0x200e, 0x200f, 0xfffd, +}; + +const SingleByteTable windows_1256{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0x067e, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, + 0x02c6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688, + 0x06af, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0x06a9, 0x2122, 0x0691, 0x203a, 0x0153, 0x200c, 0x200d, 0x06ba, + 0x00a0, 0x060c, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x06be, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x061b, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x061f, + 0x06c1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, + 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f, + 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00d7, + 0x0637, 0x0638, 0x0639, 0x063a, 0x0640, 0x0641, 0x0642, 0x0643, + 0x00e0, 0x0644, 0x00e2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x0649, 0x064a, 0x00ee, 0x00ef, + 0x064b, 0x064c, 0x064d, 0x064e, 0x00f4, 0x064f, 0x0650, 0x00f7, + 0x0651, 0x00f9, 0x0652, 0x00fb, 0x00fc, 0x200e, 0x200f, 0x06d2, +}; + +const SingleByteTable windows_1257{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0x201a, 0xfffd, 0x201e, 0x2026, 0x2020, 0x2021, + 0xfffd, 0x2030, 0xfffd, 0x2039, 0xfffd, 0x00a8, 0x02c7, 0x00b8, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0xfffd, 0x2122, 0xfffd, 0x203a, 0xfffd, 0x00af, 0x02db, 0xfffd, + 0x00a0, 0xfffd, 0x00a2, 0x00a3, 0x00a4, 0xfffd, 0x00a6, 0x00a7, + 0x00d8, 0x00a9, 0x0156, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00c6, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00f8, 0x00b9, 0x0157, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00e6, + 0x0104, 0x012e, 0x0100, 0x0106, 0x00c4, 0x00c5, 0x0118, 0x0112, + 0x010c, 0x00c9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012a, 0x013b, + 0x0160, 0x0143, 0x0145, 0x00d3, 0x014c, 0x00d5, 0x00d6, 0x00d7, + 0x0172, 0x0141, 0x015a, 0x016a, 0x00dc, 0x017b, 0x017d, 0x00df, + 0x0105, 0x012f, 0x0101, 0x0107, 0x00e4, 0x00e5, 0x0119, 0x0113, + 0x010d, 0x00e9, 0x017a, 0x0117, 0x0123, 0x0137, 0x012b, 0x013c, + 0x0161, 0x0144, 0x0146, 0x00f3, 0x014d, 0x00f5, 0x00f6, 0x00f7, + 0x0173, 0x0142, 0x015b, 0x016b, 0x00fc, 0x017c, 0x017e, 0x02d9, +}; + +const SingleByteTable windows_1258{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, + 0x02c6, 0x2030, 0xfffd, 0x2039, 0x0152, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0x02dc, 0x2122, 0xfffd, 0x203a, 0x0153, 0xfffd, 0xfffd, 0x0178, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x0300, 0x00cd, 0x00ce, 0x00cf, + 0x0110, 0x00d1, 0x0309, 0x00d3, 0x00d4, 0x01a0, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x01af, 0x0303, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x0301, 0x00ed, 0x00ee, 0x00ef, + 0x0111, 0x00f1, 0x0323, 0x00f3, 0x00f4, 0x01a1, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x01b0, 0x20ab, 0x00ff, +}; + +const SingleByteTable x_mac_cyrillic{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, + 0x2020, 0x00b0, 0x0490, 0x00a3, 0x00a7, 0x2022, 0x00b6, 0x0406, + 0x00ae, 0x00a9, 0x2122, 0x0402, 0x0452, 0x2260, 0x0403, 0x0453, + 0x221e, 0x00b1, 0x2264, 0x2265, 0x0456, 0x00b5, 0x0491, 0x0408, + 0x0404, 0x0454, 0x0407, 0x0457, 0x0409, 0x0459, 0x040a, 0x045a, + 0x0458, 0x0405, 0x00ac, 0x221a, 0x0192, 0x2248, 0x2206, 0x00ab, + 0x00bb, 0x2026, 0x00a0, 0x040b, 0x045b, 0x040c, 0x045c, 0x0455, + 0x2013, 0x2014, 0x201c, 0x201d, 0x2018, 0x2019, 0x00f7, 0x201e, + 0x040e, 0x045e, 0x040f, 0x045f, 0x2116, 0x0401, 0x0451, 0x044f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x20ac, +}; + +} // namespace odr::internal::encoding::encoding_data diff --git a/src/odr/internal/encoding/encoding_data.hpp b/src/odr/internal/encoding/encoding_data.hpp new file mode 100644 index 000000000..567f49f30 --- /dev/null +++ b/src/odr/internal/encoding/encoding_data.hpp @@ -0,0 +1,46 @@ +// Auto-generated by tools/encoding/generate_encoding_data.py -- DO NOT EDIT. +// Regenerate with: python3 tools/encoding/generate_encoding_data.py +// Data: the Python standard library's codecs. + +// clang-format off + +#pragma once + +#include + +namespace odr::internal::encoding::encoding_data { + +/// Byte -> code point for one single-byte encoding; U+FFFD where the +/// encoding leaves the byte undefined. +using SingleByteTable = std::array; + +extern const SingleByteTable ibm866; +extern const SingleByteTable iso_8859_1; +extern const SingleByteTable iso_8859_2; +extern const SingleByteTable iso_8859_3; +extern const SingleByteTable iso_8859_4; +extern const SingleByteTable iso_8859_5; +extern const SingleByteTable iso_8859_6; +extern const SingleByteTable iso_8859_7; +extern const SingleByteTable iso_8859_8; +extern const SingleByteTable iso_8859_10; +extern const SingleByteTable iso_8859_13; +extern const SingleByteTable iso_8859_14; +extern const SingleByteTable iso_8859_15; +extern const SingleByteTable iso_8859_16; +extern const SingleByteTable koi8_r; +extern const SingleByteTable koi8_u; +extern const SingleByteTable macintosh; +extern const SingleByteTable windows_874; +extern const SingleByteTable windows_1250; +extern const SingleByteTable windows_1251; +extern const SingleByteTable windows_1252; +extern const SingleByteTable windows_1253; +extern const SingleByteTable windows_1254; +extern const SingleByteTable windows_1255; +extern const SingleByteTable windows_1256; +extern const SingleByteTable windows_1257; +extern const SingleByteTable windows_1258; +extern const SingleByteTable x_mac_cyrillic; + +} // namespace odr::internal::encoding::encoding_data diff --git a/src/odr/internal/encoding/text_encoding_table.cpp b/src/odr/internal/encoding/text_encoding_table.cpp new file mode 100644 index 000000000..26ec513fc --- /dev/null +++ b/src/odr/internal/encoding/text_encoding_table.cpp @@ -0,0 +1,159 @@ +#include + +#include +#include +#include +#include + +namespace odr::internal::encoding { + +namespace { + +using text_encoding_table::Row; + +using namespace std::string_view_literals; + +// Names per encoding, canonical one first — it goes into an html +// ``, so it has to be a label a browser knows. Aliases are unique +// across encodings; `odr_test` asserts no name appears twice. + +constexpr std::array utf8_names{"UTF-8"sv, "ASCII"sv, "US-ASCII"sv}; +// a bare `UTF-16`/`UTF-32` only reaches here without a mark, where little +// endian is the likelier intent +constexpr std::array utf16le_names{"UTF-16LE"sv, "UTF-16"sv}; +constexpr std::array utf16be_names{"UTF-16BE"sv}; +constexpr std::array utf32le_names{"UTF-32LE"sv, "UTF-32"sv}; +constexpr std::array utf32be_names{"UTF-32BE"sv}; + +constexpr std::array ibm866_names{"IBM866"sv, "cp866"sv, "866"sv}; +constexpr std::array iso_8859_1_names{"ISO-8859-1"sv, "latin1"sv, "l1"sv, + "iso-ir-100"sv, "cp819"sv}; +constexpr std::array iso_8859_2_names{"ISO-8859-2"sv, "latin2"sv, "l2"sv}; +constexpr std::array iso_8859_3_names{"ISO-8859-3"sv, "latin3"sv, "l3"sv}; +constexpr std::array iso_8859_4_names{"ISO-8859-4"sv, "latin4"sv, "l4"sv}; +constexpr std::array iso_8859_5_names{"ISO-8859-5"sv, "cyrillic"sv}; +constexpr std::array iso_8859_6_names{"ISO-8859-6"sv, "arabic"sv}; +constexpr std::array iso_8859_7_names{"ISO-8859-7"sv, "greek"sv}; +constexpr std::array iso_8859_8_names{"ISO-8859-8"sv, "hebrew"sv}; +constexpr std::array iso_8859_10_names{"ISO-8859-10"sv, "latin6"sv, "l6"sv}; +constexpr std::array iso_8859_13_names{"ISO-8859-13"sv}; +constexpr std::array iso_8859_14_names{"ISO-8859-14"sv, "latin8"sv, "l8"sv}; +constexpr std::array iso_8859_15_names{"ISO-8859-15"sv, "latin9"sv, "l9"sv}; +constexpr std::array iso_8859_16_names{"ISO-8859-16"sv, "latin10"sv, "l10"sv}; +constexpr std::array koi8_r_names{"KOI8-R"sv, "cskoi8r"sv}; +constexpr std::array koi8_u_names{"KOI8-U"sv}; +constexpr std::array macintosh_names{"macintosh"sv, "mac"sv, "x-mac-roman"sv, + "macroman"sv}; +constexpr std::array windows_874_names{"windows-874"sv, "cp874"sv, "TIS-620"sv, + "ISO-8859-11"sv}; +constexpr std::array windows_1250_names{"windows-1250"sv, "cp1250"sv, + "x-cp1250"sv}; +constexpr std::array windows_1251_names{"windows-1251"sv, "cp1251"sv, + "x-cp1251"sv}; +constexpr std::array windows_1252_names{"windows-1252"sv, "cp1252"sv, + "x-cp1252"sv, "ansi"sv}; +constexpr std::array windows_1253_names{"windows-1253"sv, "cp1253"sv}; +constexpr std::array windows_1254_names{"windows-1254"sv, "cp1254"sv}; +constexpr std::array windows_1255_names{"windows-1255"sv, "cp1255"sv}; +constexpr std::array windows_1256_names{"windows-1256"sv, "cp1256"sv}; +constexpr std::array windows_1257_names{"windows-1257"sv, "cp1257"sv}; +constexpr std::array windows_1258_names{"windows-1258"sv, "cp1258"sv}; +constexpr std::array x_mac_cyrillic_names{"x-mac-cyrillic"sv, "mac-cyrillic"sv}; + +constexpr std::array big5_names{"Big5"sv, "csbig5"sv, "big5-hkscs"sv, + "cn-big5"sv}; +constexpr std::array euc_jp_names{"EUC-JP"sv, "cseucpkdfmtjapanese"sv}; +constexpr std::array euc_kr_names{"EUC-KR"sv, "windows-949"sv, "cp949"sv, + "ks_c_5601-1987"sv}; +constexpr std::array gb18030_names{"gb18030"sv, "GBK"sv, "GB2312"sv, + "csgb2312"sv, "x-gbk"sv}; +constexpr std::array iso_2022_jp_names{"ISO-2022-JP"sv, "csiso2022jp"sv}; +constexpr std::array iso_2022_kr_names{"ISO-2022-KR"sv, "csiso2022kr"sv}; +constexpr std::array shift_jis_names{"Shift_JIS"sv, "sjis"sv, "windows-31j"sv, + "cp932"sv, "ms_kanji"sv}; + +constexpr std::array table{ + Row{TextEncoding::utf8, utf8_names, true}, + Row{TextEncoding::utf16le, utf16le_names, true}, + Row{TextEncoding::utf16be, utf16be_names, true}, + Row{TextEncoding::utf32le, utf32le_names, true}, + Row{TextEncoding::utf32be, utf32be_names, true}, + + Row{TextEncoding::ibm866, ibm866_names, true}, + Row{TextEncoding::iso_8859_1, iso_8859_1_names, true}, + Row{TextEncoding::iso_8859_2, iso_8859_2_names, true}, + Row{TextEncoding::iso_8859_3, iso_8859_3_names, true}, + Row{TextEncoding::iso_8859_4, iso_8859_4_names, true}, + Row{TextEncoding::iso_8859_5, iso_8859_5_names, true}, + Row{TextEncoding::iso_8859_6, iso_8859_6_names, true}, + Row{TextEncoding::iso_8859_7, iso_8859_7_names, true}, + Row{TextEncoding::iso_8859_8, iso_8859_8_names, true}, + Row{TextEncoding::iso_8859_10, iso_8859_10_names, true}, + Row{TextEncoding::iso_8859_13, iso_8859_13_names, true}, + Row{TextEncoding::iso_8859_14, iso_8859_14_names, true}, + Row{TextEncoding::iso_8859_15, iso_8859_15_names, true}, + Row{TextEncoding::iso_8859_16, iso_8859_16_names, true}, + Row{TextEncoding::koi8_r, koi8_r_names, true}, + Row{TextEncoding::koi8_u, koi8_u_names, true}, + Row{TextEncoding::macintosh, macintosh_names, true}, + Row{TextEncoding::windows_874, windows_874_names, true}, + Row{TextEncoding::windows_1250, windows_1250_names, true}, + Row{TextEncoding::windows_1251, windows_1251_names, true}, + Row{TextEncoding::windows_1252, windows_1252_names, true}, + Row{TextEncoding::windows_1253, windows_1253_names, true}, + Row{TextEncoding::windows_1254, windows_1254_names, true}, + Row{TextEncoding::windows_1255, windows_1255_names, true}, + Row{TextEncoding::windows_1256, windows_1256_names, true}, + Row{TextEncoding::windows_1257, windows_1257_names, true}, + Row{TextEncoding::windows_1258, windows_1258_names, true}, + Row{TextEncoding::x_mac_cyrillic, x_mac_cyrillic_names, true}, + + // decoding these needs a real converter; naming them still lets a browser + // do it + Row{TextEncoding::big5, big5_names, false}, + Row{TextEncoding::euc_jp, euc_jp_names, false}, + Row{TextEncoding::euc_kr, euc_kr_names, false}, + Row{TextEncoding::gb18030, gb18030_names, false}, + Row{TextEncoding::iso_2022_jp, iso_2022_jp_names, false}, + Row{TextEncoding::iso_2022_kr, iso_2022_kr_names, false}, + Row{TextEncoding::shift_jis, shift_jis_names, false}, +}; + +/// Case, `-`, `_` and spaces carry no meaning in an encoding label. +std::string normalize(const std::string_view name) { + std::string result; + result.reserve(name.size()); + for (const char c : name) { + if (c == '-' || c == '_' || c == ' ') { + continue; + } + result.push_back( + static_cast(std::tolower(static_cast(c)))); + } + return result; +} + +} // namespace + +std::span text_encoding_table::rows() noexcept { + return table; +} + +const text_encoding_table::Row * +text_encoding_table::find(const TextEncoding encoding) noexcept { + const auto it = std::ranges::find(table, encoding, &Row::encoding); + return it == std::ranges::end(table) ? nullptr : &*it; +} + +const text_encoding_table::Row * +text_encoding_table::find_by_name(const std::string_view name) noexcept { + const std::string needle = normalize(name); + const auto it = std::ranges::find_if(table, [&](const Row &row) { + return std::ranges::any_of(row.names, [&](const std::string_view alias) { + return normalize(alias) == needle; + }); + }); + return it == std::ranges::end(table) ? nullptr : &*it; +} + +} // namespace odr::internal::encoding diff --git a/src/odr/internal/encoding/text_encoding_table.hpp b/src/odr/internal/encoding/text_encoding_table.hpp new file mode 100644 index 000000000..81d518a9a --- /dev/null +++ b/src/odr/internal/encoding/text_encoding_table.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include +#include + +namespace odr::internal::encoding::text_encoding_table { + +/// Everything the library statically knows about one text encoding. +struct Row final { + TextEncoding encoding{TextEncoding::unknown}; + std::span names; ///< canonical one first + bool decodable{}; ///< @ref encoding::to_utf8 handles it +}; + +/// The whole table, one row per @ref TextEncoding, in declaration order. +[[nodiscard]] std::span rows() noexcept; + +/// The row for @p encoding, or `nullptr` if there is none. +[[nodiscard]] const Row *find(TextEncoding encoding) noexcept; + +/// The row listing @p name among its names, or `nullptr`. Matching ignores +/// case and any `-`, `_` or space, so `windows-1252`, `WINDOWS_1252` and +/// `cp1252` all land on the same row. +[[nodiscard]] const Row *find_by_name(std::string_view name) noexcept; + +} // namespace odr::internal::encoding::text_encoding_table diff --git a/src/odr/internal/encoding/transcode.cpp b/src/odr/internal/encoding/transcode.cpp new file mode 100644 index 000000000..a514cf6ca --- /dev/null +++ b/src/odr/internal/encoding/transcode.cpp @@ -0,0 +1,230 @@ +#include + +#include +#include + +#include +#include +#include + +#include +#include + +namespace odr::internal { + +namespace { + +using encoding::encoding_data::SingleByteTable; + +using namespace std::string_view_literals; + +constexpr char32_t replacement = 0xfffd; + +constexpr auto utf8_bom = "\xef\xbb\xbf"sv; +constexpr auto utf16le_bom = "\xff\xfe"sv; +constexpr auto utf16be_bom = "\xfe\xff"sv; +constexpr auto utf32le_bom = "\xff\xfe\x00\x00"sv; +constexpr auto utf32be_bom = "\x00\x00\xfe\xff"sv; + +bool starts_with(const std::string_view bytes, const std::string_view bom) { + return bytes.substr(0, bom.size()) == bom; +} + +const SingleByteTable *single_byte_table(const TextEncoding encoding) { + namespace data = encoding::encoding_data; + + switch (encoding) { + case TextEncoding::ibm866: + return &data::ibm866; + case TextEncoding::iso_8859_1: + return &data::iso_8859_1; + case TextEncoding::iso_8859_2: + return &data::iso_8859_2; + case TextEncoding::iso_8859_3: + return &data::iso_8859_3; + case TextEncoding::iso_8859_4: + return &data::iso_8859_4; + case TextEncoding::iso_8859_5: + return &data::iso_8859_5; + case TextEncoding::iso_8859_6: + return &data::iso_8859_6; + case TextEncoding::iso_8859_7: + return &data::iso_8859_7; + case TextEncoding::iso_8859_8: + return &data::iso_8859_8; + case TextEncoding::iso_8859_10: + return &data::iso_8859_10; + case TextEncoding::iso_8859_13: + return &data::iso_8859_13; + case TextEncoding::iso_8859_14: + return &data::iso_8859_14; + case TextEncoding::iso_8859_15: + return &data::iso_8859_15; + case TextEncoding::iso_8859_16: + return &data::iso_8859_16; + case TextEncoding::koi8_r: + return &data::koi8_r; + case TextEncoding::koi8_u: + return &data::koi8_u; + case TextEncoding::macintosh: + return &data::macintosh; + case TextEncoding::windows_874: + return &data::windows_874; + case TextEncoding::windows_1250: + return &data::windows_1250; + case TextEncoding::windows_1251: + return &data::windows_1251; + case TextEncoding::windows_1252: + return &data::windows_1252; + case TextEncoding::windows_1253: + return &data::windows_1253; + case TextEncoding::windows_1254: + return &data::windows_1254; + case TextEncoding::windows_1255: + return &data::windows_1255; + case TextEncoding::windows_1256: + return &data::windows_1256; + case TextEncoding::windows_1257: + return &data::windows_1257; + case TextEncoding::windows_1258: + return &data::windows_1258; + case TextEncoding::x_mac_cyrillic: + return &data::x_mac_cyrillic; + default: + return nullptr; + } +} + +/// The mark @p encoding defines, empty for the rest: `EF BB BF` read as +/// windows-1252 is three characters, not a mark. +std::string_view own_bom(const TextEncoding encoding) { + switch (encoding) { + case TextEncoding::utf8: + return utf8_bom; + case TextEncoding::utf16le: + return utf16le_bom; + case TextEncoding::utf16be: + return utf16be_bom; + case TextEncoding::utf32le: + return utf32le_bom; + case TextEncoding::utf32be: + return utf32be_bom; + default: + return {}; + } +} + +bool is_code_point_valid(const char32_t code_point) { + return code_point <= 0x10ffff && (code_point < 0xd800 || code_point > 0xdfff); +} + +std::uint8_t byte_at(const std::string_view bytes, const std::size_t index) { + return static_cast(bytes[index]); +} + +std::string decode_single_byte(const std::string_view bytes, + const SingleByteTable &table) { + std::string result; + result.reserve(bytes.size()); + for (std::size_t i = 0; i < bytes.size(); ++i) { + utf8::append(table[byte_at(bytes, i)], std::back_inserter(result)); + } + return result; +} + +std::string decode_utf16(const std::string_view bytes, const bool little) { + std::string result; + result.reserve(bytes.size()); + + const auto unit = [&](const std::size_t i) -> char32_t { + const auto low = byte_at(bytes, little ? i : i + 1); + const auto high = byte_at(bytes, little ? i + 1 : i); + return static_cast(high) << 8 | low; + }; + + std::size_t i = 0; + for (; i + 1 < bytes.size(); i += 2) { + const char32_t first = unit(i); + + if (first < 0xd800 || first > 0xdfff) { + utf8::append(first, std::back_inserter(result)); + continue; + } + // a low surrogate first, or a high one with nothing to pair with + if (first > 0xdbff || i + 3 >= bytes.size()) { + utf8::append(replacement, std::back_inserter(result)); + continue; + } + const char32_t second = unit(i + 2); + if (second < 0xdc00 || second > 0xdfff) { + utf8::append(replacement, std::back_inserter(result)); + continue; + } + utf8::append(0x10000 + ((first - 0xd800) << 10) + (second - 0xdc00), + std::back_inserter(result)); + i += 2; + } + + // a trailing odd byte cannot form a unit + if (i < bytes.size()) { + utf8::append(replacement, std::back_inserter(result)); + } + return result; +} + +std::string decode_utf32(const std::string_view bytes, const bool little) { + std::string result; + result.reserve(bytes.size() / 2); + + std::size_t i = 0; + for (; i + 3 < bytes.size(); i += 4) { + char32_t code_point = 0; + for (std::size_t b = 0; b < 4; ++b) { + code_point = code_point << 8 | byte_at(bytes, i + (little ? 3 - b : b)); + } + utf8::append(is_code_point_valid(code_point) ? code_point : replacement, + std::back_inserter(result)); + } + + if (i < bytes.size()) { + utf8::append(replacement, std::back_inserter(result)); + } + return result; +} + +} // namespace + +std::string encoding::to_utf8(const std::string_view bytes, + const TextEncoding encoding) { + const auto *row = text_encoding_table::find(encoding); + if (row == nullptr || !row->decodable) { + throw std::runtime_error("text encoding cannot be decoded"); + } + + const std::string_view bom = own_bom(encoding); + const std::string_view body = + starts_with(bytes, bom) ? bytes.substr(bom.size()) : bytes; + + switch (encoding) { + case TextEncoding::utf8: + return utf8::replace_invalid(body); + case TextEncoding::utf16le: + return decode_utf16(body, true); + case TextEncoding::utf16be: + return decode_utf16(body, false); + case TextEncoding::utf32le: + return decode_utf32(body, true); + case TextEncoding::utf32be: + return decode_utf32(body, false); + default: + break; + } + + const SingleByteTable *table = single_byte_table(encoding); + if (table == nullptr) { + throw std::runtime_error("text encoding cannot be decoded"); + } + return decode_single_byte(body, *table); +} + +} // namespace odr::internal diff --git a/src/odr/internal/encoding/transcode.hpp b/src/odr/internal/encoding/transcode.hpp new file mode 100644 index 000000000..809076458 --- /dev/null +++ b/src/odr/internal/encoding/transcode.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include + +#include +#include + +namespace odr::internal::encoding { + +/// Decodes @p bytes to UTF-8. Malformed input becomes U+FFFD rather than an +/// error — a wrong guess is the expected failure here, and the caller +/// overrides the encoding. +/// +/// @throws std::runtime_error if @p encoding is not decodable. +[[nodiscard]] std::string to_utf8(std::string_view bytes, + TextEncoding encoding); + +} // namespace odr::internal::encoding diff --git a/src/odr/internal/html/text_file.cpp b/src/odr/internal/html/text_file.cpp index 4ef8168f1..9b43ea4bc 100644 --- a/src/odr/internal/html/text_file.cpp +++ b/src/odr/internal/html/text_file.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -60,15 +61,30 @@ class HtmlServiceImpl final : public HtmlService { throw FileNotFound("Unknown path: " + path); } + /// The bytes to write and the charset to declare. What we can decode becomes + /// UTF-8; what we can only name passes through for the browser, which works + /// because those encodings are ASCII-compatible and the page is ASCII. + [[nodiscard]] std::pair body_and_charset() const { + const TextEncoding encoding = m_text_file.encoding(); + std::string text = m_text_file.text(); + + if (encoding == TextEncoding::unknown || + text_encoding_is_decodable(encoding)) { + return {std::move(text), "UTF-8"}; + } + return {std::move(text), std::string(text_encoding_to_string(encoding))}; + } + HtmlResources write_text(HtmlWriter &out) const { HtmlResources resources; + const auto [text, charset] = body_and_charset(); + out.write_begin(); out.write_header_begin(); - // TODO charset - out.write_header_charset("UTF-8"); + out.write_header_charset(charset); out.write_header_target("_blank"); out.write_header_title("odr"); write_viewport_meta(out, config(), false); @@ -83,14 +99,14 @@ class HtmlServiceImpl final : public HtmlService { out.write_element_begin("div", HtmlElementOptions().set_class("odr-text-nr")); - std::unique_ptr in = m_text_file.stream(); - for (std::uint32_t line = 1; !in->eof(); ++line) { + std::istringstream in(text); + for (std::uint32_t line = 1; !in.eof(); ++line) { out.write_element_begin("div", HtmlElementOptions().set_inline(true)); out.out() << line; out.write_element_end("div"); NullStream ss_out; - util::stream::pipe_line(*in, ss_out, false); + util::stream::pipe_line(in, ss_out, false); } out.write_element_end("div"); @@ -102,12 +118,12 @@ class HtmlServiceImpl final : public HtmlService { clb("contenteditable", "true"); } })); - in = m_text_file.stream(); - while (!in->eof()) { + in = std::istringstream(text); + while (!in.eof()) { out.write_element_begin("div", HtmlElementOptions().set_inline(true)); std::ostringstream ss_out; - util::stream::pipe_line(*in, ss_out, false); + util::stream::pipe_line(in, ss_out, false); if (std::string line = ss_out.str(); line.empty()) { out.write_element_begin( "br", HtmlElementOptions().set_close_type(HtmlCloseType::trailing)); diff --git a/src/odr/internal/json/json_file.cpp b/src/odr/internal/json/json_file.cpp index bfcbf4bd6..e9f7a2add 100644 --- a/src/odr/internal/json/json_file.cpp +++ b/src/odr/internal/json/json_file.cpp @@ -31,4 +31,6 @@ FileMeta JsonFile::file_meta() const noexcept { bool JsonFile::is_decodable() const noexcept { return false; } +TextEncoding JsonFile::encoding() const noexcept { return m_file->encoding(); } + } // namespace odr::internal::json diff --git a/src/odr/internal/json/json_file.hpp b/src/odr/internal/json/json_file.hpp index 1ef4889a8..c9c52ad9d 100644 --- a/src/odr/internal/json/json_file.hpp +++ b/src/odr/internal/json/json_file.hpp @@ -20,9 +20,10 @@ class JsonFile final : public abstract::TextFile { [[nodiscard]] bool is_decodable() const noexcept override; + [[nodiscard]] TextEncoding encoding() const noexcept override; + private: std::shared_ptr m_file; - std::string m_charset; }; } // namespace odr::internal::json diff --git a/src/odr/internal/text/text_file.cpp b/src/odr/internal/text/text_file.cpp index 7e8913cc9..69b91b193 100644 --- a/src/odr/internal/text/text_file.cpp +++ b/src/odr/internal/text/text_file.cpp @@ -1,14 +1,28 @@ #include -#include +#include +#include + +#include +#include +#include + +#include namespace odr::internal::text { TextFile::TextFile(std::shared_ptr file) - : m_file{std::move(file)}, m_charset{guess_charset(*m_file->stream())} {} + : m_file{std::move(file)} { + const std::unique_ptr in = m_file->stream(); + // nothing here rejects: text is the fallback for bytes nothing else claims, + // so a viewer handed a random binary shows junk rather than an error, and + // bytes we cannot name are `TextEncoding::unknown` + m_encoding = encoding::detect(encoding::read_probe(*in)); +} -TextFile::TextFile(std::shared_ptr file, std::string charset) - : m_file{std::move(file)}, m_charset{std::move(charset)} {} +TextFile::TextFile(std::shared_ptr file, + const TextEncoding encoding) + : m_file{std::move(file)}, m_encoding{encoding} {} std::shared_ptr TextFile::file() const noexcept { return m_file; @@ -27,4 +41,14 @@ FileMeta TextFile::file_meta() const noexcept { bool TextFile::is_decodable() const noexcept { return false; } +TextEncoding TextFile::encoding() const noexcept { return m_encoding; } + +std::string TextFile::text() const { + if (!text_encoding_is_decodable(m_encoding)) { + throw UnsupportedTextEncoding(m_encoding); + } + const std::unique_ptr in = m_file->stream(); + return encoding::to_utf8(util::stream::read(*in), m_encoding); +} + } // namespace odr::internal::text diff --git a/src/odr/internal/text/text_file.hpp b/src/odr/internal/text/text_file.hpp index ff77487a1..0111a6956 100644 --- a/src/odr/internal/text/text_file.hpp +++ b/src/odr/internal/text/text_file.hpp @@ -11,7 +11,7 @@ namespace odr::internal::text { class TextFile final : public abstract::TextFile { public: explicit TextFile(std::shared_ptr file); - TextFile(std::shared_ptr file, std::string charset); + TextFile(std::shared_ptr file, TextEncoding encoding); [[nodiscard]] std::shared_ptr file() const noexcept override; @@ -21,9 +21,15 @@ class TextFile final : public abstract::TextFile { [[nodiscard]] bool is_decodable() const noexcept override; + [[nodiscard]] TextEncoding encoding() const noexcept override; + + /// The file's bytes decoded to UTF-8. + /// @throws UnsupportedTextEncoding if the encoding cannot be decoded. + [[nodiscard]] std::string text() const; + private: std::shared_ptr m_file; - std::string m_charset; + TextEncoding m_encoding{TextEncoding::unknown}; }; } // namespace odr::internal::text diff --git a/src/odr/internal/text/text_util.cpp b/src/odr/internal/text/text_util.cpp deleted file mode 100644 index 68c4499fd..000000000 --- a/src/odr/internal/text/text_util.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include - -#include - -#include -#include - -#include - -namespace odr::internal { - -std::string text::guess_charset(std::istream &in) { - const auto ud = uchardet_new(); - std::array buffer{}; - - while (true) { - in.read(buffer.data(), static_cast(buffer.size())); - const auto read = in.gcount(); - if (read == 0) { - break; - } - uchardet_handle_data(ud, buffer.data(), read); - } - - uchardet_data_end(ud); - std::string result = uchardet_get_charset(ud); - uchardet_delete(ud); - - if (result.empty()) { - throw UnknownCharset(); - } - return result; -} - -} // namespace odr::internal diff --git a/src/odr/internal/text/text_util.hpp b/src/odr/internal/text/text_util.hpp deleted file mode 100644 index 64d9f475b..000000000 --- a/src/odr/internal/text/text_util.hpp +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include -#include - -namespace odr::internal::text { - -// TODO pass max read distance -std::string guess_charset(std::istream &in); - -} // namespace odr::internal::text diff --git a/src/odr/odr.cpp b/src/odr/odr.cpp index ff026fd1b..247d80ab5 100644 --- a/src/odr/odr.cpp +++ b/src/odr/odr.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,9 @@ namespace { using odr::internal::file_type_table::Row; namespace table = odr::internal::file_type_table; + +using EncodingRow = odr::internal::encoding::text_encoding_table::Row; +namespace encoding_table = odr::internal::encoding::text_encoding_table; } // namespace std::string odr::version() { @@ -145,6 +149,39 @@ odr::capabilities_by_file_type(const FileType type) noexcept { return row == nullptr ? FileTypeCapabilities{} : row->capabilities; } +std::vector odr::all_text_encodings() { + std::vector result; + result.reserve(encoding_table::rows().size()); + std::ranges::transform(encoding_table::rows(), std::back_inserter(result), + &EncodingRow::encoding); + return result; +} + +std::string_view odr::text_encoding_to_string(const TextEncoding encoding) { + const EncodingRow *row = encoding_table::find(encoding); + if (row == nullptr) { + throw UnsupportedTextEncoding(encoding); + } + return row->names.front(); +} + +odr::TextEncoding +odr::text_encoding_by_name(const std::string_view name) noexcept { + const EncodingRow *row = encoding_table::find_by_name(name); + return row == nullptr ? TextEncoding::unknown : row->encoding; +} + +std::span +odr::text_encoding_names(const TextEncoding encoding) noexcept { + const EncodingRow *row = encoding_table::find(encoding); + return row == nullptr ? std::span{} : row->names; +} + +bool odr::text_encoding_is_decodable(const TextEncoding encoding) noexcept { + const EncodingRow *row = encoding_table::find(encoding); + return row != nullptr && row->decodable; +} + std::vector odr::list_file_types(const File &file, const Logger &logger) { return DecodedFile::list_file_types(file, logger); diff --git a/src/odr/odr.hpp b/src/odr/odr.hpp index 9b3a62616..6291883df 100644 --- a/src/odr/odr.hpp +++ b/src/odr/odr.hpp @@ -64,6 +64,26 @@ mimetypes_by_file_type(FileType type) noexcept; [[nodiscard]] FileTypeCapabilities capabilities_by_file_type(FileType type) noexcept; +/// @brief Every text encoding this library knows about, in declaration order, +/// excluding @ref TextEncoding::unknown. +[[nodiscard]] std::vector all_text_encodings(); + +/// @brief The text encoding's canonical name, a label a browser accepts. +/// @throws UnsupportedTextEncoding for @ref TextEncoding::unknown. +[[nodiscard]] std::string_view text_encoding_to_string(TextEncoding encoding); +/// @brief The text encoding for a name, @ref TextEncoding::unknown if none. +/// +/// Case and any `-`, `_` or space are ignored, so `windows-1252`, +/// `WINDOWS_1252` and `cp1252` all name the same encoding. +[[nodiscard]] TextEncoding +text_encoding_by_name(std::string_view name) noexcept; +/// @brief Every name accepted for the text encoding, canonical one first. +[[nodiscard]] std::span +text_encoding_names(TextEncoding encoding) noexcept; +/// @brief Whether the library can decode the text encoding, as opposed to +/// merely naming it. +[[nodiscard]] bool text_encoding_is_decodable(TextEncoding encoding) noexcept; + /// @brief The file types detected for @p file. [[nodiscard]] std::vector list_file_types(const File &file, const Logger &logger = Logger::null()); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7abfe850b..66be0c470 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -52,6 +52,7 @@ add_executable(odr_test "src/internal/crypto/crypto_util_test.cpp" "src/internal/csv/csv_file_test.cpp" + "src/internal/encoding/text_encoding_test.cpp" "src/internal/oldms/doc_test.cpp" "src/internal/oldms/ppt_test.cpp" diff --git a/test/src/internal/encoding/text_encoding_test.cpp b/test/src/internal/encoding/text_encoding_test.cpp new file mode 100644 index 000000000..cf3e76d5e --- /dev/null +++ b/test/src/internal/encoding/text_encoding_test.cpp @@ -0,0 +1,150 @@ +#include +#include +#include + +#include +#include + +#include + +#include +#include + +using namespace odr; +using namespace odr::internal; + +TEST(TextEncoding, every_encoding_has_a_name) { + for (const TextEncoding text_encoding : all_text_encodings()) { + EXPECT_FALSE(text_encoding_to_string(text_encoding).empty()); + EXPECT_FALSE(text_encoding_names(text_encoding).empty()); + } +} + +TEST(TextEncoding, unknown_has_no_name) { + EXPECT_THROW((void)text_encoding_to_string(TextEncoding::unknown), + UnsupportedTextEncoding); + EXPECT_TRUE(text_encoding_names(TextEncoding::unknown).empty()); + EXPECT_FALSE(text_encoding_is_decodable(TextEncoding::unknown)); +} + +/// The lookups take the first match, so a name on two encodings would make one +/// of them unreachable. +TEST(TextEncoding, no_name_is_claimed_twice) { + std::set seen; + for (const TextEncoding text_encoding : all_text_encodings()) { + for (const std::string_view name : text_encoding_names(text_encoding)) { + std::string normalized; + for (const char c : name) { + if (c != '-' && c != '_') { + normalized.push_back( + static_cast(std::tolower(static_cast(c)))); + } + } + EXPECT_TRUE(seen.insert(normalized).second) + << "name claimed twice: " << name; + } + } +} + +TEST(TextEncoding, every_name_finds_its_encoding) { + for (const TextEncoding text_encoding : all_text_encodings()) { + for (const std::string_view name : text_encoding_names(text_encoding)) { + EXPECT_EQ(text_encoding_by_name(name), text_encoding) << name; + } + } +} + +TEST(TextEncoding, a_name_matches_regardless_of_case_and_punctuation) { + EXPECT_EQ(text_encoding_by_name("windows-1252"), TextEncoding::windows_1252); + EXPECT_EQ(text_encoding_by_name("WINDOWS_1252"), TextEncoding::windows_1252); + EXPECT_EQ(text_encoding_by_name("cp1252"), TextEncoding::windows_1252); + EXPECT_EQ(text_encoding_by_name("utf8"), TextEncoding::utf8); + EXPECT_EQ(text_encoding_by_name("Latin1"), TextEncoding::iso_8859_1); + EXPECT_EQ(text_encoding_by_name("nonsense"), TextEncoding::unknown); +} + +TEST(TextEncoding, uchardet_names_map_home) { + // the spellings uchardet reports, which are what `detect` has to resolve + for (const std::string_view name : + {"UTF-8", "ASCII", "ISO-8859-1", "WINDOWS-1252", "KOI8-R", "SHIFT_JIS", + "EUC-JP", "Big5", "IBM866", "TIS-620"}) { + EXPECT_NE(text_encoding_by_name(name), TextEncoding::unknown) << name; + } +} + +TEST(Transcode, single_byte) { + EXPECT_EQ(encoding::to_utf8("caf\xe9", TextEncoding::iso_8859_1), "café"); + EXPECT_EQ(encoding::to_utf8("caf\xe9", TextEncoding::windows_1252), "café"); + // the euro sits where latin-1 has nothing + EXPECT_EQ(encoding::to_utf8("\x80", TextEncoding::windows_1252), "€"); + EXPECT_EQ(encoding::to_utf8("\xa4", TextEncoding::iso_8859_15), "€"); +} + +/// windows-1252 leaves a handful of codes undefined; they are not an error. +TEST(Transcode, an_undefined_byte_becomes_the_replacement_character) { + EXPECT_EQ(encoding::to_utf8("\x81", TextEncoding::windows_1252), "�"); +} + +TEST(Transcode, utf8_passes_through_and_repairs) { + EXPECT_EQ(encoding::to_utf8("café", TextEncoding::utf8), "café"); + EXPECT_EQ(encoding::to_utf8("\xef\xbb\xbfhi", TextEncoding::utf8), "hi"); + EXPECT_EQ(encoding::to_utf8("a\xff" + "b", + TextEncoding::utf8), + "a�b"); +} + +TEST(Transcode, utf16) { + EXPECT_EQ(encoding::to_utf8(std::string("h\0i\0", 4), TextEncoding::utf16le), + "hi"); + EXPECT_EQ(encoding::to_utf8(std::string("\0h\0i", 4), TextEncoding::utf16be), + "hi"); + EXPECT_EQ(encoding::to_utf8(std::string("\xff\xfeh\0i\0", 6), + TextEncoding::utf16le), + "hi"); + // a surrogate pair, U+1F600 + EXPECT_EQ(encoding::to_utf8(std::string("\x3d\xd8\x00\xde", 4), + TextEncoding::utf16le), + "😀"); +} + +TEST(Transcode, a_broken_utf16_unit_becomes_the_replacement_character) { + // an unpaired high surrogate, and a trailing odd byte + EXPECT_EQ( + encoding::to_utf8(std::string("\x3d\xd8h\0", 4), TextEncoding::utf16le), + "�h"); + EXPECT_EQ(encoding::to_utf8(std::string("h\0!", 3), TextEncoding::utf16le), + "h�"); +} + +TEST(Transcode, utf32) { + EXPECT_EQ(encoding::to_utf8(std::string("h\0\0\0i\0\0\0", 8), + TextEncoding::utf32le), + "hi"); + EXPECT_EQ(encoding::to_utf8(std::string("\0\0\0h\0\0\0i", 8), + TextEncoding::utf32be), + "hi"); +} + +/// A mark is only a mark for the encoding that defines it — read as +/// windows-1252 those same bytes are three real characters. +TEST(Transcode, a_byte_order_mark_is_stripped_only_by_its_own_encoding) { + EXPECT_EQ(encoding::to_utf8("\xef\xbb\xbf", TextEncoding::utf8), ""); + EXPECT_EQ(encoding::to_utf8("\xef\xbb\xbf", TextEncoding::windows_1252), + ""); +} + +TEST(Transcode, an_undecodable_encoding_throws) { + EXPECT_THROW((void)encoding::to_utf8("x", TextEncoding::shift_jis), + std::runtime_error); + EXPECT_THROW((void)encoding::to_utf8("x", TextEncoding::unknown), + std::runtime_error); +} + +TEST(Detect, a_byte_order_mark_wins) { + EXPECT_EQ(encoding::detect("\xef\xbb\xbfhello"), TextEncoding::utf8); + EXPECT_EQ(encoding::detect(std::string("\xff\xfe\0\0", 4)), + TextEncoding::utf32le); + EXPECT_EQ(encoding::detect(std::string("\xff\xfeh\0", 4)), + TextEncoding::utf16le); +} diff --git a/test/src/internal/text/text_file_test.cpp b/test/src/internal/text/text_file_test.cpp index 26e22ea73..03c73689e 100644 --- a/test/src/internal/text/text_file_test.cpp +++ b/test/src/internal/text/text_file_test.cpp @@ -1,5 +1,6 @@ #include #include +#include #include @@ -7,14 +8,66 @@ #include +#include +#include + using namespace odr; using namespace odr::test; -TEST(TextFile, odt) { - const File file(TestData::test_file_path("odr-public/odt/about.odt")); - EXPECT_THROW(internal::text::TextFile(file.impl()), UnknownCharset); +namespace { + +internal::text::TextFile text_file(const std::string &content) { + return internal::text::TextFile(File::from_memory(content).impl()); } +} // namespace + TEST(TextFile, txt) { File(TestData::test_file_path("odr-public/txt/lorem ipsum.txt")); } + +/// Text is the fallback for bytes nothing else claims, so nothing is rejected +/// — a viewer handed a random binary shows junk rather than an error. +TEST(TextFile, anything_reads_as_text) { + EXPECT_NO_THROW(text_file(std::string("\0\1\2\3", 4))); + EXPECT_NO_THROW(text_file("plain ascii")); + + const File odt(TestData::test_file_path("odr-public/odt/about.odt")); + EXPECT_NO_THROW(internal::text::TextFile(odt.impl())); +} + +/// The same contract seen from outside, which `wasm/tests/smoke.test.mjs` +/// pins downstream: bytes nothing recognises still open, as text. +TEST(TextFile, unrecognised_bytes_open_as_text) { + const File junk = File::from_memory(std::string("\0\1\2\3\4\5\6\7", 8)); + + EXPECT_EQ(mimetype(junk), "text/plain"); + EXPECT_EQ(DecodedFile(junk).file_type(), FileType::text_file); +} + +TEST(TextFile, encoding_comes_from_the_byte_order_mark) { + EXPECT_EQ(text_file("\xef\xbb\xbfhello").encoding(), TextEncoding::utf8); + EXPECT_EQ(text_file(std::string("\xff\xfeh\0i\0", 6)).encoding(), + TextEncoding::utf16le); + EXPECT_EQ(text_file(std::string("\xfe\xff\0h\0i", 6)).encoding(), + TextEncoding::utf16be); + EXPECT_EQ(text_file(std::string("\xff\xfe\0\0h\0\0\0", 8)).encoding(), + TextEncoding::utf32le); +} + +TEST(TextFile, text_is_decoded_to_utf8) { + // 0xe9 is `é` in latin-1 and not valid utf-8, so the encoding decides + const TextFile file(std::make_shared( + File::from_memory("caf\xe9").impl(), TextEncoding::iso_8859_1)); + EXPECT_EQ(file.text(), "café"); +} + +/// Nothing decodes these, so the bytes come back as they are and the caller +/// hands them to something that does. +TEST(TextFile, an_undecodable_encoding_yields_its_bytes) { + const std::string content = "\x82\xa0\x82\xa2"; + const TextFile file(std::make_shared( + File::from_memory(content).impl(), TextEncoding::shift_jis)); + EXPECT_FALSE(text_encoding_is_decodable(TextEncoding::shift_jis)); + EXPECT_EQ(file.text(), content); +} diff --git a/tools/encoding/generate_encoding_data.py b/tools/encoding/generate_encoding_data.py new file mode 100644 index 000000000..e85760a36 --- /dev/null +++ b/tools/encoding/generate_encoding_data.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Generates the single-byte decoding tables in `internal/encoding`. + +Each supported single-byte encoding becomes a 256-entry byte -> code point +table. The mappings come from Python's own codecs, which carry the standard +data; codes the encoding leaves undefined become U+FFFD. + +The set mirrors the WHATWG Encoding Standard's single-byte encodings, i.e. what +browsers implement, plus a true ISO-8859-1 (WHATWG deliberately folds that one +into windows-1252; we do not, because we report what we detected). + +Regenerate with: + + python3 tools/encoding/generate_encoding_data.py +""" + +from __future__ import annotations + +import pathlib +import sys + +REPLACEMENT = 0xFFFD + +# (identifier in the generated code, Python codec name) +ENCODINGS = [ + ("ibm866", "cp866"), + ("iso_8859_1", "iso8859_1"), + ("iso_8859_2", "iso8859_2"), + ("iso_8859_3", "iso8859_3"), + ("iso_8859_4", "iso8859_4"), + ("iso_8859_5", "iso8859_5"), + ("iso_8859_6", "iso8859_6"), + ("iso_8859_7", "iso8859_7"), + ("iso_8859_8", "iso8859_8"), + ("iso_8859_10", "iso8859_10"), + ("iso_8859_13", "iso8859_13"), + ("iso_8859_14", "iso8859_14"), + ("iso_8859_15", "iso8859_15"), + ("iso_8859_16", "iso8859_16"), + ("koi8_r", "koi8_r"), + ("koi8_u", "koi8_u"), + ("macintosh", "mac_roman"), + ("windows_874", "cp874"), + ("windows_1250", "cp1250"), + ("windows_1251", "cp1251"), + ("windows_1252", "cp1252"), + ("windows_1253", "cp1253"), + ("windows_1254", "cp1254"), + ("windows_1255", "cp1255"), + ("windows_1256", "cp1256"), + ("windows_1257", "cp1257"), + ("windows_1258", "cp1258"), + ("x_mac_cyrillic", "mac_cyrillic"), +] + +HEADER_NOTE = """// Auto-generated by tools/encoding/generate_encoding_data.py -- DO NOT EDIT. +// Regenerate with: python3 tools/encoding/generate_encoding_data.py +// Data: the Python standard library's codecs. + +// clang-format off +""" + + +def table_for(codec: str) -> list[int]: + result = [] + for byte in range(256): + try: + decoded = bytes([byte]).decode(codec) + except UnicodeDecodeError: + result.append(REPLACEMENT) + continue + # a few codecs map one byte to a combining sequence; none of ours do, + # and silently truncating would be a wrong table + if len(decoded) != 1: + raise ValueError(f"{codec}: byte {byte:#04x} decodes to {decoded!r}") + result.append(ord(decoded)) + return result + + +def write_header(path: pathlib.Path) -> None: + lines = [ + HEADER_NOTE, + "#pragma once", + "", + "#include ", + "", + "namespace odr::internal::encoding::encoding_data {", + "", + "/// Byte -> code point for one single-byte encoding; U+FFFD where the", + "/// encoding leaves the byte undefined.", + "using SingleByteTable = std::array;", + "", + ] + lines += [f"extern const SingleByteTable {name};" for name, _ in ENCODINGS] + lines += ["", "} // namespace odr::internal::encoding::encoding_data", ""] + path.write_text("\n".join(lines)) + + +def write_source(path: pathlib.Path) -> None: + lines = [ + HEADER_NOTE, + "#include ", + "", + "namespace odr::internal::encoding::encoding_data {", + "", + ] + for name, codec in ENCODINGS: + table = table_for(codec) + lines.append(f"const SingleByteTable {name}{{") + for start in range(0, 256, 8): + row = ", ".join(f"0x{cp:04x}" for cp in table[start : start + 8]) + lines.append(f" {row},") + lines.append("};") + lines.append("") + lines += ["} // namespace odr::internal::encoding::encoding_data", ""] + path.write_text("\n".join(lines)) + + +def main() -> int: + root = pathlib.Path(__file__).resolve().parents[2] + out = root / "src" / "odr" / "internal" / "encoding" + out.mkdir(parents=True, exist_ok=True) + write_header(out / "encoding_data.hpp") + write_source(out / "encoding_data.cpp") + print(f"wrote {len(ENCODINGS)} tables to {out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())