diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e4299d96..d37d8ee5e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,6 +100,10 @@ set(ODR_SOURCE_FILES "src/odr/table_dimension.cpp" "src/odr/table_position.cpp" + "src/odr/internal/encoding/detect.cpp" + "src/odr/internal/encoding/encoding_data.cpp" + "src/odr/internal/encoding/text_encoding_table.cpp" + "src/odr/internal/encoding/transcode.cpp" "src/odr/internal/file_type_table.cpp" "${CMAKE_CURRENT_BINARY_DIR}/src/odr/internal/git_info.cpp" "src/odr/internal/magic.cpp" @@ -233,7 +237,6 @@ set(ODR_SOURCE_FILES "src/odr/internal/svm/svm_to_svg.cpp" "src/odr/internal/text/text_file.cpp" - "src/odr/internal/text/text_util.cpp" "src/odr/internal/util/byte_util.cpp" "src/odr/internal/util/byte_stream_util.cpp" diff --git a/apple/src/ODRFile.mm b/apple/src/ODRFile.mm index f7167a1ec..d1dad972d 100644 --- a/apple/src/ODRFile.mm +++ b/apple/src/ODRFile.mm @@ -4,6 +4,7 @@ #import "ODRPrivate.h" #include +#include #include #include @@ -516,9 +517,12 @@ @implementation ODRTextFile - (nullable NSString *)charset { return guarded_value( [&]() -> NSString * { - const std::optional charset = - self.handle.as_text_file().charset(); - return charset.has_value() ? to_nsstring(*charset) : nil; + const odr::TextEncoding encoding = + self.handle.as_text_file().encoding(); + return encoding == odr::TextEncoding::unknown + ? nil + : to_nsstring( + std::string(odr::text_encoding_to_string(encoding))); }, nil); } diff --git a/jni/src/jni_file.cpp b/jni/src/jni_file.cpp index 9331d8f3c..b55218ccf 100644 --- a/jni/src/jni_file.cpp +++ b/jni/src/jni_file.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -288,8 +289,12 @@ extern "C" JNIEXPORT jstring JNICALL Java_app_opendocument_core_TextFile_charsetNative(JNIEnv *env, jobject, jlong handle) { return guarded(env, [&] { - return odr_jni::make_string_opt(env, - decoded(handle).as_text_file().charset()); + const odr::TextEncoding encoding = + decoded(handle).as_text_file().encoding(); + return odr_jni::make_string_opt( + env, encoding == odr::TextEncoding::unknown + ? std::optional{} + : std::string(odr::text_encoding_to_string(encoding))); }); } diff --git a/python/src/bind_file.cpp b/python/src/bind_file.cpp index c9159fb9a..b0eaf79d6 100644 --- a/python/src/bind_file.cpp +++ b/python/src/bind_file.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -215,7 +216,14 @@ void odr_python::bind_file(py::module_ &m) { .def("as_font_file", &odr::DecodedFile::as_font_file); py::class_(m, "TextFile") - .def("charset", &odr::TextFile::charset) + .def("charset", + [](const odr::TextFile &file) -> std::optional { + const odr::TextEncoding encoding = file.encoding(); + if (encoding == odr::TextEncoding::unknown) { + return {}; + } + return std::string(odr::text_encoding_to_string(encoding)); + }) .def("text", &odr::TextFile::text); py::class_(m, "ImageFile") diff --git a/src/odr/exceptions.cpp b/src/odr/exceptions.cpp index 1ff5b674d..065ca8e05 100644 --- a/src/odr/exceptions.cpp +++ b/src/odr/exceptions.cpp @@ -22,6 +22,10 @@ UnsupportedFileType::UnsupportedFileType(const FileType file_type) : Exception("unsupported file type: " + file_type_to_string(file_type)), file_type{file_type} {} +UnsupportedTextEncoding::UnsupportedTextEncoding( + const TextEncoding text_encoding) + : Exception("unsupported text encoding"), text_encoding{text_encoding} {} + FileReadError::FileReadError() : Exception("file read error") {} FileWriteError::FileWriteError(const std::string &path) diff --git a/src/odr/exceptions.hpp b/src/odr/exceptions.hpp index 4e0d2fdf8..dac37e653 100644 --- a/src/odr/exceptions.hpp +++ b/src/odr/exceptions.hpp @@ -5,6 +5,7 @@ namespace odr { enum class FileType; +enum class TextEncoding; /// @brief Base of every exception type this library declares. The decoders also /// throw plain `std::runtime_error` for malformed input with no dedicated type, @@ -37,6 +38,13 @@ struct UnsupportedFileType final : Exception { explicit UnsupportedFileType(FileType file_type); }; +/// @brief Unsupported text encoding exception +struct UnsupportedTextEncoding final : Exception { + TextEncoding text_encoding; + + explicit UnsupportedTextEncoding(TextEncoding text_encoding); +}; + /// @brief File read error struct FileReadError final : Exception { FileReadError(); @@ -88,7 +96,9 @@ struct NoJsonFile final : Exception { }; /// @brief Unknown charset exception -struct UnknownCharset final : Exception { +/// @deprecated Nothing throws this any more: a file whose encoding cannot be +/// named is still text, and reports @ref TextEncoding::unknown. +struct [[deprecated("nothing throws this")]] UnknownCharset final : Exception { UnknownCharset(); }; diff --git a/src/odr/file.cpp b/src/odr/file.cpp index d14d3c482..d30944cb5 100644 --- a/src/odr/file.cpp +++ b/src/odr/file.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -264,8 +265,14 @@ FontFile DecodedFile::as_font_file() const { TextFile::TextFile(std::shared_ptr impl) : DecodedFile(impl), m_impl{std::move(impl)} {} +TextEncoding TextFile::encoding() const { return m_impl->encoding(); } + std::optional TextFile::charset() const { - return {}; // TODO + const TextEncoding encoding = this->encoding(); + if (encoding == TextEncoding::unknown) { + return {}; + } + return std::string(text_encoding_to_string(encoding)); } std::unique_ptr TextFile::stream() const { @@ -274,7 +281,13 @@ std::unique_ptr TextFile::stream() const { std::string TextFile::text() const { const auto stream = m_impl->file()->stream(); - return internal::util::stream::read(*stream); + std::string bytes = internal::util::stream::read(*stream); + + const TextEncoding encoding = this->encoding(); + if (!text_encoding_is_decodable(encoding)) { + return bytes; + } + return internal::encoding::to_utf8(bytes, encoding); } ImageFile::ImageFile(std::shared_ptr impl) diff --git a/src/odr/file.hpp b/src/odr/file.hpp index 0b2f872fd..9dc2c230c 100644 --- a/src/odr/file.hpp +++ b/src/odr/file.hpp @@ -208,6 +208,60 @@ enum class DocumentType { drawing, }; +/// @brief Collection of text encodings. +/// +/// Only some can be decoded — see @ref text_encoding_is_decodable; the rest are +/// named so a caller can say what a file is and hand it to something that can. +enum class TextEncoding { + unknown, + + // unicode + utf8, + utf16le, + utf16be, + utf32le, + utf32be, + + // single byte + ibm866, + iso_8859_1, + iso_8859_2, + iso_8859_3, + iso_8859_4, + iso_8859_5, + iso_8859_6, + iso_8859_7, + iso_8859_8, + iso_8859_10, + iso_8859_13, + iso_8859_14, + iso_8859_15, + iso_8859_16, + koi8_r, + koi8_u, + macintosh, + windows_874, + windows_1250, + windows_1251, + windows_1252, + windows_1253, + windows_1254, + windows_1255, + windows_1256, + windows_1257, + windows_1258, + x_mac_cyrillic, + + // multi byte; named but not decoded + big5, + euc_jp, + euc_kr, + gb18030, + iso_2022_jp, + iso_2022_kr, + shift_jis, +}; + /// @brief Meta information about a file. /// /// The document fields are only meaningful when @ref document_type is set; @@ -336,9 +390,18 @@ class TextFile final : public DecodedFile { public: explicit TextFile(std::shared_ptr); - [[nodiscard]] std::optional charset() const; + /// @brief The encoding the file's bytes were detected as, or decoded with. + [[nodiscard]] TextEncoding encoding() const; + + /// @deprecated See @ref encoding. Returns the encoding's canonical name, and + /// `nullopt` for @ref TextEncoding::unknown. + [[deprecated("use encoding()")]] [[nodiscard]] std::optional + charset() const; + /// @brief The file's bytes as they are. [[nodiscard]] std::unique_ptr stream() const; + /// @brief The file's text, decoded to UTF-8 where @ref encoding is + /// decodable, and the raw bytes where it is not. [[nodiscard]] std::string text() const; private: diff --git a/src/odr/internal/abstract/file.hpp b/src/odr/internal/abstract/file.hpp index 94a9ea519..05de574ae 100644 --- a/src/odr/internal/abstract/file.hpp +++ b/src/odr/internal/abstract/file.hpp @@ -62,6 +62,9 @@ class TextFile : public DecodedFile { [[nodiscard]] FileCategory file_category() const noexcept final { return FileCategory::text; } + + /// The encoding the bytes were detected as, or decoded with. + [[nodiscard]] virtual TextEncoding encoding() const noexcept = 0; }; class ImageFile : public DecodedFile { diff --git a/src/odr/internal/csv/csv_file.cpp b/src/odr/internal/csv/csv_file.cpp index a129c3924..1c5751905 100644 --- a/src/odr/internal/csv/csv_file.cpp +++ b/src/odr/internal/csv/csv_file.cpp @@ -29,4 +29,6 @@ FileMeta CsvFile::file_meta() const noexcept { bool CsvFile::is_decodable() const noexcept { return false; } +TextEncoding CsvFile::encoding() const noexcept { return m_file->encoding(); } + } // namespace odr::internal::csv diff --git a/src/odr/internal/csv/csv_file.hpp b/src/odr/internal/csv/csv_file.hpp index 50a0c84fe..2f4df0c7f 100644 --- a/src/odr/internal/csv/csv_file.hpp +++ b/src/odr/internal/csv/csv_file.hpp @@ -20,9 +20,10 @@ class CsvFile final : public abstract::TextFile { [[nodiscard]] bool is_decodable() const noexcept override; + [[nodiscard]] TextEncoding encoding() const noexcept override; + private: std::shared_ptr m_file; - std::string m_charset; }; } // namespace odr::internal::csv diff --git a/src/odr/internal/encoding/detect.cpp b/src/odr/internal/encoding/detect.cpp new file mode 100644 index 000000000..52edc32e5 --- /dev/null +++ b/src/odr/internal/encoding/detect.cpp @@ -0,0 +1,67 @@ +#include + +#include + +#include +#include + +#include + +namespace odr::internal { + +namespace { + +using namespace std::string_view_literals; + +/// The encoding a byte-order mark names, `unknown` when there is none. +TextEncoding encoding_by_bom(const std::string_view probe) { + const auto starts_with = [&](const std::string_view bom) { + return probe.substr(0, bom.size()) == bom; + }; + + // utf-32le opens with the utf-16le mark, so it has to be tested first + if (starts_with("\xff\xfe\x00\x00"sv)) { + return TextEncoding::utf32le; + } + if (starts_with("\x00\x00\xfe\xff"sv)) { + return TextEncoding::utf32be; + } + if (starts_with("\xef\xbb\xbf"sv)) { + return TextEncoding::utf8; + } + if (starts_with("\xff\xfe"sv)) { + return TextEncoding::utf16le; + } + if (starts_with("\xfe\xff"sv)) { + return TextEncoding::utf16be; + } + return TextEncoding::unknown; +} + +} // namespace + +std::string encoding::read_probe(std::istream &in, + const std::size_t max_bytes) { + std::string result(max_bytes, '\0'); + in.read(result.data(), static_cast(max_bytes)); + result.resize(static_cast(in.gcount())); + return result; +} + +TextEncoding encoding::detect(const std::string_view probe) { + if (const TextEncoding by_bom = encoding_by_bom(probe); + by_bom != TextEncoding::unknown) { + return by_bom; + } + + auto *const detector = uchardet_new(); + uchardet_handle_data(detector, probe.data(), probe.size()); + uchardet_data_end(detector); + const std::string name = uchardet_get_charset(detector); + uchardet_delete(detector); + + const auto *row = text_encoding_table::find_by_name(name); + return row == nullptr ? TextEncoding::unknown : row->encoding; +} + +} // namespace odr::internal diff --git a/src/odr/internal/encoding/detect.hpp b/src/odr/internal/encoding/detect.hpp new file mode 100644 index 000000000..51e451360 --- /dev/null +++ b/src/odr/internal/encoding/detect.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace odr::internal::encoding { + +/// Enough for uchardet, and bounded so classifying a large file costs no pass +/// over it. +constexpr std::size_t default_probe_size = std::size_t{64} * 1024; + +/// Reads at most @p max_bytes from @p in, from wherever it stands. +[[nodiscard]] std::string +read_probe(std::istream &in, std::size_t max_bytes = default_probe_size); + +/// The encoding of @p probe, @ref TextEncoding::unknown if it cannot be named. +/// A byte-order mark decides alone, else uchardet guesses — from the probe +/// only, so an encoding invisible in an ASCII prefix is missed. +[[nodiscard]] TextEncoding detect(std::string_view probe); + +} // namespace odr::internal::encoding diff --git a/src/odr/internal/encoding/encoding_data.cpp b/src/odr/internal/encoding/encoding_data.cpp new file mode 100644 index 000000000..e6d02d685 --- /dev/null +++ b/src/odr/internal/encoding/encoding_data.cpp @@ -0,0 +1,991 @@ +// Auto-generated by tools/encoding/generate_encoding_data.py -- DO NOT EDIT. +// Regenerate with: python3 tools/encoding/generate_encoding_data.py +// Data: the Python standard library's codecs. + +// clang-format off + +#include + +namespace odr::internal::encoding::encoding_data { + +const SingleByteTable ibm866{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556, + 0x2555, 0x2563, 0x2551, 0x2557, 0x255d, 0x255c, 0x255b, 0x2510, + 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x255e, 0x255f, + 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x2567, + 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256b, + 0x256a, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, + 0x0401, 0x0451, 0x0404, 0x0454, 0x0407, 0x0457, 0x040e, 0x045e, + 0x00b0, 0x2219, 0x00b7, 0x221a, 0x2116, 0x00a4, 0x25a0, 0x00a0, +}; + +const SingleByteTable iso_8859_1{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff, +}; + +const SingleByteTable iso_8859_2{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0104, 0x02d8, 0x0141, 0x00a4, 0x013d, 0x015a, 0x00a7, + 0x00a8, 0x0160, 0x015e, 0x0164, 0x0179, 0x00ad, 0x017d, 0x017b, + 0x00b0, 0x0105, 0x02db, 0x0142, 0x00b4, 0x013e, 0x015b, 0x02c7, + 0x00b8, 0x0161, 0x015f, 0x0165, 0x017a, 0x02dd, 0x017e, 0x017c, + 0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7, + 0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e, + 0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7, + 0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df, + 0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7, + 0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f, + 0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7, + 0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9, +}; + +const SingleByteTable iso_8859_3{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0126, 0x02d8, 0x00a3, 0x00a4, 0xfffd, 0x0124, 0x00a7, + 0x00a8, 0x0130, 0x015e, 0x011e, 0x0134, 0x00ad, 0xfffd, 0x017b, + 0x00b0, 0x0127, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x0125, 0x00b7, + 0x00b8, 0x0131, 0x015f, 0x011f, 0x0135, 0x00bd, 0xfffd, 0x017c, + 0x00c0, 0x00c1, 0x00c2, 0xfffd, 0x00c4, 0x010a, 0x0108, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0xfffd, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x0120, 0x00d6, 0x00d7, + 0x011c, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x016c, 0x015c, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0xfffd, 0x00e4, 0x010b, 0x0109, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0xfffd, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x0121, 0x00f6, 0x00f7, + 0x011d, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x016d, 0x015d, 0x02d9, +}; + +const SingleByteTable iso_8859_4{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0104, 0x0138, 0x0156, 0x00a4, 0x0128, 0x013b, 0x00a7, + 0x00a8, 0x0160, 0x0112, 0x0122, 0x0166, 0x00ad, 0x017d, 0x00af, + 0x00b0, 0x0105, 0x02db, 0x0157, 0x00b4, 0x0129, 0x013c, 0x02c7, + 0x00b8, 0x0161, 0x0113, 0x0123, 0x0167, 0x014a, 0x017e, 0x014b, + 0x0100, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x012e, + 0x010c, 0x00c9, 0x0118, 0x00cb, 0x0116, 0x00cd, 0x00ce, 0x012a, + 0x0110, 0x0145, 0x014c, 0x0136, 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x0172, 0x00da, 0x00db, 0x00dc, 0x0168, 0x016a, 0x00df, + 0x0101, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x012f, + 0x010d, 0x00e9, 0x0119, 0x00eb, 0x0117, 0x00ed, 0x00ee, 0x012b, + 0x0111, 0x0146, 0x014d, 0x0137, 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x0173, 0x00fa, 0x00fb, 0x00fc, 0x0169, 0x016b, 0x02d9, +}; + +const SingleByteTable iso_8859_5{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0401, 0x0402, 0x0403, 0x0404, 0x0405, 0x0406, 0x0407, + 0x0408, 0x0409, 0x040a, 0x040b, 0x040c, 0x00ad, 0x040e, 0x040f, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, + 0x2116, 0x0451, 0x0452, 0x0453, 0x0454, 0x0455, 0x0456, 0x0457, + 0x0458, 0x0459, 0x045a, 0x045b, 0x045c, 0x00a7, 0x045e, 0x045f, +}; + +const SingleByteTable iso_8859_6{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0xfffd, 0xfffd, 0xfffd, 0x00a4, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0x060c, 0x00ad, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0x061b, 0xfffd, 0xfffd, 0xfffd, 0x061f, + 0xfffd, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, + 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f, + 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x0637, + 0x0638, 0x0639, 0x063a, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647, + 0x0648, 0x0649, 0x064a, 0x064b, 0x064c, 0x064d, 0x064e, 0x064f, + 0x0650, 0x0651, 0x0652, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, +}; + +const SingleByteTable iso_8859_7{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x2018, 0x2019, 0x00a3, 0x20ac, 0x20af, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x037a, 0x00ab, 0x00ac, 0x00ad, 0xfffd, 0x2015, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x0384, 0x0385, 0x0386, 0x00b7, + 0x0388, 0x0389, 0x038a, 0x00bb, 0x038c, 0x00bd, 0x038e, 0x038f, + 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, + 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f, + 0x03a0, 0x03a1, 0xfffd, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7, + 0x03a8, 0x03a9, 0x03aa, 0x03ab, 0x03ac, 0x03ad, 0x03ae, 0x03af, + 0x03b0, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, + 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, + 0x03c0, 0x03c1, 0x03c2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, + 0x03c8, 0x03c9, 0x03ca, 0x03cb, 0x03cc, 0x03cd, 0x03ce, 0xfffd, +}; + +const SingleByteTable iso_8859_8{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0xfffd, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00d7, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00f7, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0x2017, + 0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0x05d7, + 0x05d8, 0x05d9, 0x05da, 0x05db, 0x05dc, 0x05dd, 0x05de, 0x05df, + 0x05e0, 0x05e1, 0x05e2, 0x05e3, 0x05e4, 0x05e5, 0x05e6, 0x05e7, + 0x05e8, 0x05e9, 0x05ea, 0xfffd, 0xfffd, 0x200e, 0x200f, 0xfffd, +}; + +const SingleByteTable iso_8859_10{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0104, 0x0112, 0x0122, 0x012a, 0x0128, 0x0136, 0x00a7, + 0x013b, 0x0110, 0x0160, 0x0166, 0x017d, 0x00ad, 0x016a, 0x014a, + 0x00b0, 0x0105, 0x0113, 0x0123, 0x012b, 0x0129, 0x0137, 0x00b7, + 0x013c, 0x0111, 0x0161, 0x0167, 0x017e, 0x2015, 0x016b, 0x014b, + 0x0100, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x012e, + 0x010c, 0x00c9, 0x0118, 0x00cb, 0x0116, 0x00cd, 0x00ce, 0x00cf, + 0x00d0, 0x0145, 0x014c, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x0168, + 0x00d8, 0x0172, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df, + 0x0101, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x012f, + 0x010d, 0x00e9, 0x0119, 0x00eb, 0x0117, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x0146, 0x014d, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x0169, + 0x00f8, 0x0173, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x0138, +}; + +const SingleByteTable iso_8859_13{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x201d, 0x00a2, 0x00a3, 0x00a4, 0x201e, 0x00a6, 0x00a7, + 0x00d8, 0x00a9, 0x0156, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00c6, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x201c, 0x00b5, 0x00b6, 0x00b7, + 0x00f8, 0x00b9, 0x0157, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00e6, + 0x0104, 0x012e, 0x0100, 0x0106, 0x00c4, 0x00c5, 0x0118, 0x0112, + 0x010c, 0x00c9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012a, 0x013b, + 0x0160, 0x0143, 0x0145, 0x00d3, 0x014c, 0x00d5, 0x00d6, 0x00d7, + 0x0172, 0x0141, 0x015a, 0x016a, 0x00dc, 0x017b, 0x017d, 0x00df, + 0x0105, 0x012f, 0x0101, 0x0107, 0x00e4, 0x00e5, 0x0119, 0x0113, + 0x010d, 0x00e9, 0x017a, 0x0117, 0x0123, 0x0137, 0x012b, 0x013c, + 0x0161, 0x0144, 0x0146, 0x00f3, 0x014d, 0x00f5, 0x00f6, 0x00f7, + 0x0173, 0x0142, 0x015b, 0x016b, 0x00fc, 0x017c, 0x017e, 0x2019, +}; + +const SingleByteTable iso_8859_14{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x1e02, 0x1e03, 0x00a3, 0x010a, 0x010b, 0x1e0a, 0x00a7, + 0x1e80, 0x00a9, 0x1e82, 0x1e0b, 0x1ef2, 0x00ad, 0x00ae, 0x0178, + 0x1e1e, 0x1e1f, 0x0120, 0x0121, 0x1e40, 0x1e41, 0x00b6, 0x1e56, + 0x1e81, 0x1e57, 0x1e83, 0x1e60, 0x1ef3, 0x1e84, 0x1e85, 0x1e61, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x0174, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x1e6a, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x0176, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x0175, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x1e6b, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x0177, 0x00ff, +}; + +const SingleByteTable iso_8859_15{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x20ac, 0x00a5, 0x0160, 0x00a7, + 0x0161, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x017d, 0x00b5, 0x00b6, 0x00b7, + 0x017e, 0x00b9, 0x00ba, 0x00bb, 0x0152, 0x0153, 0x0178, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff, +}; + +const SingleByteTable iso_8859_16{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0104, 0x0105, 0x0141, 0x20ac, 0x201e, 0x0160, 0x00a7, + 0x0161, 0x00a9, 0x0218, 0x00ab, 0x0179, 0x00ad, 0x017a, 0x017b, + 0x00b0, 0x00b1, 0x010c, 0x0142, 0x017d, 0x201d, 0x00b6, 0x00b7, + 0x017e, 0x010d, 0x0219, 0x00bb, 0x0152, 0x0153, 0x0178, 0x017c, + 0x00c0, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0106, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x0110, 0x0143, 0x00d2, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x015a, + 0x0170, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x0118, 0x021a, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x0107, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x0111, 0x0144, 0x00f2, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x015b, + 0x0171, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x0119, 0x021b, 0x00ff, +}; + +const SingleByteTable koi8_r{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x2500, 0x2502, 0x250c, 0x2510, 0x2514, 0x2518, 0x251c, 0x2524, + 0x252c, 0x2534, 0x253c, 0x2580, 0x2584, 0x2588, 0x258c, 0x2590, + 0x2591, 0x2592, 0x2593, 0x2320, 0x25a0, 0x2219, 0x221a, 0x2248, + 0x2264, 0x2265, 0x00a0, 0x2321, 0x00b0, 0x00b2, 0x00b7, 0x00f7, + 0x2550, 0x2551, 0x2552, 0x0451, 0x2553, 0x2554, 0x2555, 0x2556, + 0x2557, 0x2558, 0x2559, 0x255a, 0x255b, 0x255c, 0x255d, 0x255e, + 0x255f, 0x2560, 0x2561, 0x0401, 0x2562, 0x2563, 0x2564, 0x2565, + 0x2566, 0x2567, 0x2568, 0x2569, 0x256a, 0x256b, 0x256c, 0x00a9, + 0x044e, 0x0430, 0x0431, 0x0446, 0x0434, 0x0435, 0x0444, 0x0433, + 0x0445, 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, + 0x043f, 0x044f, 0x0440, 0x0441, 0x0442, 0x0443, 0x0436, 0x0432, + 0x044c, 0x044b, 0x0437, 0x0448, 0x044d, 0x0449, 0x0447, 0x044a, + 0x042e, 0x0410, 0x0411, 0x0426, 0x0414, 0x0415, 0x0424, 0x0413, + 0x0425, 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, + 0x041f, 0x042f, 0x0420, 0x0421, 0x0422, 0x0423, 0x0416, 0x0412, + 0x042c, 0x042b, 0x0417, 0x0428, 0x042d, 0x0429, 0x0427, 0x042a, +}; + +const SingleByteTable koi8_u{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x2500, 0x2502, 0x250c, 0x2510, 0x2514, 0x2518, 0x251c, 0x2524, + 0x252c, 0x2534, 0x253c, 0x2580, 0x2584, 0x2588, 0x258c, 0x2590, + 0x2591, 0x2592, 0x2593, 0x2320, 0x25a0, 0x2219, 0x221a, 0x2248, + 0x2264, 0x2265, 0x00a0, 0x2321, 0x00b0, 0x00b2, 0x00b7, 0x00f7, + 0x2550, 0x2551, 0x2552, 0x0451, 0x0454, 0x2554, 0x0456, 0x0457, + 0x2557, 0x2558, 0x2559, 0x255a, 0x255b, 0x0491, 0x255d, 0x255e, + 0x255f, 0x2560, 0x2561, 0x0401, 0x0404, 0x2563, 0x0406, 0x0407, + 0x2566, 0x2567, 0x2568, 0x2569, 0x256a, 0x0490, 0x256c, 0x00a9, + 0x044e, 0x0430, 0x0431, 0x0446, 0x0434, 0x0435, 0x0444, 0x0433, + 0x0445, 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, + 0x043f, 0x044f, 0x0440, 0x0441, 0x0442, 0x0443, 0x0436, 0x0432, + 0x044c, 0x044b, 0x0437, 0x0448, 0x044d, 0x0449, 0x0447, 0x044a, + 0x042e, 0x0410, 0x0411, 0x0426, 0x0414, 0x0415, 0x0424, 0x0413, + 0x0425, 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, + 0x041f, 0x042f, 0x0420, 0x0421, 0x0422, 0x0423, 0x0416, 0x0412, + 0x042c, 0x042b, 0x0417, 0x0428, 0x042d, 0x0429, 0x0427, 0x042a, +}; + +const SingleByteTable macintosh{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x00c4, 0x00c5, 0x00c7, 0x00c9, 0x00d1, 0x00d6, 0x00dc, 0x00e1, + 0x00e0, 0x00e2, 0x00e4, 0x00e3, 0x00e5, 0x00e7, 0x00e9, 0x00e8, + 0x00ea, 0x00eb, 0x00ed, 0x00ec, 0x00ee, 0x00ef, 0x00f1, 0x00f3, + 0x00f2, 0x00f4, 0x00f6, 0x00f5, 0x00fa, 0x00f9, 0x00fb, 0x00fc, + 0x2020, 0x00b0, 0x00a2, 0x00a3, 0x00a7, 0x2022, 0x00b6, 0x00df, + 0x00ae, 0x00a9, 0x2122, 0x00b4, 0x00a8, 0x2260, 0x00c6, 0x00d8, + 0x221e, 0x00b1, 0x2264, 0x2265, 0x00a5, 0x00b5, 0x2202, 0x2211, + 0x220f, 0x03c0, 0x222b, 0x00aa, 0x00ba, 0x03a9, 0x00e6, 0x00f8, + 0x00bf, 0x00a1, 0x00ac, 0x221a, 0x0192, 0x2248, 0x2206, 0x00ab, + 0x00bb, 0x2026, 0x00a0, 0x00c0, 0x00c3, 0x00d5, 0x0152, 0x0153, + 0x2013, 0x2014, 0x201c, 0x201d, 0x2018, 0x2019, 0x00f7, 0x25ca, + 0x00ff, 0x0178, 0x2044, 0x20ac, 0x2039, 0x203a, 0xfb01, 0xfb02, + 0x2021, 0x00b7, 0x201a, 0x201e, 0x2030, 0x00c2, 0x00ca, 0x00c1, + 0x00cb, 0x00c8, 0x00cd, 0x00ce, 0x00cf, 0x00cc, 0x00d3, 0x00d4, + 0xf8ff, 0x00d2, 0x00da, 0x00db, 0x00d9, 0x0131, 0x02c6, 0x02dc, + 0x00af, 0x02d8, 0x02d9, 0x02da, 0x00b8, 0x02dd, 0x02db, 0x02c7, +}; + +const SingleByteTable windows_874{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0x2026, 0xfffd, 0xfffd, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0x00a0, 0x0e01, 0x0e02, 0x0e03, 0x0e04, 0x0e05, 0x0e06, 0x0e07, + 0x0e08, 0x0e09, 0x0e0a, 0x0e0b, 0x0e0c, 0x0e0d, 0x0e0e, 0x0e0f, + 0x0e10, 0x0e11, 0x0e12, 0x0e13, 0x0e14, 0x0e15, 0x0e16, 0x0e17, + 0x0e18, 0x0e19, 0x0e1a, 0x0e1b, 0x0e1c, 0x0e1d, 0x0e1e, 0x0e1f, + 0x0e20, 0x0e21, 0x0e22, 0x0e23, 0x0e24, 0x0e25, 0x0e26, 0x0e27, + 0x0e28, 0x0e29, 0x0e2a, 0x0e2b, 0x0e2c, 0x0e2d, 0x0e2e, 0x0e2f, + 0x0e30, 0x0e31, 0x0e32, 0x0e33, 0x0e34, 0x0e35, 0x0e36, 0x0e37, + 0x0e38, 0x0e39, 0x0e3a, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0x0e3f, + 0x0e40, 0x0e41, 0x0e42, 0x0e43, 0x0e44, 0x0e45, 0x0e46, 0x0e47, + 0x0e48, 0x0e49, 0x0e4a, 0x0e4b, 0x0e4c, 0x0e4d, 0x0e4e, 0x0e4f, + 0x0e50, 0x0e51, 0x0e52, 0x0e53, 0x0e54, 0x0e55, 0x0e56, 0x0e57, + 0x0e58, 0x0e59, 0x0e5a, 0x0e5b, 0xfffd, 0xfffd, 0xfffd, 0xfffd, +}; + +const SingleByteTable windows_1250{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0x201a, 0xfffd, 0x201e, 0x2026, 0x2020, 0x2021, + 0xfffd, 0x2030, 0x0160, 0x2039, 0x015a, 0x0164, 0x017d, 0x0179, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0xfffd, 0x2122, 0x0161, 0x203a, 0x015b, 0x0165, 0x017e, 0x017a, + 0x00a0, 0x02c7, 0x02d8, 0x0141, 0x00a4, 0x0104, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x015e, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x017b, + 0x00b0, 0x00b1, 0x02db, 0x0142, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x0105, 0x015f, 0x00bb, 0x013d, 0x02dd, 0x013e, 0x017c, + 0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7, + 0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e, + 0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7, + 0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df, + 0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7, + 0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f, + 0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7, + 0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9, +}; + +const SingleByteTable windows_1251{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0402, 0x0403, 0x201a, 0x0453, 0x201e, 0x2026, 0x2020, 0x2021, + 0x20ac, 0x2030, 0x0409, 0x2039, 0x040a, 0x040c, 0x040b, 0x040f, + 0x0452, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0xfffd, 0x2122, 0x0459, 0x203a, 0x045a, 0x045c, 0x045b, 0x045f, + 0x00a0, 0x040e, 0x045e, 0x0408, 0x00a4, 0x0490, 0x00a6, 0x00a7, + 0x0401, 0x00a9, 0x0404, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x0407, + 0x00b0, 0x00b1, 0x0406, 0x0456, 0x0491, 0x00b5, 0x00b6, 0x00b7, + 0x0451, 0x2116, 0x0454, 0x00bb, 0x0458, 0x0405, 0x0455, 0x0457, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, +}; + +const SingleByteTable windows_1252{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, + 0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0xfffd, 0x017d, 0xfffd, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0xfffd, 0x017e, 0x0178, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff, +}; + +const SingleByteTable windows_1253{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, + 0xfffd, 0x2030, 0xfffd, 0x2039, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0xfffd, 0x2122, 0xfffd, 0x203a, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0x00a0, 0x0385, 0x0386, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0xfffd, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x2015, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x0384, 0x00b5, 0x00b6, 0x00b7, + 0x0388, 0x0389, 0x038a, 0x00bb, 0x038c, 0x00bd, 0x038e, 0x038f, + 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, + 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f, + 0x03a0, 0x03a1, 0xfffd, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7, + 0x03a8, 0x03a9, 0x03aa, 0x03ab, 0x03ac, 0x03ad, 0x03ae, 0x03af, + 0x03b0, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, + 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, + 0x03c0, 0x03c1, 0x03c2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, + 0x03c8, 0x03c9, 0x03ca, 0x03cb, 0x03cc, 0x03cd, 0x03ce, 0xfffd, +}; + +const SingleByteTable windows_1254{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, + 0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0xfffd, 0xfffd, 0x0178, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x011e, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x0130, 0x015e, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x011f, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x0131, 0x015f, 0x00ff, +}; + +const SingleByteTable windows_1255{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, + 0x02c6, 0x2030, 0xfffd, 0x2039, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0x02dc, 0x2122, 0xfffd, 0x203a, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x20aa, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00d7, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00f7, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x05b0, 0x05b1, 0x05b2, 0x05b3, 0x05b4, 0x05b5, 0x05b6, 0x05b7, + 0x05b8, 0x05b9, 0xfffd, 0x05bb, 0x05bc, 0x05bd, 0x05be, 0x05bf, + 0x05c0, 0x05c1, 0x05c2, 0x05c3, 0x05f0, 0x05f1, 0x05f2, 0x05f3, + 0x05f4, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, + 0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0x05d7, + 0x05d8, 0x05d9, 0x05da, 0x05db, 0x05dc, 0x05dd, 0x05de, 0x05df, + 0x05e0, 0x05e1, 0x05e2, 0x05e3, 0x05e4, 0x05e5, 0x05e6, 0x05e7, + 0x05e8, 0x05e9, 0x05ea, 0xfffd, 0xfffd, 0x200e, 0x200f, 0xfffd, +}; + +const SingleByteTable windows_1256{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0x067e, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, + 0x02c6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688, + 0x06af, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0x06a9, 0x2122, 0x0691, 0x203a, 0x0153, 0x200c, 0x200d, 0x06ba, + 0x00a0, 0x060c, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x06be, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x061b, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x061f, + 0x06c1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, + 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f, + 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00d7, + 0x0637, 0x0638, 0x0639, 0x063a, 0x0640, 0x0641, 0x0642, 0x0643, + 0x00e0, 0x0644, 0x00e2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x0649, 0x064a, 0x00ee, 0x00ef, + 0x064b, 0x064c, 0x064d, 0x064e, 0x00f4, 0x064f, 0x0650, 0x00f7, + 0x0651, 0x00f9, 0x0652, 0x00fb, 0x00fc, 0x200e, 0x200f, 0x06d2, +}; + +const SingleByteTable windows_1257{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0x201a, 0xfffd, 0x201e, 0x2026, 0x2020, 0x2021, + 0xfffd, 0x2030, 0xfffd, 0x2039, 0xfffd, 0x00a8, 0x02c7, 0x00b8, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0xfffd, 0x2122, 0xfffd, 0x203a, 0xfffd, 0x00af, 0x02db, 0xfffd, + 0x00a0, 0xfffd, 0x00a2, 0x00a3, 0x00a4, 0xfffd, 0x00a6, 0x00a7, + 0x00d8, 0x00a9, 0x0156, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00c6, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00f8, 0x00b9, 0x0157, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00e6, + 0x0104, 0x012e, 0x0100, 0x0106, 0x00c4, 0x00c5, 0x0118, 0x0112, + 0x010c, 0x00c9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012a, 0x013b, + 0x0160, 0x0143, 0x0145, 0x00d3, 0x014c, 0x00d5, 0x00d6, 0x00d7, + 0x0172, 0x0141, 0x015a, 0x016a, 0x00dc, 0x017b, 0x017d, 0x00df, + 0x0105, 0x012f, 0x0101, 0x0107, 0x00e4, 0x00e5, 0x0119, 0x0113, + 0x010d, 0x00e9, 0x017a, 0x0117, 0x0123, 0x0137, 0x012b, 0x013c, + 0x0161, 0x0144, 0x0146, 0x00f3, 0x014d, 0x00f5, 0x00f6, 0x00f7, + 0x0173, 0x0142, 0x015b, 0x016b, 0x00fc, 0x017c, 0x017e, 0x02d9, +}; + +const SingleByteTable windows_1258{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xfffd, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, + 0x02c6, 0x2030, 0xfffd, 0x2039, 0x0152, 0xfffd, 0xfffd, 0xfffd, + 0xfffd, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0x02dc, 0x2122, 0xfffd, 0x203a, 0x0153, 0xfffd, 0xfffd, 0x0178, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x0300, 0x00cd, 0x00ce, 0x00cf, + 0x0110, 0x00d1, 0x0309, 0x00d3, 0x00d4, 0x01a0, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x01af, 0x0303, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x0301, 0x00ed, 0x00ee, 0x00ef, + 0x0111, 0x00f1, 0x0323, 0x00f3, 0x00f4, 0x01a1, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x01b0, 0x20ab, 0x00ff, +}; + +const SingleByteTable x_mac_cyrillic{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, + 0x2020, 0x00b0, 0x0490, 0x00a3, 0x00a7, 0x2022, 0x00b6, 0x0406, + 0x00ae, 0x00a9, 0x2122, 0x0402, 0x0452, 0x2260, 0x0403, 0x0453, + 0x221e, 0x00b1, 0x2264, 0x2265, 0x0456, 0x00b5, 0x0491, 0x0408, + 0x0404, 0x0454, 0x0407, 0x0457, 0x0409, 0x0459, 0x040a, 0x045a, + 0x0458, 0x0405, 0x00ac, 0x221a, 0x0192, 0x2248, 0x2206, 0x00ab, + 0x00bb, 0x2026, 0x00a0, 0x040b, 0x045b, 0x040c, 0x045c, 0x0455, + 0x2013, 0x2014, 0x201c, 0x201d, 0x2018, 0x2019, 0x00f7, 0x201e, + 0x040e, 0x045e, 0x040f, 0x045f, 0x2116, 0x0401, 0x0451, 0x044f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x20ac, +}; + +} // namespace odr::internal::encoding::encoding_data diff --git a/src/odr/internal/encoding/encoding_data.hpp b/src/odr/internal/encoding/encoding_data.hpp new file mode 100644 index 000000000..567f49f30 --- /dev/null +++ b/src/odr/internal/encoding/encoding_data.hpp @@ -0,0 +1,46 @@ +// Auto-generated by tools/encoding/generate_encoding_data.py -- DO NOT EDIT. +// Regenerate with: python3 tools/encoding/generate_encoding_data.py +// Data: the Python standard library's codecs. + +// clang-format off + +#pragma once + +#include + +namespace odr::internal::encoding::encoding_data { + +/// Byte -> code point for one single-byte encoding; U+FFFD where the +/// encoding leaves the byte undefined. +using SingleByteTable = std::array; + +extern const SingleByteTable ibm866; +extern const SingleByteTable iso_8859_1; +extern const SingleByteTable iso_8859_2; +extern const SingleByteTable iso_8859_3; +extern const SingleByteTable iso_8859_4; +extern const SingleByteTable iso_8859_5; +extern const SingleByteTable iso_8859_6; +extern const SingleByteTable iso_8859_7; +extern const SingleByteTable iso_8859_8; +extern const SingleByteTable iso_8859_10; +extern const SingleByteTable iso_8859_13; +extern const SingleByteTable iso_8859_14; +extern const SingleByteTable iso_8859_15; +extern const SingleByteTable iso_8859_16; +extern const SingleByteTable koi8_r; +extern const SingleByteTable koi8_u; +extern const SingleByteTable macintosh; +extern const SingleByteTable windows_874; +extern const SingleByteTable windows_1250; +extern const SingleByteTable windows_1251; +extern const SingleByteTable windows_1252; +extern const SingleByteTable windows_1253; +extern const SingleByteTable windows_1254; +extern const SingleByteTable windows_1255; +extern const SingleByteTable windows_1256; +extern const SingleByteTable windows_1257; +extern const SingleByteTable windows_1258; +extern const SingleByteTable x_mac_cyrillic; + +} // namespace odr::internal::encoding::encoding_data diff --git a/src/odr/internal/encoding/text_encoding_table.cpp b/src/odr/internal/encoding/text_encoding_table.cpp new file mode 100644 index 000000000..26ec513fc --- /dev/null +++ b/src/odr/internal/encoding/text_encoding_table.cpp @@ -0,0 +1,159 @@ +#include + +#include +#include +#include +#include + +namespace odr::internal::encoding { + +namespace { + +using text_encoding_table::Row; + +using namespace std::string_view_literals; + +// Names per encoding, canonical one first — it goes into an html +// ``, so it has to be a label a browser knows. Aliases are unique +// across encodings; `odr_test` asserts no name appears twice. + +constexpr std::array utf8_names{"UTF-8"sv, "ASCII"sv, "US-ASCII"sv}; +// a bare `UTF-16`/`UTF-32` only reaches here without a mark, where little +// endian is the likelier intent +constexpr std::array utf16le_names{"UTF-16LE"sv, "UTF-16"sv}; +constexpr std::array utf16be_names{"UTF-16BE"sv}; +constexpr std::array utf32le_names{"UTF-32LE"sv, "UTF-32"sv}; +constexpr std::array utf32be_names{"UTF-32BE"sv}; + +constexpr std::array ibm866_names{"IBM866"sv, "cp866"sv, "866"sv}; +constexpr std::array iso_8859_1_names{"ISO-8859-1"sv, "latin1"sv, "l1"sv, + "iso-ir-100"sv, "cp819"sv}; +constexpr std::array iso_8859_2_names{"ISO-8859-2"sv, "latin2"sv, "l2"sv}; +constexpr std::array iso_8859_3_names{"ISO-8859-3"sv, "latin3"sv, "l3"sv}; +constexpr std::array iso_8859_4_names{"ISO-8859-4"sv, "latin4"sv, "l4"sv}; +constexpr std::array iso_8859_5_names{"ISO-8859-5"sv, "cyrillic"sv}; +constexpr std::array iso_8859_6_names{"ISO-8859-6"sv, "arabic"sv}; +constexpr std::array iso_8859_7_names{"ISO-8859-7"sv, "greek"sv}; +constexpr std::array iso_8859_8_names{"ISO-8859-8"sv, "hebrew"sv}; +constexpr std::array iso_8859_10_names{"ISO-8859-10"sv, "latin6"sv, "l6"sv}; +constexpr std::array iso_8859_13_names{"ISO-8859-13"sv}; +constexpr std::array iso_8859_14_names{"ISO-8859-14"sv, "latin8"sv, "l8"sv}; +constexpr std::array iso_8859_15_names{"ISO-8859-15"sv, "latin9"sv, "l9"sv}; +constexpr std::array iso_8859_16_names{"ISO-8859-16"sv, "latin10"sv, "l10"sv}; +constexpr std::array koi8_r_names{"KOI8-R"sv, "cskoi8r"sv}; +constexpr std::array koi8_u_names{"KOI8-U"sv}; +constexpr std::array macintosh_names{"macintosh"sv, "mac"sv, "x-mac-roman"sv, + "macroman"sv}; +constexpr std::array windows_874_names{"windows-874"sv, "cp874"sv, "TIS-620"sv, + "ISO-8859-11"sv}; +constexpr std::array windows_1250_names{"windows-1250"sv, "cp1250"sv, + "x-cp1250"sv}; +constexpr std::array windows_1251_names{"windows-1251"sv, "cp1251"sv, + "x-cp1251"sv}; +constexpr std::array windows_1252_names{"windows-1252"sv, "cp1252"sv, + "x-cp1252"sv, "ansi"sv}; +constexpr std::array windows_1253_names{"windows-1253"sv, "cp1253"sv}; +constexpr std::array windows_1254_names{"windows-1254"sv, "cp1254"sv}; +constexpr std::array windows_1255_names{"windows-1255"sv, "cp1255"sv}; +constexpr std::array windows_1256_names{"windows-1256"sv, "cp1256"sv}; +constexpr std::array windows_1257_names{"windows-1257"sv, "cp1257"sv}; +constexpr std::array windows_1258_names{"windows-1258"sv, "cp1258"sv}; +constexpr std::array x_mac_cyrillic_names{"x-mac-cyrillic"sv, "mac-cyrillic"sv}; + +constexpr std::array big5_names{"Big5"sv, "csbig5"sv, "big5-hkscs"sv, + "cn-big5"sv}; +constexpr std::array euc_jp_names{"EUC-JP"sv, "cseucpkdfmtjapanese"sv}; +constexpr std::array euc_kr_names{"EUC-KR"sv, "windows-949"sv, "cp949"sv, + "ks_c_5601-1987"sv}; +constexpr std::array gb18030_names{"gb18030"sv, "GBK"sv, "GB2312"sv, + "csgb2312"sv, "x-gbk"sv}; +constexpr std::array iso_2022_jp_names{"ISO-2022-JP"sv, "csiso2022jp"sv}; +constexpr std::array iso_2022_kr_names{"ISO-2022-KR"sv, "csiso2022kr"sv}; +constexpr std::array shift_jis_names{"Shift_JIS"sv, "sjis"sv, "windows-31j"sv, + "cp932"sv, "ms_kanji"sv}; + +constexpr std::array table{ + Row{TextEncoding::utf8, utf8_names, true}, + Row{TextEncoding::utf16le, utf16le_names, true}, + Row{TextEncoding::utf16be, utf16be_names, true}, + Row{TextEncoding::utf32le, utf32le_names, true}, + Row{TextEncoding::utf32be, utf32be_names, true}, + + Row{TextEncoding::ibm866, ibm866_names, true}, + Row{TextEncoding::iso_8859_1, iso_8859_1_names, true}, + Row{TextEncoding::iso_8859_2, iso_8859_2_names, true}, + Row{TextEncoding::iso_8859_3, iso_8859_3_names, true}, + Row{TextEncoding::iso_8859_4, iso_8859_4_names, true}, + Row{TextEncoding::iso_8859_5, iso_8859_5_names, true}, + Row{TextEncoding::iso_8859_6, iso_8859_6_names, true}, + Row{TextEncoding::iso_8859_7, iso_8859_7_names, true}, + Row{TextEncoding::iso_8859_8, iso_8859_8_names, true}, + Row{TextEncoding::iso_8859_10, iso_8859_10_names, true}, + Row{TextEncoding::iso_8859_13, iso_8859_13_names, true}, + Row{TextEncoding::iso_8859_14, iso_8859_14_names, true}, + Row{TextEncoding::iso_8859_15, iso_8859_15_names, true}, + Row{TextEncoding::iso_8859_16, iso_8859_16_names, true}, + Row{TextEncoding::koi8_r, koi8_r_names, true}, + Row{TextEncoding::koi8_u, koi8_u_names, true}, + Row{TextEncoding::macintosh, macintosh_names, true}, + Row{TextEncoding::windows_874, windows_874_names, true}, + Row{TextEncoding::windows_1250, windows_1250_names, true}, + Row{TextEncoding::windows_1251, windows_1251_names, true}, + Row{TextEncoding::windows_1252, windows_1252_names, true}, + Row{TextEncoding::windows_1253, windows_1253_names, true}, + Row{TextEncoding::windows_1254, windows_1254_names, true}, + Row{TextEncoding::windows_1255, windows_1255_names, true}, + Row{TextEncoding::windows_1256, windows_1256_names, true}, + Row{TextEncoding::windows_1257, windows_1257_names, true}, + Row{TextEncoding::windows_1258, windows_1258_names, true}, + Row{TextEncoding::x_mac_cyrillic, x_mac_cyrillic_names, true}, + + // decoding these needs a real converter; naming them still lets a browser + // do it + Row{TextEncoding::big5, big5_names, false}, + Row{TextEncoding::euc_jp, euc_jp_names, false}, + Row{TextEncoding::euc_kr, euc_kr_names, false}, + Row{TextEncoding::gb18030, gb18030_names, false}, + Row{TextEncoding::iso_2022_jp, iso_2022_jp_names, false}, + Row{TextEncoding::iso_2022_kr, iso_2022_kr_names, false}, + Row{TextEncoding::shift_jis, shift_jis_names, false}, +}; + +/// Case, `-`, `_` and spaces carry no meaning in an encoding label. +std::string normalize(const std::string_view name) { + std::string result; + result.reserve(name.size()); + for (const char c : name) { + if (c == '-' || c == '_' || c == ' ') { + continue; + } + result.push_back( + static_cast(std::tolower(static_cast(c)))); + } + return result; +} + +} // namespace + +std::span text_encoding_table::rows() noexcept { + return table; +} + +const text_encoding_table::Row * +text_encoding_table::find(const TextEncoding encoding) noexcept { + const auto it = std::ranges::find(table, encoding, &Row::encoding); + return it == std::ranges::end(table) ? nullptr : &*it; +} + +const text_encoding_table::Row * +text_encoding_table::find_by_name(const std::string_view name) noexcept { + const std::string needle = normalize(name); + const auto it = std::ranges::find_if(table, [&](const Row &row) { + return std::ranges::any_of(row.names, [&](const std::string_view alias) { + return normalize(alias) == needle; + }); + }); + return it == std::ranges::end(table) ? nullptr : &*it; +} + +} // namespace odr::internal::encoding diff --git a/src/odr/internal/encoding/text_encoding_table.hpp b/src/odr/internal/encoding/text_encoding_table.hpp new file mode 100644 index 000000000..81d518a9a --- /dev/null +++ b/src/odr/internal/encoding/text_encoding_table.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include +#include + +namespace odr::internal::encoding::text_encoding_table { + +/// Everything the library statically knows about one text encoding. +struct Row final { + TextEncoding encoding{TextEncoding::unknown}; + std::span names; ///< canonical one first + bool decodable{}; ///< @ref encoding::to_utf8 handles it +}; + +/// The whole table, one row per @ref TextEncoding, in declaration order. +[[nodiscard]] std::span rows() noexcept; + +/// The row for @p encoding, or `nullptr` if there is none. +[[nodiscard]] const Row *find(TextEncoding encoding) noexcept; + +/// The row listing @p name among its names, or `nullptr`. Matching ignores +/// case and any `-`, `_` or space, so `windows-1252`, `WINDOWS_1252` and +/// `cp1252` all land on the same row. +[[nodiscard]] const Row *find_by_name(std::string_view name) noexcept; + +} // namespace odr::internal::encoding::text_encoding_table diff --git a/src/odr/internal/encoding/transcode.cpp b/src/odr/internal/encoding/transcode.cpp new file mode 100644 index 000000000..a514cf6ca --- /dev/null +++ b/src/odr/internal/encoding/transcode.cpp @@ -0,0 +1,230 @@ +#include + +#include +#include + +#include +#include +#include + +#include +#include + +namespace odr::internal { + +namespace { + +using encoding::encoding_data::SingleByteTable; + +using namespace std::string_view_literals; + +constexpr char32_t replacement = 0xfffd; + +constexpr auto utf8_bom = "\xef\xbb\xbf"sv; +constexpr auto utf16le_bom = "\xff\xfe"sv; +constexpr auto utf16be_bom = "\xfe\xff"sv; +constexpr auto utf32le_bom = "\xff\xfe\x00\x00"sv; +constexpr auto utf32be_bom = "\x00\x00\xfe\xff"sv; + +bool starts_with(const std::string_view bytes, const std::string_view bom) { + return bytes.substr(0, bom.size()) == bom; +} + +const SingleByteTable *single_byte_table(const TextEncoding encoding) { + namespace data = encoding::encoding_data; + + switch (encoding) { + case TextEncoding::ibm866: + return &data::ibm866; + case TextEncoding::iso_8859_1: + return &data::iso_8859_1; + case TextEncoding::iso_8859_2: + return &data::iso_8859_2; + case TextEncoding::iso_8859_3: + return &data::iso_8859_3; + case TextEncoding::iso_8859_4: + return &data::iso_8859_4; + case TextEncoding::iso_8859_5: + return &data::iso_8859_5; + case TextEncoding::iso_8859_6: + return &data::iso_8859_6; + case TextEncoding::iso_8859_7: + return &data::iso_8859_7; + case TextEncoding::iso_8859_8: + return &data::iso_8859_8; + case TextEncoding::iso_8859_10: + return &data::iso_8859_10; + case TextEncoding::iso_8859_13: + return &data::iso_8859_13; + case TextEncoding::iso_8859_14: + return &data::iso_8859_14; + case TextEncoding::iso_8859_15: + return &data::iso_8859_15; + case TextEncoding::iso_8859_16: + return &data::iso_8859_16; + case TextEncoding::koi8_r: + return &data::koi8_r; + case TextEncoding::koi8_u: + return &data::koi8_u; + case TextEncoding::macintosh: + return &data::macintosh; + case TextEncoding::windows_874: + return &data::windows_874; + case TextEncoding::windows_1250: + return &data::windows_1250; + case TextEncoding::windows_1251: + return &data::windows_1251; + case TextEncoding::windows_1252: + return &data::windows_1252; + case TextEncoding::windows_1253: + return &data::windows_1253; + case TextEncoding::windows_1254: + return &data::windows_1254; + case TextEncoding::windows_1255: + return &data::windows_1255; + case TextEncoding::windows_1256: + return &data::windows_1256; + case TextEncoding::windows_1257: + return &data::windows_1257; + case TextEncoding::windows_1258: + return &data::windows_1258; + case TextEncoding::x_mac_cyrillic: + return &data::x_mac_cyrillic; + default: + return nullptr; + } +} + +/// The mark @p encoding defines, empty for the rest: `EF BB BF` read as +/// windows-1252 is three characters, not a mark. +std::string_view own_bom(const TextEncoding encoding) { + switch (encoding) { + case TextEncoding::utf8: + return utf8_bom; + case TextEncoding::utf16le: + return utf16le_bom; + case TextEncoding::utf16be: + return utf16be_bom; + case TextEncoding::utf32le: + return utf32le_bom; + case TextEncoding::utf32be: + return utf32be_bom; + default: + return {}; + } +} + +bool is_code_point_valid(const char32_t code_point) { + return code_point <= 0x10ffff && (code_point < 0xd800 || code_point > 0xdfff); +} + +std::uint8_t byte_at(const std::string_view bytes, const std::size_t index) { + return static_cast(bytes[index]); +} + +std::string decode_single_byte(const std::string_view bytes, + const SingleByteTable &table) { + std::string result; + result.reserve(bytes.size()); + for (std::size_t i = 0; i < bytes.size(); ++i) { + utf8::append(table[byte_at(bytes, i)], std::back_inserter(result)); + } + return result; +} + +std::string decode_utf16(const std::string_view bytes, const bool little) { + std::string result; + result.reserve(bytes.size()); + + const auto unit = [&](const std::size_t i) -> char32_t { + const auto low = byte_at(bytes, little ? i : i + 1); + const auto high = byte_at(bytes, little ? i + 1 : i); + return static_cast(high) << 8 | low; + }; + + std::size_t i = 0; + for (; i + 1 < bytes.size(); i += 2) { + const char32_t first = unit(i); + + if (first < 0xd800 || first > 0xdfff) { + utf8::append(first, std::back_inserter(result)); + continue; + } + // a low surrogate first, or a high one with nothing to pair with + if (first > 0xdbff || i + 3 >= bytes.size()) { + utf8::append(replacement, std::back_inserter(result)); + continue; + } + const char32_t second = unit(i + 2); + if (second < 0xdc00 || second > 0xdfff) { + utf8::append(replacement, std::back_inserter(result)); + continue; + } + utf8::append(0x10000 + ((first - 0xd800) << 10) + (second - 0xdc00), + std::back_inserter(result)); + i += 2; + } + + // a trailing odd byte cannot form a unit + if (i < bytes.size()) { + utf8::append(replacement, std::back_inserter(result)); + } + return result; +} + +std::string decode_utf32(const std::string_view bytes, const bool little) { + std::string result; + result.reserve(bytes.size() / 2); + + std::size_t i = 0; + for (; i + 3 < bytes.size(); i += 4) { + char32_t code_point = 0; + for (std::size_t b = 0; b < 4; ++b) { + code_point = code_point << 8 | byte_at(bytes, i + (little ? 3 - b : b)); + } + utf8::append(is_code_point_valid(code_point) ? code_point : replacement, + std::back_inserter(result)); + } + + if (i < bytes.size()) { + utf8::append(replacement, std::back_inserter(result)); + } + return result; +} + +} // namespace + +std::string encoding::to_utf8(const std::string_view bytes, + const TextEncoding encoding) { + const auto *row = text_encoding_table::find(encoding); + if (row == nullptr || !row->decodable) { + throw std::runtime_error("text encoding cannot be decoded"); + } + + const std::string_view bom = own_bom(encoding); + const std::string_view body = + starts_with(bytes, bom) ? bytes.substr(bom.size()) : bytes; + + switch (encoding) { + case TextEncoding::utf8: + return utf8::replace_invalid(body); + case TextEncoding::utf16le: + return decode_utf16(body, true); + case TextEncoding::utf16be: + return decode_utf16(body, false); + case TextEncoding::utf32le: + return decode_utf32(body, true); + case TextEncoding::utf32be: + return decode_utf32(body, false); + default: + break; + } + + const SingleByteTable *table = single_byte_table(encoding); + if (table == nullptr) { + throw std::runtime_error("text encoding cannot be decoded"); + } + return decode_single_byte(body, *table); +} + +} // namespace odr::internal diff --git a/src/odr/internal/encoding/transcode.hpp b/src/odr/internal/encoding/transcode.hpp new file mode 100644 index 000000000..809076458 --- /dev/null +++ b/src/odr/internal/encoding/transcode.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include + +#include +#include + +namespace odr::internal::encoding { + +/// Decodes @p bytes to UTF-8. Malformed input becomes U+FFFD rather than an +/// error — a wrong guess is the expected failure here, and the caller +/// overrides the encoding. +/// +/// @throws std::runtime_error if @p encoding is not decodable. +[[nodiscard]] std::string to_utf8(std::string_view bytes, + TextEncoding encoding); + +} // namespace odr::internal::encoding diff --git a/src/odr/internal/html/text_file.cpp b/src/odr/internal/html/text_file.cpp index 4ef8168f1..9b43ea4bc 100644 --- a/src/odr/internal/html/text_file.cpp +++ b/src/odr/internal/html/text_file.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -60,15 +61,30 @@ class HtmlServiceImpl final : public HtmlService { throw FileNotFound("Unknown path: " + path); } + /// The bytes to write and the charset to declare. What we can decode becomes + /// UTF-8; what we can only name passes through for the browser, which works + /// because those encodings are ASCII-compatible and the page is ASCII. + [[nodiscard]] std::pair body_and_charset() const { + const TextEncoding encoding = m_text_file.encoding(); + std::string text = m_text_file.text(); + + if (encoding == TextEncoding::unknown || + text_encoding_is_decodable(encoding)) { + return {std::move(text), "UTF-8"}; + } + return {std::move(text), std::string(text_encoding_to_string(encoding))}; + } + HtmlResources write_text(HtmlWriter &out) const { HtmlResources resources; + const auto [text, charset] = body_and_charset(); + out.write_begin(); out.write_header_begin(); - // TODO charset - out.write_header_charset("UTF-8"); + out.write_header_charset(charset); out.write_header_target("_blank"); out.write_header_title("odr"); write_viewport_meta(out, config(), false); @@ -83,14 +99,14 @@ class HtmlServiceImpl final : public HtmlService { out.write_element_begin("div", HtmlElementOptions().set_class("odr-text-nr")); - std::unique_ptr in = m_text_file.stream(); - for (std::uint32_t line = 1; !in->eof(); ++line) { + std::istringstream in(text); + for (std::uint32_t line = 1; !in.eof(); ++line) { out.write_element_begin("div", HtmlElementOptions().set_inline(true)); out.out() << line; out.write_element_end("div"); NullStream ss_out; - util::stream::pipe_line(*in, ss_out, false); + util::stream::pipe_line(in, ss_out, false); } out.write_element_end("div"); @@ -102,12 +118,12 @@ class HtmlServiceImpl final : public HtmlService { clb("contenteditable", "true"); } })); - in = m_text_file.stream(); - while (!in->eof()) { + in = std::istringstream(text); + while (!in.eof()) { out.write_element_begin("div", HtmlElementOptions().set_inline(true)); std::ostringstream ss_out; - util::stream::pipe_line(*in, ss_out, false); + util::stream::pipe_line(in, ss_out, false); if (std::string line = ss_out.str(); line.empty()) { out.write_element_begin( "br", HtmlElementOptions().set_close_type(HtmlCloseType::trailing)); diff --git a/src/odr/internal/json/json_file.cpp b/src/odr/internal/json/json_file.cpp index bfcbf4bd6..e9f7a2add 100644 --- a/src/odr/internal/json/json_file.cpp +++ b/src/odr/internal/json/json_file.cpp @@ -31,4 +31,6 @@ FileMeta JsonFile::file_meta() const noexcept { bool JsonFile::is_decodable() const noexcept { return false; } +TextEncoding JsonFile::encoding() const noexcept { return m_file->encoding(); } + } // namespace odr::internal::json diff --git a/src/odr/internal/json/json_file.hpp b/src/odr/internal/json/json_file.hpp index 1ef4889a8..c9c52ad9d 100644 --- a/src/odr/internal/json/json_file.hpp +++ b/src/odr/internal/json/json_file.hpp @@ -20,9 +20,10 @@ class JsonFile final : public abstract::TextFile { [[nodiscard]] bool is_decodable() const noexcept override; + [[nodiscard]] TextEncoding encoding() const noexcept override; + private: std::shared_ptr m_file; - std::string m_charset; }; } // namespace odr::internal::json diff --git a/src/odr/internal/text/text_file.cpp b/src/odr/internal/text/text_file.cpp index 7e8913cc9..69b91b193 100644 --- a/src/odr/internal/text/text_file.cpp +++ b/src/odr/internal/text/text_file.cpp @@ -1,14 +1,28 @@ #include -#include +#include +#include + +#include +#include +#include + +#include namespace odr::internal::text { TextFile::TextFile(std::shared_ptr file) - : m_file{std::move(file)}, m_charset{guess_charset(*m_file->stream())} {} + : m_file{std::move(file)} { + const std::unique_ptr in = m_file->stream(); + // nothing here rejects: text is the fallback for bytes nothing else claims, + // so a viewer handed a random binary shows junk rather than an error, and + // bytes we cannot name are `TextEncoding::unknown` + m_encoding = encoding::detect(encoding::read_probe(*in)); +} -TextFile::TextFile(std::shared_ptr file, std::string charset) - : m_file{std::move(file)}, m_charset{std::move(charset)} {} +TextFile::TextFile(std::shared_ptr file, + const TextEncoding encoding) + : m_file{std::move(file)}, m_encoding{encoding} {} std::shared_ptr TextFile::file() const noexcept { return m_file; @@ -27,4 +41,14 @@ FileMeta TextFile::file_meta() const noexcept { bool TextFile::is_decodable() const noexcept { return false; } +TextEncoding TextFile::encoding() const noexcept { return m_encoding; } + +std::string TextFile::text() const { + if (!text_encoding_is_decodable(m_encoding)) { + throw UnsupportedTextEncoding(m_encoding); + } + const std::unique_ptr in = m_file->stream(); + return encoding::to_utf8(util::stream::read(*in), m_encoding); +} + } // namespace odr::internal::text diff --git a/src/odr/internal/text/text_file.hpp b/src/odr/internal/text/text_file.hpp index ff77487a1..0111a6956 100644 --- a/src/odr/internal/text/text_file.hpp +++ b/src/odr/internal/text/text_file.hpp @@ -11,7 +11,7 @@ namespace odr::internal::text { class TextFile final : public abstract::TextFile { public: explicit TextFile(std::shared_ptr file); - TextFile(std::shared_ptr file, std::string charset); + TextFile(std::shared_ptr file, TextEncoding encoding); [[nodiscard]] std::shared_ptr file() const noexcept override; @@ -21,9 +21,15 @@ class TextFile final : public abstract::TextFile { [[nodiscard]] bool is_decodable() const noexcept override; + [[nodiscard]] TextEncoding encoding() const noexcept override; + + /// The file's bytes decoded to UTF-8. + /// @throws UnsupportedTextEncoding if the encoding cannot be decoded. + [[nodiscard]] std::string text() const; + private: std::shared_ptr m_file; - std::string m_charset; + TextEncoding m_encoding{TextEncoding::unknown}; }; } // namespace odr::internal::text diff --git a/src/odr/internal/text/text_util.cpp b/src/odr/internal/text/text_util.cpp deleted file mode 100644 index 68c4499fd..000000000 --- a/src/odr/internal/text/text_util.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include - -#include - -#include -#include - -#include - -namespace odr::internal { - -std::string text::guess_charset(std::istream &in) { - const auto ud = uchardet_new(); - std::array buffer{}; - - while (true) { - in.read(buffer.data(), static_cast(buffer.size())); - const auto read = in.gcount(); - if (read == 0) { - break; - } - uchardet_handle_data(ud, buffer.data(), read); - } - - uchardet_data_end(ud); - std::string result = uchardet_get_charset(ud); - uchardet_delete(ud); - - if (result.empty()) { - throw UnknownCharset(); - } - return result; -} - -} // namespace odr::internal diff --git a/src/odr/internal/text/text_util.hpp b/src/odr/internal/text/text_util.hpp deleted file mode 100644 index 64d9f475b..000000000 --- a/src/odr/internal/text/text_util.hpp +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include -#include - -namespace odr::internal::text { - -// TODO pass max read distance -std::string guess_charset(std::istream &in); - -} // namespace odr::internal::text diff --git a/src/odr/odr.cpp b/src/odr/odr.cpp index ff026fd1b..247d80ab5 100644 --- a/src/odr/odr.cpp +++ b/src/odr/odr.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,9 @@ namespace { using odr::internal::file_type_table::Row; namespace table = odr::internal::file_type_table; + +using EncodingRow = odr::internal::encoding::text_encoding_table::Row; +namespace encoding_table = odr::internal::encoding::text_encoding_table; } // namespace std::string odr::version() { @@ -145,6 +149,39 @@ odr::capabilities_by_file_type(const FileType type) noexcept { return row == nullptr ? FileTypeCapabilities{} : row->capabilities; } +std::vector odr::all_text_encodings() { + std::vector result; + result.reserve(encoding_table::rows().size()); + std::ranges::transform(encoding_table::rows(), std::back_inserter(result), + &EncodingRow::encoding); + return result; +} + +std::string_view odr::text_encoding_to_string(const TextEncoding encoding) { + const EncodingRow *row = encoding_table::find(encoding); + if (row == nullptr) { + throw UnsupportedTextEncoding(encoding); + } + return row->names.front(); +} + +odr::TextEncoding +odr::text_encoding_by_name(const std::string_view name) noexcept { + const EncodingRow *row = encoding_table::find_by_name(name); + return row == nullptr ? TextEncoding::unknown : row->encoding; +} + +std::span +odr::text_encoding_names(const TextEncoding encoding) noexcept { + const EncodingRow *row = encoding_table::find(encoding); + return row == nullptr ? std::span{} : row->names; +} + +bool odr::text_encoding_is_decodable(const TextEncoding encoding) noexcept { + const EncodingRow *row = encoding_table::find(encoding); + return row != nullptr && row->decodable; +} + std::vector odr::list_file_types(const File &file, const Logger &logger) { return DecodedFile::list_file_types(file, logger); diff --git a/src/odr/odr.hpp b/src/odr/odr.hpp index 9b3a62616..6291883df 100644 --- a/src/odr/odr.hpp +++ b/src/odr/odr.hpp @@ -64,6 +64,26 @@ mimetypes_by_file_type(FileType type) noexcept; [[nodiscard]] FileTypeCapabilities capabilities_by_file_type(FileType type) noexcept; +/// @brief Every text encoding this library knows about, in declaration order, +/// excluding @ref TextEncoding::unknown. +[[nodiscard]] std::vector all_text_encodings(); + +/// @brief The text encoding's canonical name, a label a browser accepts. +/// @throws UnsupportedTextEncoding for @ref TextEncoding::unknown. +[[nodiscard]] std::string_view text_encoding_to_string(TextEncoding encoding); +/// @brief The text encoding for a name, @ref TextEncoding::unknown if none. +/// +/// Case and any `-`, `_` or space are ignored, so `windows-1252`, +/// `WINDOWS_1252` and `cp1252` all name the same encoding. +[[nodiscard]] TextEncoding +text_encoding_by_name(std::string_view name) noexcept; +/// @brief Every name accepted for the text encoding, canonical one first. +[[nodiscard]] std::span +text_encoding_names(TextEncoding encoding) noexcept; +/// @brief Whether the library can decode the text encoding, as opposed to +/// merely naming it. +[[nodiscard]] bool text_encoding_is_decodable(TextEncoding encoding) noexcept; + /// @brief The file types detected for @p file. [[nodiscard]] std::vector list_file_types(const File &file, const Logger &logger = Logger::null()); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7abfe850b..66be0c470 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -52,6 +52,7 @@ add_executable(odr_test "src/internal/crypto/crypto_util_test.cpp" "src/internal/csv/csv_file_test.cpp" + "src/internal/encoding/text_encoding_test.cpp" "src/internal/oldms/doc_test.cpp" "src/internal/oldms/ppt_test.cpp" diff --git a/test/src/internal/encoding/text_encoding_test.cpp b/test/src/internal/encoding/text_encoding_test.cpp new file mode 100644 index 000000000..cf3e76d5e --- /dev/null +++ b/test/src/internal/encoding/text_encoding_test.cpp @@ -0,0 +1,150 @@ +#include +#include +#include + +#include +#include + +#include + +#include +#include + +using namespace odr; +using namespace odr::internal; + +TEST(TextEncoding, every_encoding_has_a_name) { + for (const TextEncoding text_encoding : all_text_encodings()) { + EXPECT_FALSE(text_encoding_to_string(text_encoding).empty()); + EXPECT_FALSE(text_encoding_names(text_encoding).empty()); + } +} + +TEST(TextEncoding, unknown_has_no_name) { + EXPECT_THROW((void)text_encoding_to_string(TextEncoding::unknown), + UnsupportedTextEncoding); + EXPECT_TRUE(text_encoding_names(TextEncoding::unknown).empty()); + EXPECT_FALSE(text_encoding_is_decodable(TextEncoding::unknown)); +} + +/// The lookups take the first match, so a name on two encodings would make one +/// of them unreachable. +TEST(TextEncoding, no_name_is_claimed_twice) { + std::set seen; + for (const TextEncoding text_encoding : all_text_encodings()) { + for (const std::string_view name : text_encoding_names(text_encoding)) { + std::string normalized; + for (const char c : name) { + if (c != '-' && c != '_') { + normalized.push_back( + static_cast(std::tolower(static_cast(c)))); + } + } + EXPECT_TRUE(seen.insert(normalized).second) + << "name claimed twice: " << name; + } + } +} + +TEST(TextEncoding, every_name_finds_its_encoding) { + for (const TextEncoding text_encoding : all_text_encodings()) { + for (const std::string_view name : text_encoding_names(text_encoding)) { + EXPECT_EQ(text_encoding_by_name(name), text_encoding) << name; + } + } +} + +TEST(TextEncoding, a_name_matches_regardless_of_case_and_punctuation) { + EXPECT_EQ(text_encoding_by_name("windows-1252"), TextEncoding::windows_1252); + EXPECT_EQ(text_encoding_by_name("WINDOWS_1252"), TextEncoding::windows_1252); + EXPECT_EQ(text_encoding_by_name("cp1252"), TextEncoding::windows_1252); + EXPECT_EQ(text_encoding_by_name("utf8"), TextEncoding::utf8); + EXPECT_EQ(text_encoding_by_name("Latin1"), TextEncoding::iso_8859_1); + EXPECT_EQ(text_encoding_by_name("nonsense"), TextEncoding::unknown); +} + +TEST(TextEncoding, uchardet_names_map_home) { + // the spellings uchardet reports, which are what `detect` has to resolve + for (const std::string_view name : + {"UTF-8", "ASCII", "ISO-8859-1", "WINDOWS-1252", "KOI8-R", "SHIFT_JIS", + "EUC-JP", "Big5", "IBM866", "TIS-620"}) { + EXPECT_NE(text_encoding_by_name(name), TextEncoding::unknown) << name; + } +} + +TEST(Transcode, single_byte) { + EXPECT_EQ(encoding::to_utf8("caf\xe9", TextEncoding::iso_8859_1), "café"); + EXPECT_EQ(encoding::to_utf8("caf\xe9", TextEncoding::windows_1252), "café"); + // the euro sits where latin-1 has nothing + EXPECT_EQ(encoding::to_utf8("\x80", TextEncoding::windows_1252), "€"); + EXPECT_EQ(encoding::to_utf8("\xa4", TextEncoding::iso_8859_15), "€"); +} + +/// windows-1252 leaves a handful of codes undefined; they are not an error. +TEST(Transcode, an_undefined_byte_becomes_the_replacement_character) { + EXPECT_EQ(encoding::to_utf8("\x81", TextEncoding::windows_1252), "�"); +} + +TEST(Transcode, utf8_passes_through_and_repairs) { + EXPECT_EQ(encoding::to_utf8("café", TextEncoding::utf8), "café"); + EXPECT_EQ(encoding::to_utf8("\xef\xbb\xbfhi", TextEncoding::utf8), "hi"); + EXPECT_EQ(encoding::to_utf8("a\xff" + "b", + TextEncoding::utf8), + "a�b"); +} + +TEST(Transcode, utf16) { + EXPECT_EQ(encoding::to_utf8(std::string("h\0i\0", 4), TextEncoding::utf16le), + "hi"); + EXPECT_EQ(encoding::to_utf8(std::string("\0h\0i", 4), TextEncoding::utf16be), + "hi"); + EXPECT_EQ(encoding::to_utf8(std::string("\xff\xfeh\0i\0", 6), + TextEncoding::utf16le), + "hi"); + // a surrogate pair, U+1F600 + EXPECT_EQ(encoding::to_utf8(std::string("\x3d\xd8\x00\xde", 4), + TextEncoding::utf16le), + "😀"); +} + +TEST(Transcode, a_broken_utf16_unit_becomes_the_replacement_character) { + // an unpaired high surrogate, and a trailing odd byte + EXPECT_EQ( + encoding::to_utf8(std::string("\x3d\xd8h\0", 4), TextEncoding::utf16le), + "�h"); + EXPECT_EQ(encoding::to_utf8(std::string("h\0!", 3), TextEncoding::utf16le), + "h�"); +} + +TEST(Transcode, utf32) { + EXPECT_EQ(encoding::to_utf8(std::string("h\0\0\0i\0\0\0", 8), + TextEncoding::utf32le), + "hi"); + EXPECT_EQ(encoding::to_utf8(std::string("\0\0\0h\0\0\0i", 8), + TextEncoding::utf32be), + "hi"); +} + +/// A mark is only a mark for the encoding that defines it — read as +/// windows-1252 those same bytes are three real characters. +TEST(Transcode, a_byte_order_mark_is_stripped_only_by_its_own_encoding) { + EXPECT_EQ(encoding::to_utf8("\xef\xbb\xbf", TextEncoding::utf8), ""); + EXPECT_EQ(encoding::to_utf8("\xef\xbb\xbf", TextEncoding::windows_1252), + ""); +} + +TEST(Transcode, an_undecodable_encoding_throws) { + EXPECT_THROW((void)encoding::to_utf8("x", TextEncoding::shift_jis), + std::runtime_error); + EXPECT_THROW((void)encoding::to_utf8("x", TextEncoding::unknown), + std::runtime_error); +} + +TEST(Detect, a_byte_order_mark_wins) { + EXPECT_EQ(encoding::detect("\xef\xbb\xbfhello"), TextEncoding::utf8); + EXPECT_EQ(encoding::detect(std::string("\xff\xfe\0\0", 4)), + TextEncoding::utf32le); + EXPECT_EQ(encoding::detect(std::string("\xff\xfeh\0", 4)), + TextEncoding::utf16le); +} diff --git a/test/src/internal/text/text_file_test.cpp b/test/src/internal/text/text_file_test.cpp index 26e22ea73..03c73689e 100644 --- a/test/src/internal/text/text_file_test.cpp +++ b/test/src/internal/text/text_file_test.cpp @@ -1,5 +1,6 @@ #include #include +#include #include @@ -7,14 +8,66 @@ #include +#include +#include + using namespace odr; using namespace odr::test; -TEST(TextFile, odt) { - const File file(TestData::test_file_path("odr-public/odt/about.odt")); - EXPECT_THROW(internal::text::TextFile(file.impl()), UnknownCharset); +namespace { + +internal::text::TextFile text_file(const std::string &content) { + return internal::text::TextFile(File::from_memory(content).impl()); } +} // namespace + TEST(TextFile, txt) { File(TestData::test_file_path("odr-public/txt/lorem ipsum.txt")); } + +/// Text is the fallback for bytes nothing else claims, so nothing is rejected +/// — a viewer handed a random binary shows junk rather than an error. +TEST(TextFile, anything_reads_as_text) { + EXPECT_NO_THROW(text_file(std::string("\0\1\2\3", 4))); + EXPECT_NO_THROW(text_file("plain ascii")); + + const File odt(TestData::test_file_path("odr-public/odt/about.odt")); + EXPECT_NO_THROW(internal::text::TextFile(odt.impl())); +} + +/// The same contract seen from outside, which `wasm/tests/smoke.test.mjs` +/// pins downstream: bytes nothing recognises still open, as text. +TEST(TextFile, unrecognised_bytes_open_as_text) { + const File junk = File::from_memory(std::string("\0\1\2\3\4\5\6\7", 8)); + + EXPECT_EQ(mimetype(junk), "text/plain"); + EXPECT_EQ(DecodedFile(junk).file_type(), FileType::text_file); +} + +TEST(TextFile, encoding_comes_from_the_byte_order_mark) { + EXPECT_EQ(text_file("\xef\xbb\xbfhello").encoding(), TextEncoding::utf8); + EXPECT_EQ(text_file(std::string("\xff\xfeh\0i\0", 6)).encoding(), + TextEncoding::utf16le); + EXPECT_EQ(text_file(std::string("\xfe\xff\0h\0i", 6)).encoding(), + TextEncoding::utf16be); + EXPECT_EQ(text_file(std::string("\xff\xfe\0\0h\0\0\0", 8)).encoding(), + TextEncoding::utf32le); +} + +TEST(TextFile, text_is_decoded_to_utf8) { + // 0xe9 is `é` in latin-1 and not valid utf-8, so the encoding decides + const TextFile file(std::make_shared( + File::from_memory("caf\xe9").impl(), TextEncoding::iso_8859_1)); + EXPECT_EQ(file.text(), "café"); +} + +/// Nothing decodes these, so the bytes come back as they are and the caller +/// hands them to something that does. +TEST(TextFile, an_undecodable_encoding_yields_its_bytes) { + const std::string content = "\x82\xa0\x82\xa2"; + const TextFile file(std::make_shared( + File::from_memory(content).impl(), TextEncoding::shift_jis)); + EXPECT_FALSE(text_encoding_is_decodable(TextEncoding::shift_jis)); + EXPECT_EQ(file.text(), content); +} diff --git a/tools/encoding/generate_encoding_data.py b/tools/encoding/generate_encoding_data.py new file mode 100644 index 000000000..e85760a36 --- /dev/null +++ b/tools/encoding/generate_encoding_data.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Generates the single-byte decoding tables in `internal/encoding`. + +Each supported single-byte encoding becomes a 256-entry byte -> code point +table. The mappings come from Python's own codecs, which carry the standard +data; codes the encoding leaves undefined become U+FFFD. + +The set mirrors the WHATWG Encoding Standard's single-byte encodings, i.e. what +browsers implement, plus a true ISO-8859-1 (WHATWG deliberately folds that one +into windows-1252; we do not, because we report what we detected). + +Regenerate with: + + python3 tools/encoding/generate_encoding_data.py +""" + +from __future__ import annotations + +import pathlib +import sys + +REPLACEMENT = 0xFFFD + +# (identifier in the generated code, Python codec name) +ENCODINGS = [ + ("ibm866", "cp866"), + ("iso_8859_1", "iso8859_1"), + ("iso_8859_2", "iso8859_2"), + ("iso_8859_3", "iso8859_3"), + ("iso_8859_4", "iso8859_4"), + ("iso_8859_5", "iso8859_5"), + ("iso_8859_6", "iso8859_6"), + ("iso_8859_7", "iso8859_7"), + ("iso_8859_8", "iso8859_8"), + ("iso_8859_10", "iso8859_10"), + ("iso_8859_13", "iso8859_13"), + ("iso_8859_14", "iso8859_14"), + ("iso_8859_15", "iso8859_15"), + ("iso_8859_16", "iso8859_16"), + ("koi8_r", "koi8_r"), + ("koi8_u", "koi8_u"), + ("macintosh", "mac_roman"), + ("windows_874", "cp874"), + ("windows_1250", "cp1250"), + ("windows_1251", "cp1251"), + ("windows_1252", "cp1252"), + ("windows_1253", "cp1253"), + ("windows_1254", "cp1254"), + ("windows_1255", "cp1255"), + ("windows_1256", "cp1256"), + ("windows_1257", "cp1257"), + ("windows_1258", "cp1258"), + ("x_mac_cyrillic", "mac_cyrillic"), +] + +HEADER_NOTE = """// Auto-generated by tools/encoding/generate_encoding_data.py -- DO NOT EDIT. +// Regenerate with: python3 tools/encoding/generate_encoding_data.py +// Data: the Python standard library's codecs. + +// clang-format off +""" + + +def table_for(codec: str) -> list[int]: + result = [] + for byte in range(256): + try: + decoded = bytes([byte]).decode(codec) + except UnicodeDecodeError: + result.append(REPLACEMENT) + continue + # a few codecs map one byte to a combining sequence; none of ours do, + # and silently truncating would be a wrong table + if len(decoded) != 1: + raise ValueError(f"{codec}: byte {byte:#04x} decodes to {decoded!r}") + result.append(ord(decoded)) + return result + + +def write_header(path: pathlib.Path) -> None: + lines = [ + HEADER_NOTE, + "#pragma once", + "", + "#include ", + "", + "namespace odr::internal::encoding::encoding_data {", + "", + "/// Byte -> code point for one single-byte encoding; U+FFFD where the", + "/// encoding leaves the byte undefined.", + "using SingleByteTable = std::array;", + "", + ] + lines += [f"extern const SingleByteTable {name};" for name, _ in ENCODINGS] + lines += ["", "} // namespace odr::internal::encoding::encoding_data", ""] + path.write_text("\n".join(lines)) + + +def write_source(path: pathlib.Path) -> None: + lines = [ + HEADER_NOTE, + "#include ", + "", + "namespace odr::internal::encoding::encoding_data {", + "", + ] + for name, codec in ENCODINGS: + table = table_for(codec) + lines.append(f"const SingleByteTable {name}{{") + for start in range(0, 256, 8): + row = ", ".join(f"0x{cp:04x}" for cp in table[start : start + 8]) + lines.append(f" {row},") + lines.append("};") + lines.append("") + lines += ["} // namespace odr::internal::encoding::encoding_data", ""] + path.write_text("\n".join(lines)) + + +def main() -> int: + root = pathlib.Path(__file__).resolve().parents[2] + out = root / "src" / "odr" / "internal" / "encoding" + out.mkdir(parents=True, exist_ok=True) + write_header(out / "encoding_data.hpp") + write_source(out / "encoding_data.cpp") + print(f"wrote {len(ENCODINGS)} tables to {out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())