diff --git a/AGENTS.md b/AGENTS.md index 03af3d120..a8270b69f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,6 +62,7 @@ bytes ─▶ magic/open_strategy ─▶ DecodedFile ─▶ Document ─▶ Eleme | `src/odr/internal/ooxml/` | OOXML (docx/pptx/xlsx); see [`ooxml/AGENTS.md`](src/odr/internal/ooxml/AGENTS.md) + per-format docs. | | `src/odr/internal/oldms/` | **Legacy MS binary** (.doc/.ppt/.xls). | | `src/odr/internal/pdf/` | PDF (own parser). | +| `src/odr/internal/xml/` | XML, rendered as a source view; see [`xml/AGENTS.md`](src/odr/internal/xml/AGENTS.md). | | `src/odr/internal/{csv,json,text,svm}/` | Smaller formats. | | `cli/src/` | CLI tools: `translate`, `back_translate`, `meta`, `server`. | | `python/` | Python bindings (`pyodr`, pybind11); see [`python/AGENTS.md`](python/AGENTS.md). | diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d69f063b..c0f152038 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ The release run heads these entries with the version and opens a fresh ## Unreleased +- An xml file opens as xml and reads as a foldable, highlighted source view + rather than as one very long line, in the encoding its declaration names. + ## v6.4.0 - 2026-08-09 - csv opens as a spreadsheet, its dialect probed unless the caller gives one, diff --git a/CMakeLists.txt b/CMakeLists.txt index 464dd92fd..6d1b30001 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -148,6 +148,7 @@ set(ODR_SOURCE_FILES "src/odr/internal/html/media_file.cpp" "src/odr/internal/html/pdf_file.cpp" "src/odr/internal/html/text_file.cpp" + "src/odr/internal/html/xml_file.cpp" "src/odr/internal/json/json_file.cpp" "src/odr/internal/json/json_util.cpp" @@ -254,6 +255,8 @@ set(ODR_SOURCE_FILES "src/odr/internal/util/string_util.cpp" "src/odr/internal/util/xml_util.cpp" + "src/odr/internal/xml/xml_file.cpp" + "src/odr/internal/zip/zip_archive.cpp" "src/odr/internal/zip/zip_exceptions.cpp" "src/odr/internal/zip/zip_file.cpp" diff --git a/src/odr/file.cpp b/src/odr/file.cpp index 2a13d4885..e64fc15e1 100644 --- a/src/odr/file.cpp +++ b/src/odr/file.cpp @@ -305,6 +305,10 @@ std::string TextFile::text() const { return internal::encoding::to_utf8(bytes, encoding); } +std::shared_ptr TextFile::impl() const { + return m_impl; +} + CsvFile CsvFile::from_file(const File &file, const CsvOptions &options, const Logger &logger) { ODR_VERBOSE(logger, "open as csv with options"); diff --git a/src/odr/file.hpp b/src/odr/file.hpp index 0b8b77361..8343a95e7 100644 --- a/src/odr/file.hpp +++ b/src/odr/file.hpp @@ -144,8 +144,8 @@ enum class FileType { // https://en.wikipedia.org/wiki/Windows_Metafile#Enhanced_Metafile enhanced_metafile, - // Classification only - reported under the formats built on it (an svg comes - // back as `[text_file, xml, scalable_vector_graphics]`), no decoder yet. + // Also reported under the formats built on it - an svg comes back as + // `[text_file, xml, scalable_vector_graphics]`. // https://en.wikipedia.org/wiki/XML xml, }; @@ -408,6 +408,8 @@ class TextFile final : public DecodedFile { /// decodable, and the raw bytes where it is not. [[nodiscard]] std::string text() const; + [[nodiscard]] std::shared_ptr impl() const; + private: std::shared_ptr m_impl; }; diff --git a/src/odr/html.cpp b/src/odr/html.cpp index 2eb2a1072..8fea45503 100644 --- a/src/odr/html.cpp +++ b/src/odr/html.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -215,6 +216,12 @@ HtmlService html::translate(const DecodedFile &file, const HtmlConfig &config, if (file.is_csv_file()) { return translate(file.as_csv_file().document(), config, logger); } + // and before it for the same reason. Translating it as a text file by hand + // still writes the line list. + if (file.file_type() == FileType::xml) { + return internal::html::create_xml_service(file.as_text_file(), config, + logger); + } if (file.is_text_file()) { return translate(file.as_text_file(), config, logger); } diff --git a/src/odr/internal/file_type_table.cpp b/src/odr/internal/file_type_table.cpp index b1c262aac..c5063b54c 100644 --- a/src/odr/internal/file_type_table.cpp +++ b/src/odr/internal/file_type_table.cpp @@ -659,15 +659,14 @@ constexpr std::array table{ DocumentType::unknown, {.detect_by_content = true, .open = true, .translate_html = true}}, - // Detection reports it, nothing opens it yet - a plain xml file still - // decodes as text. + // Not decoded: it renders as a source view of itself. Row{FileType::xml, "xml"sv, xml_extensions, xml_mimetypes, FileCategory::text, DocumentType::unknown, - {.detect_by_content = true}}, + {.detect_by_content = true, .open = true, .translate_html = true}}, }; /// Finds the row whose list, selected by @p list, contains @p needle. diff --git a/src/odr/internal/html/frontend.cpp b/src/odr/internal/html/frontend.cpp index 3de7a2c48..a6ab74007 100644 --- a/src/odr/internal/html/frontend.cpp +++ b/src/odr/internal/html/frontend.cpp @@ -88,6 +88,41 @@ constexpr std::string_view text_css = R"css( [contenteditable]:focus{outline:none} )css"; +/// No numbered gutter - the numbers would be ours, not the file's. The column +/// carries the fold handles, and every line reserves it. +constexpr std::string_view xml_css = R"css( +:root{ +--odr-xml-text:#1f2328; +--odr-xml-muted:#6e7781; +--odr-xml-punct:#57606a; +--odr-xml-name:#116329; +--odr-xml-attr:#953800; +--odr-xml-value:#0a3069; +--odr-xml-meta:#8250df; +--odr-xml-mono:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace; +} +body{background:#fff} +.odr-xml{color:var(--odr-xml-text);font:13px/1.6 var(--odr-xml-mono);word-break:break-word;overflow-wrap:anywhere} +.odr-xml-line,.odr-xml summary{padding-left:1.5em} +.odr-xml summary{display:block;position:relative;list-style:none;cursor:pointer} +.odr-xml summary::-webkit-details-marker{display:none} +/* U+25BE, U+25B8 */ +.odr-xml summary::before{content:"\25BE";position:absolute;left:.35em;color:var(--odr-xml-muted)} +.odr-xml details:not([open])>summary::before{content:"\25B8"} +.odr-xml summary:hover{background:rgba(0,0,0,.04)} +/* Indentation is spaces, not padding, so a copy of the page carries it. */ +.odr-xml-indent{white-space:pre} +.odr-xml-tag{color:var(--odr-xml-punct)} +.odr-xml-name{color:var(--odr-xml-name)} +.odr-xml-attr{color:var(--odr-xml-attr)} +.odr-xml-value{color:var(--odr-xml-value)} +/* Source text, everywhere it is written - html would fold a run of spaces. */ +.odr-xml-text,.odr-xml-cdata,.odr-xml-comment,.odr-xml-value,.odr-xml-decl,.odr-xml-doctype,.odr-xml-pi{white-space:pre-wrap} +.odr-xml-cdata{color:var(--odr-xml-value)} +.odr-xml-comment{color:var(--odr-xml-muted)} +.odr-xml-decl,.odr-xml-doctype,.odr-xml-pi{color:var(--odr-xml-meta)} +)css"; + constexpr std::string_view filesystem_css = R"css( :root{ --odr-files-line:#e3e5e8; @@ -892,6 +927,8 @@ constexpr Asset spreadsheet_css_asset{HtmlResourceType::css, "text/css", "spreadsheet.css", spreadsheet_css}; constexpr Asset text_css_asset{HtmlResourceType::css, "text/css", "text.css", text_css}; +constexpr Asset xml_css_asset{HtmlResourceType::css, "text/css", "xml.css", + xml_css}; constexpr Asset filesystem_css_asset{HtmlResourceType::css, "text/css", "filesystem.css", filesystem_css}; constexpr Asset media_css_asset{HtmlResourceType::css, "text/css", "media.css", @@ -968,6 +1005,10 @@ void html::write_text_style(const WritingState &state) { write_style(text_css_asset, state); } +void html::write_xml_style(const WritingState &state) { + write_style(xml_css_asset, state); +} + void html::write_filesystem_style(const WritingState &state) { write_style(filesystem_css_asset, state); } @@ -993,6 +1034,11 @@ HtmlResources html::locate_text_resources(const HtmlConfig &config) { return locate_all(assets, config); } +HtmlResources html::locate_xml_resources(const HtmlConfig &config) { + static constexpr std::array assets{xml_css_asset}; + return locate_all(assets, config); +} + HtmlResources html::locate_media_resources(const HtmlConfig &config) { static constexpr std::array assets{media_css_asset}; return locate_all(assets, config); diff --git a/src/odr/internal/html/frontend.hpp b/src/odr/internal/html/frontend.hpp index baed98e5e..4b433bb07 100644 --- a/src/odr/internal/html/frontend.hpp +++ b/src/odr/internal/html/frontend.hpp @@ -13,6 +13,7 @@ void write_document_style(const WritingState &state); /// Written in addition to the document style. void write_spreadsheet_style(const WritingState &state); void write_text_style(const WritingState &state); +void write_xml_style(const WritingState &state); void write_filesystem_style(const WritingState &state); void write_media_style(const WritingState &state); @@ -27,6 +28,7 @@ void write_text_script(const WritingState &state); /// a service has to answer for these paths as well as for its views. Every /// entry is located `nullopt` when the config embeds them. HtmlResources locate_text_resources(const HtmlConfig &config); +HtmlResources locate_xml_resources(const HtmlConfig &config); HtmlResources locate_media_resources(const HtmlConfig &config); } // namespace odr::internal::html diff --git a/src/odr/internal/html/xml_file.cpp b/src/odr/internal/html/xml_file.cpp new file mode 100644 index 000000000..cb7daf1fe --- /dev/null +++ b/src/odr/internal/html/xml_file.cpp @@ -0,0 +1,306 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace odr::internal::html { +namespace { + +/// Not `escape_text`, which folds a run of spaces into ` ` and a tab into +/// ` `. +std::string escape_source(std::string text) { + util::string::replace_all(text, "&", "&"); + util::string::replace_all(text, "<", "<"); + util::string::replace_all(text, ">", ">"); + return text; +} + +void write_span(std::ostream &out, const std::string_view clazz, + const std::string &content) { + out << "" << content << ""; +} + +void write_indent(std::ostream &out, const std::uint32_t depth) { + if (depth == 0) { + return; + } + write_span(out, "odr-xml-indent", std::string(std::size_t{2} * depth, ' ')); +} + +/// `name="value"`, single-quoted where that is the delimiter the value does not +/// carry; a value carrying both takes the double quote as `"`. The file's +/// own quoting is not in the tree. +void write_attributes(std::ostream &out, const pugi::xml_node &node) { + for (const pugi::xml_attribute &attribute : node.attributes()) { + out << " "; + write_span(out, "odr-xml-attr", escape_source(attribute.name())); + out << "="; + + const std::string value = attribute.value(); + const bool single_quoted = value.find('"') != std::string::npos && + value.find('\'') == std::string::npos; + const char quote = single_quoted ? '\'' : '"'; + std::string content = escape_source(value); + if (!single_quoted) { + util::string::replace_all(content, "\"", """); + } + write_span(out, "odr-xml-value", quote + content + quote); + } +} + +void write_start_tag(std::ostream &out, const pugi::xml_node &node, + const bool self_closing) { + out << "<"; + write_span(out, "odr-xml-name", escape_source(node.name())); + write_attributes(out, node); + out << (self_closing ? "/>" : ">"); + out << ""; +} + +void write_end_tag(std::ostream &out, const pugi::xml_node &node) { + out << "</"; + write_span(out, "odr-xml-name", escape_source(node.name())); + out << ">"; +} + +/// One node and, for an element, everything under it, on a single line. +void write_inline(std::ostream &out, const pugi::xml_node &node) { + switch (node.type()) { + case pugi::node_element: + if (!node.first_child()) { + write_start_tag(out, node, true); + return; + } + write_start_tag(out, node, false); + for (const pugi::xml_node &child : node.children()) { + write_inline(out, child); + } + write_end_tag(out, node); + return; + case pugi::node_pcdata: + write_span(out, "odr-xml-text", escape_source(node.value())); + return; + case pugi::node_cdata: + write_span(out, "odr-xml-cdata", + "<![CDATA[" + escape_source(node.value()) + "]]>"); + return; + case pugi::node_comment: + write_span(out, "odr-xml-comment", + "<!--" + escape_source(node.value()) + "-->"); + return; + case pugi::node_pi: { + std::string content = "<?" + escape_source(node.name()); + if (const std::string value = node.value(); !value.empty()) { + content += " " + escape_source(value); + } + content += "?>"; + write_span(out, "odr-xml-pi", content); + return; + } + case pugi::node_declaration: + out << "<?"; + out << escape_source(node.name()); + write_attributes(out, node); + out << "?>"; + return; + case pugi::node_doctype: + write_span(out, "odr-xml-doctype", + "<!DOCTYPE " + escape_source(node.value()) + ">"); + return; + default: + return; + } +} + +template +void write_line(HtmlWriter &out, const std::uint32_t depth, + const Content &content) { + out.write_element_begin( + "div", HtmlElementOptions().set_inline(true).set_class("odr-xml-line")); + write_indent(out.out(), depth); + content(out.out()); + out.write_element_end("div"); +} + +void write_node(HtmlWriter &out, const pugi::xml_node &node, + std::uint32_t depth); + +/// Nothing short of a schema tells significant whitespace from the other kind, +/// so an element holding any text keeps the shape it came in. +bool has_text_child(const pugi::xml_node &node) { + return std::ranges::any_of(node.children(), [](const pugi::xml_node &child) { + const pugi::xml_node_type type = child.type(); + return type == pugi::node_pcdata || type == pugi::node_cdata; + }); +} + +/// Open, always: collapsed by default hides what the file was opened to see, +/// and costs find-in-page the section it would expand into. +void write_element(HtmlWriter &out, const pugi::xml_node &node, + const std::uint32_t depth) { + if (!node.first_child() || has_text_child(node)) { + write_line(out, depth, + [&node](std::ostream &line) { write_inline(line, node); }); + return; + } + + out.write_element_begin( + "details", + HtmlElementOptions().set_class("odr-xml-node").set_extra("open")); + + out.write_element_begin("summary", HtmlElementOptions().set_inline(true)); + write_indent(out.out(), depth); + write_start_tag(out.out(), node, false); + out.write_element_end("summary"); + + for (const pugi::xml_node &child : node.children()) { + write_node(out, child, depth + 1); + } + write_line(out, depth, + [&node](std::ostream &line) { write_end_tag(line, node); }); + + out.write_element_end("details"); +} + +void write_node(HtmlWriter &out, const pugi::xml_node &node, + const std::uint32_t depth) { + if (node.type() == pugi::node_element) { + write_element(out, node, depth); + return; + } + write_line(out, depth, + [&node](std::ostream &line) { write_inline(line, node); }); +} + +class HtmlServiceImpl final : public HtmlService { +public: + HtmlServiceImpl(std::shared_ptr xml_file, HtmlConfig config, + const Logger &logger) + : HtmlService(std::move(config), logger), m_xml_file{std::move(xml_file)}, + m_resources{locate_xml_resources(this->config())} { + m_views.emplace_back( + std::make_shared(*this, "xml", 0, "xml.html")); + } + + void warmup() const override {} + + [[nodiscard]] const HtmlViews &list_views() const override { return m_views; } + + [[nodiscard]] bool exists(const std::string &path) const override { + return path == "xml.html" || resource_at(m_resources, path) != nullptr; + } + + [[nodiscard]] std::string mimetype(const std::string &path) const override { + if (path == "xml.html") { + return "text/html"; + } + if (const odr::HtmlResource *resource = resource_at(m_resources, path); + resource != nullptr) { + return resource->mime_type(); + } + + throw FileNotFound("Unknown path: " + path); + } + + void write(const std::string &path, std::ostream &out) const override { + if (path == "xml.html") { + HtmlWriter writer(out, config()); + write_xml(writer); + return; + } + if (const odr::HtmlResource *resource = resource_at(m_resources, path); + resource != nullptr) { + resource->write_resource(out); + return; + } + + throw FileNotFound("Unknown path: " + path); + } + + HtmlResources write_html(const std::string &path, + HtmlWriter &out) const override { + if (path == "xml.html") { + return write_xml(out); + } + + throw FileNotFound("Unknown path: " + path); + } + + HtmlResources write_xml(HtmlWriter &out) const { + HtmlResources resources; + const WritingState state(out, config(), resources); + + const pugi::xml_document &document = m_xml_file->document(); + + out.write_begin(); + + out.write_header_begin(); + + out.write_header_charset("UTF-8"); + out.write_header_target("_blank"); + out.write_header_title("odr"); + write_viewport_meta(out, config(), false); + + write_xml_style(state); + + out.write_header_end(); + + out.write_body_begin(); + + out.write_element_begin("div", HtmlElementOptions().set_class("odr-xml")); + for (const pugi::xml_node &child : document.children()) { + write_node(out, child, 0); + } + out.write_element_end("div"); + + out.write_body_end(); + + out.write_end(); + + return resources; + } + +protected: + std::shared_ptr m_xml_file; + /// The css this view links; empty of locations when the config embeds it. + HtmlResources m_resources; + + HtmlViews m_views; +}; + +} // namespace +} // namespace odr::internal::html + +namespace odr::internal { + +HtmlService html::create_xml_service(const TextFile &text_file, + HtmlConfig config, const Logger &logger) { + std::shared_ptr xml_file = + std::dynamic_pointer_cast(text_file.impl()); + if (xml_file == nullptr) { + throw NoXmlFile(); + } + return odr::HtmlService(std::make_unique( + std::move(xml_file), std::move(config), logger)); +} + +} // namespace odr::internal diff --git a/src/odr/internal/html/xml_file.hpp b/src/odr/internal/html/xml_file.hpp new file mode 100644 index 000000000..64abb167c --- /dev/null +++ b/src/odr/internal/html/xml_file.hpp @@ -0,0 +1,18 @@ +#pragma once + +namespace odr { +class TextFile; +struct HtmlConfig; +class HtmlService; +class Logger; +} // namespace odr + +namespace odr::internal::html { + +/// Renders @p text_file as an indented, highlighted, foldable source view. +/// @throws NoXmlFile if @p text_file was not decoded as a +/// @ref odr::FileType::xml. +HtmlService create_xml_service(const TextFile &text_file, HtmlConfig config, + const Logger &logger); + +} // namespace odr::internal::html diff --git a/src/odr/internal/open_strategy.cpp b/src/odr/internal/open_strategy.cpp index 04dcb1ed3..ccf821428 100644 --- a/src/odr/internal/open_strategy.cpp +++ b/src/odr/internal/open_strategy.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include @@ -175,6 +175,17 @@ open_file_as(const std::shared_ptr &file, const FileType as, throw NoJsonFile(); } + if (as == FileType::xml) { + ODR_VERBOSE(logger, "open as xml"); + try { + auto text = std::make_shared(file); + return std::make_unique(text); + } catch (...) { + ODR_VERBOSE(logger, "failed to open as xml"); + } + throw NoXmlFile(); + } + if (as == FileType::zip) { ODR_VERBOSE(logger, "open as zip"); try { @@ -294,8 +305,7 @@ open_strategy::list_file_types(const std::shared_ptr &file, // xml, so both are reported try { ODR_VERBOSE(logger, "try open as xml"); - util::xml::check_xml_file(*file->stream()); - result.push_back(FileType::xml); + result.push_back(xml::XmlFile(text).file_type()); try { ODR_VERBOSE(logger, "try open as svg"); @@ -417,8 +427,8 @@ open_strategy::open_file(const std::shared_ptr &file, ODR_VERBOSE(logger, "failed to open as json"); } - // an svg is only recognised by parsing it; plain xml has no decoder of - // its own and stays text + // svg first - it is the more specific reading of the same bytes - and + // xml last, before the line list try { ODR_VERBOSE(logger, "try open as svg"); svg::check_svg_file(*file->stream()); @@ -428,6 +438,13 @@ open_strategy::open_file(const std::shared_ptr &file, ODR_VERBOSE(logger, "failed to open as svg"); } + try { + ODR_VERBOSE(logger, "try open as xml"); + return std::make_unique(text); + } catch (...) { + ODR_VERBOSE(logger, "failed to open as xml"); + } + ODR_VERBOSE(logger, "open as text file"); // TODO looks dirty return std::make_unique(file); diff --git a/src/odr/internal/util/xml_util.cpp b/src/odr/internal/util/xml_util.cpp index 59e224bbc..466be5308 100644 --- a/src/odr/internal/util/xml_util.cpp +++ b/src/odr/internal/util/xml_util.cpp @@ -8,6 +8,8 @@ #include +#include +#include #include namespace odr::internal::util { @@ -30,6 +32,50 @@ pugi::xml_document xml::parse(std::istream &in) { void xml::check_xml_file(std::istream &in) { std::ignore = parse(in); } +std::string xml::read_declared_encoding(std::istream &in) { + static constexpr std::size_t probe_size = 1024; + static constexpr std::string_view space = " \t\r\n"; + + std::string probe(probe_size, '\0'); + in.read(probe.data(), static_cast(probe.size())); + probe.resize(static_cast(in.gcount())); + + std::string_view head(probe); + if (head.starts_with("\xef\xbb\xbf")) { + head.remove_prefix(3); + } + if (!head.starts_with(""); + if (declaration_end == std::string_view::npos) { + return {}; + } + head = head.substr(0, declaration_end); + + const std::size_t name = head.find("encoding"); + if (name == std::string_view::npos) { + return {}; + } + head.remove_prefix(name + std::string_view("encoding").size()); + + std::size_t at = head.find_first_not_of(space); + if (at == std::string_view::npos || head[at] != '=') { + return {}; + } + at = head.find_first_not_of(space, at + 1); + if (at == std::string_view::npos || (head[at] != '"' && head[at] != '\'')) { + return {}; + } + const char quote = head[at]; + ++at; + const std::size_t value_end = head.find(quote, at); + if (value_end == std::string_view::npos) { + return {}; + } + return std::string(head.substr(at, value_end - at)); +} + pugi::xml_document xml::parse(const abstract::ReadableFilesystem &filesystem, const AbsPath &path) { pugi::xml_document result; diff --git a/src/odr/internal/util/xml_util.hpp b/src/odr/internal/util/xml_util.hpp index 490a7727b..5b6624b33 100644 --- a/src/odr/internal/util/xml_util.hpp +++ b/src/odr/internal/util/xml_util.hpp @@ -25,6 +25,11 @@ pugi::xml_document parse(const abstract::ReadableFilesystem &, const AbsPath &); /// Throws unless @p in holds a well formed xml document. void check_xml_file(std::istream &in); +/// The `encoding` pseudo-attribute of an `` declaration at the head of +/// @p in, empty if there is none. Ascii only - utf-16 and utf-32 are named by +/// their byte order mark. +[[nodiscard]] std::string read_declared_encoding(std::istream &in); + struct StringToken { enum class Type { none, diff --git a/src/odr/internal/xml/AGENTS.md b/src/odr/internal/xml/AGENTS.md new file mode 100644 index 000000000..71215fe96 --- /dev/null +++ b/src/odr/internal/xml/AGENTS.md @@ -0,0 +1,129 @@ +# AGENTS.md — `internal/xml` + +Read the root [`AGENTS.md`](../../../../AGENTS.md) first. This file covers what +xml does differently, and why. What is not built yet is in [`PLAN.md`](PLAN.md). + +## A source view, not a document + +Xml has no document semantics: no paragraph, no page, no sheet, only nesting. +An `ElementAdapter` would mean picking a `DocumentType` that is a lie for a +renderer that has nothing to contribute. So `XmlFile` mirrors `JsonFile` — an +`abstract::TextFile` over a `text::TextFile`, probe in the constructor, +`is_decodable()` false — and renders through one `HtmlService`. + +It stays a text file, so `TextFile::text()` keeps working on it. + +The browser's own xml viewer is not an option: it fires on a *response* served +as `text/xml`, and `HtmlService` hands the host an html document. + +## The dispatch is on the decoded file + +`html::translate(const DecodedFile &)` sends `FileType::xml` to +`create_xml_service`, next to the csv branch and for the same reason: a line +list is never what a viewer wants from a file with no line breaks in it. +`html::translate(const TextFile &)` is left alone — asking for the text +rendering gets the text rendering. + +The service takes the public `odr::TextFile` and casts its impl back to +`XmlFile` — the parse is the file's, not the writer's, so the writer never gets +to parse something the decoder did not accept. + +## The tree is not the bytes + +A pugixml tree is a normalisation. Lost: the original indentation, attribute +quote style, whether a character arrived as `A` or `A`, and — `parse_eol` +and `parse_wconv_attribute` are both on — line-ending and in-attribute newline +spelling. Pretty-printing is the feature; fidelity is the price. + +One cost worth naming: a **malformed** file has no tree, so `XmlFile` throws +`NoXmlFile` and it falls back to the line list — which is exactly when a viewer +is most wanted. + +## The parse flags, and where they live + +`parse_source` (`xml_file.cpp`, file-local) is the only place they are written, +and `XmlFile`'s constructor is its only caller. Not `util::xml::parse`, whose +every other caller wants pugixml's defaults. + +- `parse_full` adds the four node kinds `parse_default` drops — comments, + processing instructions, the declaration, the doctype. +- `parse_ws_pcdata_single` keeps whitespace-only text where it is an element's + only child, so ` ` survives while the newline between two siblings + does not. It is what makes the mixed-content rule decidable. + +**No DTD processing, and that is a feature.** pugixml resolves no external +entities and expands no internal ones, which closes XXE and entity expansion by +construction. An undefined entity is shown as written, which a source view +wants anyway. + +## The encoding is declared in band + +pugixml's `encoding_auto` resolves UTF-8/16/32 from a BOM or the ` `, and the looser rule leaves it alone. + +## Writing decisions + +- **Highlighting is server-side spans**, one per token, emitted as the writer + walks the tree. A JavaScript highlighter would undo the self-contained output + for a job the writer already does. +- **Folding is `
`/``, with no script** — keyboard access, + screen-reader semantics, and find-in-page that natively expands a collapsed + section. Start tag in the ``, children then end tag in the body, so + collapsing hides the whole node. Everything is open by default. Bulk + expand-all/collapse-all would need JavaScript, and there is none. +- **No line numbers**; the column carries the fold handles, and every line + reserves it so folding does not shift siblings. +- **Indentation is spaces, not padding**, so a copy of the page carries it. +- **Not `html::escape_text`** — it folds spaces into ` ` and tabs into + ` `. `escape_source` in `html/xml_file.cpp` escapes `&`, `<` and `>`. +- **Attribute values take whichever quote needs no entity**, and where the + value carries both, the double quote becomes `"`. Their whitespace is + source text like any other, so the span preserves it. + +## One parse, and the file holds it + +`XmlFile`'s constructor parses, and keeps the tree; `document()` hands out a +`const &` and the html service walks it per render. Recognising the file and +rendering it are the same parse. + +Two things ruled the alternatives out. pugixml is dom-only — no sax, no +incremental mode — so there is no validating the head of a file the way csv's +`probe` scores its opening bytes; and a source view's whole contract is that +what opened will render, which a sniff gives up. And a document cannot be +copied out (`reset(proto)` is a deep clone, no cheaper than reparsing), so +lending it beats returning it. + +The price is memory: the dom is roughly twice the file, held for as long as the +`XmlFile` is, whether or not anyone renders it. `PLAN.md`'s size section owns +that number. + +## Detection + +Xml is the **last resort** in `open_file`'s unknown-type path, after csv, json +and svg. + +Two consequences. A flat-xml ODF (`.fods` and friends) has no detection — the +flat mimetypes are only aliases on the zip-backed rows — so it decodes as xml. +Better than text, **not** flat-ODF support. Likewise `.xhtml`, `.rels`, +`.plist` and rss feeds become source views: correct for a source viewer, and +correct that we do not try to *render* xhtml. diff --git a/src/odr/internal/xml/PLAN.md b/src/odr/internal/xml/PLAN.md index 2e9e3b408..d5af45638 100644 --- a/src/odr/internal/xml/PLAN.md +++ b/src/odr/internal/xml/PLAN.md @@ -1,298 +1,52 @@ # XML plan -Where an xml renderer would go, and in what order. Written before stage 1; keep -it honest as stages land. +What is left to build. The module as it stands is in [`AGENTS.md`](AGENTS.md). -## Today - -`FileType::xml` exists (`file.hpp:150`, under the "Classification only" -comment) and carries a table row — `xml` extension, `application/xml` and -`text/xml`, `FileCategory::text`, `DocumentType::unknown`, -`{.detect_by_content = true}` (`file_type_table.cpp:664`). - -Detection already works. `list_file_types` parses the file with -`util::xml::check_xml_file` and reports `[text_file, xml]`, plus -`scalable_vector_graphics` on top when the root element says so -(`open_strategy.cpp:296`). What is missing is everything after that: -`open_file_as` has no `FileType::xml` branch, and `open_file`'s unknown-type -path tries csv, json and svg before falling through to `text::TextFile` -(`open_strategy.cpp:432`). So a `.xml` decodes as `text_file` and renders -through `html::create_text_service` as a numbered line list. - -Which is the whole problem. The xml files anyone opens on purpose — -`content.xml`, `document.xml`, anything a writer emitted rather than a human — -have no newlines in them, so the line list is one line several megabytes wide. - -Already in place and reusable: `NoXmlFile` (`exceptions.hpp:146`), thrown by -`util::xml::parse` (`util/xml_util.cpp:18`); pugixml 1.15 as a dependency -(`conanfile.py:51`); and `internal/encoding` for transcoding. - -## Target - -An xml file opens as `XmlFile` and renders as a **source view**: indented, -syntax-highlighted, foldable, in one self-contained html document with no -JavaScript and no external resources. It stays a `TextFile`. The work is a -decoder shell plus one html service — no `Document`, no element adapters. - -## Why not leave it to the browser - -Every current browser ships an xml tree viewer, and none of them is reachable -from here. They fire on a *response* served as `text/xml`; odr serves nothing — -`HtmlService` hands the host an html document that a WebView displays, and -inside an html document the viewer never engages. - -It would be the wrong lever even if it could be pulled. An -`` PI makes the browser run the XSLT instead of showing the -tree — silently rendering something else entirely. The viewers differ from each -other in folding, attribute display and error reporting. And the two engines -that matter most for this library, Android's WebView and WKWebView, are not the -browsers whose behaviour anyone checked. - -## Decisions taken up front - -**A file-level html service, not a document.** Xml has no document semantics. -Routing it through `ElementAdapter` would mean picking a `DocumentType` that is -a lie, and the generic renderer would have nothing to contribute — there is no -paragraph, no page, no sheet, only nesting. This is the opposite call from the -csv plan, and for a reason that transfers: a csv *is* a sheet, so the model -earns its keep and every binding gets a table for free. An xml file is not a -document that happens to be serialised as xml; it is the serialisation. So it -renders the way an image or a media file does — one `HtmlService`, html output -only, no element api. See also *the archive seam* below, which is where the -api-free choice does eventually cost something. - -**`XmlFile` mirrors `JsonFile`.** `abstract::TextFile` over a -`std::shared_ptr`, constructed with the same probe-in-the- -constructor shape (`json/json_file.cpp:10`), `is_decodable()` false. The table -row flips to `{.detect_by_content = true, .open = true, .translate_html = -true}`. - -The api-visible consequence: a `.xml` that reports `text_file` today will -report `xml`. No binding work — the enumerator already exists and the bindings -mirror the enum by ordinal — but a caller switching on `file_type()` sees the -change. That is the point of the change, and it is the same step csv and json -already took. - -**The tree is not the bytes.** "We can already read it" is true only of the -parsed tree, and a pugixml tree is a normalisation of the file, not a view of -it. Lost, unavoidably: the original indentation, attribute quote style, whether -a character came in as `A` or `A`, and — with `parse_eol` and -`parse_wconv_attribute`, both on by default — line-ending and in-attribute -newline spelling. - -Accept that, deliberately. The alternative is a byte-faithful lexer over the -raw text, which keeps the author's formatting but does nothing for the minified -file that motivated the whole feature, and still has to reconstruct nesting -before it can fold anything. Pretty-printing is the feature; fidelity is the -price. If someone later wants a true "view source", it is a second mode over -the same css, not a redesign — noted under *Deferred*. - -**Parse with `parse_full`, and keep whitespace-only text.** `util::xml::parse` -uses pugixml's defaults, which drop comments, processing instructions, the -declaration and the doctype — invisible in a viewer whose job is to show what -is in the file. `parse_full` is exactly those four added to `parse_default`. - -`parse_ws_pcdata` is a separate flag and a separate question: keeping every -whitespace-only text node preserves fidelity but fills the tree with nodes we -are about to reindent anyway. Take `parse_ws_pcdata_single`, which keeps -whitespace-only text only where it is an element's sole child — so ` ` -survives as content while the newline-and-tab between two sibling elements does -not. Note that this is the flag that makes the mixed-content rule below -decidable at all. - -Do **not** reuse `util::xml::parse` for this; it hard-codes the default flags -and every existing caller wants them. Add the options at the xml module's own -call site. - -**Encoding is declared in band, and pugixml will not honour it.** An xml file -names its own encoding in the declaration, which is better information than -`encoding::detect`'s uchardet guess over a 64 KiB probe. pugixml's -`encoding_auto` resolves UTF-8/16/32 from a BOM or the `` document is read as UTF-8 -and yields invalid UTF-8 in the node strings, silently. (Verify against 1.15 -before relying on the negative — but the design below is right either way.) - -So: read the declaration's `encoding` pseudo-attribute from the head of the -file, map it through `text_encoding_by_name`, transcode with -`encoding::to_utf8`, and hand pugixml UTF-8. Precedence is declaration, then -BOM, then `text::TextFile::encoding()`'s guess. `XmlFile::encoding()` returns -the resolved value, so a caller can show it. An encoding we can name but not -decode throws `UnsupportedTextEncoding`, as the csv sheet path does and for the -same reason: the tree path has to produce UTF-8, and there is no "let the -browser sort it out" once the bytes are inside a parser. - -**Mixed content is not reindented.** `

a x b

` carries significant -whitespace, and nothing short of a schema can tell it from the insignificant -kind. The rule: an element with any non-whitespace text child renders its -children inline, on one line, untouched; an element whose children are all -elements is indented and foldable. This is what every xml viewer does, it is -the one non-trivial rule in the writer, and it is the first thing a test should -pin. - -**Highlighting is server-side spans.** One `` per -token, emitted by the writer. Not a JavaScript highlighter: the output has been -self-contained with no external resources since the css and js moved into the -document (`HtmlResource::is_shipped`, `html.hpp:51`), and shipping a -highlighter would reverse that for a job the writer is already doing as it -walks the tree. - -Classes, following the `odr-text-*` naming in `frontend.cpp`: `odr-xml` on the -root, then `-tag`, `-name`, `-attr`, `-value`, `-text`, `-cdata`, `-comment`, -`-pi`, `-decl`, `-doctype`. Light palette only, as `text_css` is — a dark mode -is a question for every view at once, not for this one. - -**Folding is `
`/``, with no script.** The disclosure element -gets keyboard access, screen-reader semantics and — the reason it wins — -find-in-page that expands a collapsed section natively, which a `display:none` -toggle does not. Verify the Safari behaviour before promising it; Chrome and -Firefox have done it for years. - -The layout objection is answerable: `details`/`summary` both `display:block`, -`summary::marker` removed via `list-style:none`, indentation carried inside the -summary so the `white-space:pre` flow stays intact. The start tag goes in the -``, the children in the body, the end tag on a line after it. - -The cost, recorded honestly: bulk expand-all/collapse-all needs JavaScript, so -stage 2 ships without it. - -**No line numbers.** The text view has a numbered gutter (`html/text_file.cpp:101`). -Reproducing it here would number *our* lines, not the file's, which for a -reindented minified document is actively misleading. The gutter column goes to -the fold handles instead. - -**Xml is the last resort in detection.** Insert the branch in `open_file`'s -unknown-type path *after* svg and before the text fallthrough -(`open_strategy.cpp:423`), so anything with a more specific reading keeps it. - -Two behaviour changes fall out, both worth naming before they surprise someone. -A flat-xml ODF (`.fods` and friends) has no detection today — the flat mimetypes -are only aliases on the zip-backed rows (`file_type_table.cpp:26`) — so it -currently decodes as text and would now decode as xml. That is an improvement -(a source tree beats a single line) but it is not what a flat ODF should -eventually do, and it must not be mistaken for support. Likewise `.xhtml`, -`.rels`, `.plist` and every rss feed become source views rather than line -lists — correct for a source viewer, and correct that we do not try to *render* -xhtml. - -**No DTD processing, and that is a feature.** pugixml does not resolve external -entities and does not expand internal entity declarations; it handles the five -predefined entities and numeric character references, and leaves `&foo;` as -literal text. For a viewer that opens files from the internet this closes XXE -and entity-expansion attacks by construction rather than by policy. The -fidelity note is the same sentence read the other way: an undefined entity is -shown as written, which for a source view is the right answer anyway. - -**A parse failure falls back to text.** `XmlFile`'s constructor throws -`NoXmlFile`, `open_file` catches it and reaches `text::TextFile`, and a -malformed file renders as the line list it renders as today. Automatic and -correct — with one thing conceded up front: "show me the broken xml" is exactly -when a viewer is most wanted, and the tree path structurally cannot serve it. -That is the strongest argument for the byte-faithful second mode, and it is not -strong enough to build both now. - -## Module layout - -``` -src/odr/internal/xml/ - xml_file.hpp/.cpp abstract::TextFile over a text::TextFile; parse probe, encoding resolution -src/odr/internal/html/ - xml_file.hpp/.cpp create_xml_service — the HtmlService and the writer -``` - -Both `.cpp` go into `ODR_SOURCE_FILES`: the html one next to -`html/text_file.cpp` (`CMakeLists.txt:148`), the module one after `util/` and -before `zip/` (`CMakeLists.txt:252`). - -No `xml_util.cpp` — `internal/util/xml_util` is the shared xml helper and stays -where it is. Anything this module needs that is genuinely general (the -declaration sniff) belongs there, not in a second utility with the same name. - ---- - -## Stage 1 — it opens, and it renders - -The skeleton end to end, flat: highlighted and indented, not yet foldable. - -- `XmlFile` per the `JsonFile` shape; declaration sniff, transcode, - `parse_full | parse_ws_pcdata_single`; `encoding()` resolved as above. -- `open_file_as` gains a `FileType::xml` branch throwing `NoXmlFile`; - `open_file` gains one after svg. Table row flipped to `{.open = true, - .translate_html = true}`. -- `html/xml_file.cpp` with `create_xml_service`, and a `file_type()` branch in - `html::translate(const TextFile &)` (`html.cpp:256`) so xml gets the tree and - everything else keeps the line list. One view, `xml.html`, mirroring - `html/text_file.cpp:26`. -- the writer: declaration, doctype, PI, comment, element, attribute, text and - CDATA, escaped through `html::escape_text` / `escape_attribute`, indented, - each token in its span. The mixed-content rule lands here, not later. -- `xml_css` in `frontend.cpp`, `write_xml_style` in `frontend.hpp`. -- `test/src/internal/xml/xml_file_test.cpp`, inline string literals in, html - out (`text_file_test.cpp` is the shape). Minimum set: minified input - reindents; mixed content does not; comments/PI/doctype/CDATA all survive; - a declared non-UTF-8 encoding decodes; malformed input throws `NoXmlFile`. +## Owed — test data `FileTypeCapabilities.declaration_matches_the_engines` (`odr_test.cpp`) opens -files per type from the test data against the row, so a handful of `.xml` -samples go into the test-data repo with this stage — see *Test data*. - -## Stage 2 — folding +files per type from the test data, and there are no `.xml` samples for it to +open. Wanted: a minified `content.xml` lifted from an odt, a hand-formatted +document with comments and a doctype, one non-UTF-8 declared encoding, and one +file that is xml-shaped but malformed. -- `
`/`` per element with element children, per the - markup and css above. Elements with no children stay a plain line. -- everything open by default. Collapsing by default hurts find-in-page and - hides the thing the user opened the file to see; the only case for it is - size, which stage 3 handles with a threshold rather than a habit. -- the fold handle in the gutter column the line numbers do not occupy. +Everything a string literal can express stays inline in +`test/src/internal/xml/xml_file_test.cpp`. -## Stage 3 — size +## Size A `content.xml` is routinely tens of megabytes, and this path multiplies it: -pugixml's dom is roughly 1.5–2× the file plus the in-memory buffer, and a span -per token can be 5–10× the input in emitted html. Both land in a WebView on a -phone. +pugixml's dom is roughly 1.5–2× the file and `XmlFile` holds it for its +lifetime, and a span per token can be 5–10× the input in emitted html. Both +land in a WebView on a phone. -- a node budget in `HtmlConfig`, following `spreadsheet_limit` - (`html.hpp:124`) — `std::optional xml_node_limit`, `nullopt` - for unlimited — and, past it, stop and emit a visible truncation notice - rather than a silently short document. +- a node budget in `HtmlConfig`, following `spreadsheet_limit` — + `std::optional xml_node_limit`, `nullopt` for unlimited — and + past it a visible truncation notice rather than a silently short document. - past a lower threshold, default the fold state to closed below some depth. - This is the only case where collapsed-by-default is right, and it is a - response to a measurement, not a preference. + The only case where collapsed-by-default is right. - measure before choosing the numbers, on a real `content.xml`. -## Stage 4 — the archive seam +## The archive seam The filesystem view links every entry as an `application/octet-stream` data url -(`html/filesystem.cpp:110`), so browsing into a zip and looking at -`word/document.xml` downloads it. Routing entries through `html::translate` -instead is a separate feature with its own questions (which types, resource -paths, how deep), but it is the one that turns this from an xml-file viewer -into a way to inspect any office document's parts. Named here so the -dependency is on record; not scoped here. +(`html/filesystem.cpp`), so browsing into a zip and looking at +`word/document.xml` downloads it. Routing entries through `html::translate` is +a separate feature with its own questions (which types, resource paths, how +deep), and it is the one that turns this into a way to inspect any office +document's parts. Named so the dependency is on record; not scoped here. ## Deferred, by decision - **Byte-faithful mode.** A lexer over the raw text, sharing the css, keeping - the author's formatting and — the real payoff — able to render a malformed - file up to the point where it breaks. Wanted; not wanted enough to build two - renderers before one exists. -- **XSLT.** An `` PI is shown as the processing instruction it - is. Applying it means an XSLT engine, which is larger than every format in - this repository put together. -- **Rendering xhtml as html.** Same class of decision, and the answer is no for - the same reason: this is a source viewer. -- **Namespace resolution.** pugixml does not process namespaces - (`svg/svg_util.cpp:15` works around exactly this), and a source view should - show the prefixes the file actually uses. Nothing to do. -- **Expand-all / collapse-all**, and **search within the tree** — both need - JavaScript, and neither is worth being the reason this view starts shipping a - script. -- **Dark mode**, which is a question for `frontend.cpp` as a whole. - -## Test data - -Content, not fixtures, for everything a string literal can express — the parser -and writer tests are inline. The test-data repo needs only what -`declaration_matches_the_engines` opens: a minified `content.xml` lifted from an -odt, a hand-formatted document with comments and a doctype, one non-UTF-8 -declared encoding, and one file that is xml-shaped but malformed. + the author's formatting and able to render a malformed file up to the point + where it breaks. Not worth two renderers before there is one. +- **XSLT.** An `` PI is shown as the processing instruction + it is. Applying it means an XSLT engine, larger than every format here put + together. +- **Rendering xhtml as html.** This is a source viewer. +- **Namespace resolution.** pugixml does not process namespaces, and a source + view should show the prefixes the file uses. +- **Expand-all / collapse-all** and **search within the tree** — both need + JavaScript, and neither is worth being the reason this view ships a script. +- **Dark mode**, a question for `frontend.cpp` as a whole. diff --git a/src/odr/internal/xml/xml_file.cpp b/src/odr/internal/xml/xml_file.cpp new file mode 100644 index 000000000..6103fbdaf --- /dev/null +++ b/src/odr/internal/xml/xml_file.cpp @@ -0,0 +1,94 @@ +#include + +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include + +namespace odr::internal { + +namespace { + +/// The declaration beats any guess over the bytes; a name we do not know +/// leaves the guess in place. +TextEncoding resolve_encoding(const text::TextFile &file) { + const std::unique_ptr in = file.file()->stream(); + const std::string declared = util::xml::read_declared_encoding(*in); + + if (const TextEncoding encoding = text_encoding_by_name(declared); + encoding != TextEncoding::unknown) { + return encoding; + } + return file.encoding(); +} + +/// @throws NoXmlFile if @p text is not a well formed xml document. +std::unique_ptr parse_source(const std::string &text) { + // `parse_full` adds the four node kinds `parse_default` drops, all of which + // a viewer has to show; `parse_ws_pcdata_single` keeps ` ` while + // dropping the newline between two siblings. + static constexpr unsigned int options = + pugi::parse_full | pugi::parse_ws_pcdata_single; + + auto result = std::make_unique(); + if (const pugi::xml_parse_result success = result->load_buffer( + text.data(), text.size(), options, pugi::encoding_utf8); + !success) { + throw NoXmlFile(); + } + return result; +} + +} // namespace + +xml::XmlFile::XmlFile(std::shared_ptr file) + : m_file{std::move(file)} { + m_encoding = resolve_encoding(*m_file); + m_document = parse_source(text()); +} + +xml::XmlFile::~XmlFile() = default; + +std::shared_ptr xml::XmlFile::file() const noexcept { + return m_file->file(); +} + +FileType xml::XmlFile::file_type() const noexcept { return FileType::xml; } + +std::string_view xml::XmlFile::mimetype() const noexcept { + return "application/xml"; +} + +FileMeta xml::XmlFile::file_meta() const noexcept { + FileMeta result; + result.type = file_type(); + result.mimetype = mimetype(); + return result; +} + +bool xml::XmlFile::is_decodable() const noexcept { return false; } + +TextEncoding xml::XmlFile::encoding() const noexcept { return m_encoding; } + +std::string xml::XmlFile::text() const { + // a parser only ever sees utf-8; there is no handing the bytes on undecoded + if (!text_encoding_is_decodable(m_encoding)) { + throw UnsupportedTextEncoding(m_encoding); + } + const std::unique_ptr in = m_file->file()->stream(); + return encoding::to_utf8(util::stream::read(*in), m_encoding); +} + +const pugi::xml_document &xml::XmlFile::document() const noexcept { + return *m_document; +} + +} // namespace odr::internal diff --git a/src/odr/internal/xml/xml_file.hpp b/src/odr/internal/xml/xml_file.hpp new file mode 100644 index 000000000..5b610e276 --- /dev/null +++ b/src/odr/internal/xml/xml_file.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include + +#include + +#include +#include + +namespace pugi { +class xml_document; +} // namespace pugi + +namespace odr::internal::xml { + +/// An xml file. Nothing is decoded beyond the parse that recognises it: it +/// renders as a source view, not as a document. +class XmlFile final : public abstract::TextFile { +public: + /// @throws NoXmlFile if @p file is not a well formed xml document. + /// @throws UnsupportedTextEncoding if its encoding cannot be decoded. + explicit XmlFile(std::shared_ptr file); + ~XmlFile() override; + + [[nodiscard]] std::shared_ptr file() const noexcept override; + + [[nodiscard]] FileType file_type() const noexcept override; + [[nodiscard]] std::string_view mimetype() const noexcept override; + [[nodiscard]] FileMeta file_meta() const noexcept override; + + [[nodiscard]] bool is_decodable() const noexcept override; + + /// The declaration's `encoding` where it names one we know, else what the + /// bytes were detected as. + [[nodiscard]] TextEncoding encoding() const noexcept override; + + /// The file's bytes decoded to utf-8. + /// @throws UnsupportedTextEncoding if @ref encoding cannot be decoded. + [[nodiscard]] std::string text() const; + + /// The tree the constructor parsed, keeping what a source view has to show: + /// the declaration, the doctype, processing instructions, comments, and + /// whitespace-only text where it is an element's only child. + /// + /// Held rather than reparsed per render, and pugixml's dom is roughly twice + /// the file — so an `XmlFile` costs that for as long as it is open. + [[nodiscard]] const pugi::xml_document &document() const noexcept; + +private: + std::shared_ptr m_file; + TextEncoding m_encoding{TextEncoding::unknown}; + std::unique_ptr m_document; +}; + +} // namespace odr::internal::xml diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d07c04513..f796f7f82 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -55,6 +55,7 @@ add_executable(odr_test "src/internal/csv/csv_file_test.cpp" "src/internal/encoding/text_encoding_test.cpp" + "src/internal/xml/xml_file_test.cpp" "src/internal/oldms/doc_test.cpp" "src/internal/oldms/ppt_test.cpp" diff --git a/test/src/internal/xml/xml_file_test.cpp b/test/src/internal/xml/xml_file_test.cpp new file mode 100644 index 000000000..eb7e63304 --- /dev/null +++ b/test/src/internal/xml/xml_file_test.cpp @@ -0,0 +1,201 @@ +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include + +using namespace odr; +using namespace odr::internal; +using testing::HasSubstr; +using testing::Not; + +namespace { + +std::shared_ptr xml_file(const std::string &content) { + return std::make_shared( + std::make_shared(File::from_memory(content).impl())); +} + +/// The source view, with the css embedded as the default config asks for. +std::string xml_html(const std::string &content) { + const HtmlService service = html::translate(DecodedFile(xml_file(content)), + HtmlConfig(), Logger::null()); + + std::ostringstream out; + service.write("xml.html", out); + return out.str(); +} + +std::string declared_encoding(const std::string &content) { + std::istringstream in(content); + return util::xml::read_declared_encoding(in); +} + +} // namespace + +TEST(XmlFile, an_xml_file_opens_as_xml) { + const DecodedFile file(File::from_memory("")); + + EXPECT_EQ(file.file_type(), FileType::xml); + EXPECT_EQ(file.file_meta().mimetype, "application/xml"); + EXPECT_TRUE(file.is_text_file()); + EXPECT_FALSE(file.is_document_file()); + EXPECT_TRUE(file.capabilities().translate_html); + + EXPECT_THAT(DecodedFile::list_file_types(File::from_memory("")), + testing::Contains(FileType::xml)); +} + +/// Xml is the last resort: anything with a more specific reading keeps it. +TEST(XmlFile, an_svg_still_opens_as_an_image) { + const DecodedFile file( + File::from_memory(R"()")); + + EXPECT_EQ(file.file_type(), FileType::scalable_vector_graphics); + EXPECT_TRUE(file.is_image_file()); +} + +TEST(XmlFile, malformed_xml_is_no_xml_file_and_stays_text) { + EXPECT_THROW(std::ignore = xml_file(""), NoXmlFile); + + // which is what leaves the line list in place for it + const DecodedFile file(File::from_memory("")); + EXPECT_EQ(file.file_type(), FileType::text_file); +} + +TEST(XmlFile, the_declaration_names_the_encoding) { + // 0xe9 is `é` in latin-1 and not valid utf-8, so only the declaration can + // get it right + const std::string content = + "\xe9"; + + EXPECT_EQ(xml_file(content)->encoding(), TextEncoding::iso_8859_1); + EXPECT_THAT(xml_html(content), HasSubstr("é")); +} + +/// An encoding we can name but not decode has no tree at all — there is no +/// handing the bytes to the browser once they are inside a parser. +TEST(XmlFile, an_encoding_we_cannot_decode_has_no_source_view) { + const std::string content = + ""; + + EXPECT_THROW(std::ignore = xml_file(content), UnsupportedTextEncoding); + EXPECT_EQ(DecodedFile(File::from_memory(content)).file_type(), + FileType::text_file); +} + +TEST(XmlDeclaration, the_encoding_pseudo_attribute_is_read_off_the_bytes) { + EXPECT_EQ(declared_encoding(R"()"), + "UTF-8"); + EXPECT_EQ(declared_encoding(""), + "latin1"); + EXPECT_EQ(declared_encoding("\xef\xbb\xbf"), + "UTF-8"); + + EXPECT_EQ(declared_encoding(R"()"), ""); + EXPECT_EQ(declared_encoding(""), ""); + EXPECT_EQ(declared_encoding(""), ""); + // no `?>` in the probe: an unterminated declaration names nothing + EXPECT_EQ(declared_encoding(R"(")), HtmlConfig(), Logger::null()); + + std::ostringstream out; + service.write("text.html", out); + EXPECT_THAT(out.str(), HasSubstr("odr-text-nr")); +} + +/// The whole point of the view: the files anyone opens on purpose have no line +/// breaks in them. +TEST(XmlHtml, a_minified_document_is_reindented) { + const std::string html = xml_html(""); + + EXPECT_THAT(html, HasSubstr(R"( )")); + EXPECT_THAT(html, HasSubstr(R"( )")); + EXPECT_THAT(html, HasSubstr(R"(c/>)")); +} + +/// `

a x b

` carries significant whitespace and nothing short of a +/// schema can tell it from the insignificant kind. +TEST(XmlHtml, mixed_content_is_left_alone) { + const std::string html = xml_html("

a x b

"); + + EXPECT_THAT(html, HasSubstr(R"(a )")); + EXPECT_THAT(html, HasSubstr(R"( b)")); + // one line, so no fold and no indentation inside it + EXPECT_THAT(html, Not(HasSubstr(" "), + HasSubstr(R"( )")); + // between two siblings it is not + EXPECT_THAT(xml_html("\n \n"), + Not(HasSubstr(R"(" + "" + "" + "" + ""); + + EXPECT_THAT(html, HasSubstr(R"(version)")); + EXPECT_THAT(html, HasSubstr(R"(<!DOCTYPE a SYSTEM "a.dtd">)")); + EXPECT_THAT(html, HasSubstr(R"(<?stylesheet href="a.xsl"?>)")); + EXPECT_THAT(html, HasSubstr("<!-- a note -->")); + EXPECT_THAT(html, HasSubstr("<![CDATA[x < y]]>")); +} + +TEST(XmlHtml, an_element_with_element_children_folds) { + const std::string html = xml_html(""); + + EXPECT_THAT(html, HasSubstr(R"(
)")); + EXPECT_THAT(html, HasSubstr("")); + // the end tag is inside the fold, so collapsing hides the whole node + EXPECT_THAT(html, HasSubstr(R"(</a)")); + EXPECT_THAT(html, Not(HasSubstr(")"), + HasSubstr(R"('say "hi"')")); +} + +/// Neither delimiter is free, so one of them has to be written as an entity. +TEST(XmlHtml, an_attribute_value_carrying_both_quotes_escapes_the_delimiter) { + EXPECT_THAT( + xml_html(R"()"), + HasSubstr( + R"("can't say "hi"")")); +} + +/// The value is source text, and html would fold its spaces. +TEST(XmlHtml, an_attribute_values_whitespace_is_preserved) { + const std::string html = xml_html(R"()"); + + EXPECT_THAT(html, HasSubstr(R"("x y")")); + EXPECT_THAT(html, HasSubstr(".odr-xml-value,")); +}