From d79e6a5ed9acb5b6b07882ee560a3de5c83b010d Mon Sep 17 00:00:00 2001 From: che cheng Date: Fri, 14 Aug 2026 01:21:27 +0800 Subject: [PATCH 1/7] feat: add native Markdown OMath conversion (#141) --- CONVERSIONS.md | 10 +- Package.resolved | 9 + README.md | 13 + Sources/MacDocCLI/MacDoc+Convert.swift | 22 +- .../MarkdownOMathRouteTests.swift | 174 +++++ .../markdown-omath-conversion/.openspec.yaml | 4 + .../markdown-omath-conversion/design.md | 134 ++++ .../markdown-omath-conversion/proposal.md | 47 ++ .../specs/e2e-conversion-routes/spec.md | 43 ++ .../specs/markdown-omath-conversion/spec.md | 134 ++++ .../markdown-omath-conversion/tasks.md | 33 + packages/md-to-word-swift/Package.swift | 4 +- .../MarkdownMathConversionError.swift | 22 + .../MDToWord/MarkdownMathScanner.swift | 604 ++++++++++++++++++ .../MDToWord/MarkdownToWordConverter.swift | 251 ++++++-- .../MarkdownMathScannerTests.swift | 124 ++++ .../MarkdownOMathConversionTests.swift | 288 +++++++++ 17 files changed, 1875 insertions(+), 41 deletions(-) create mode 100644 Tests/MacDocCLITests/MarkdownOMathRouteTests.swift create mode 100644 openspec/changes/markdown-omath-conversion/.openspec.yaml create mode 100644 openspec/changes/markdown-omath-conversion/design.md create mode 100644 openspec/changes/markdown-omath-conversion/proposal.md create mode 100644 openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md create mode 100644 openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md create mode 100644 openspec/changes/markdown-omath-conversion/tasks.md create mode 100644 packages/md-to-word-swift/Sources/MDToWord/MarkdownMathConversionError.swift create mode 100644 packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift create mode 100644 packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift create mode 100644 packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift diff --git a/CONVERSIONS.md b/CONVERSIONS.md index cb4c8f32..e20bc135 100644 --- a/CONVERSIONS.md +++ b/CONVERSIONS.md @@ -11,7 +11,7 @@ | → Target | Markdown | HTML | Word (.docx) | LaTeX | JSON | PDF | SRT | |----------:|:--------:|:----:|:------------:|:-----:|:----:|:---:|:---:| -| **Markdown** | — | ✅ `md-to-html` | ✅ `md-to-word` | · | · | · | · | +| **Markdown** | — | ✅ `md-to-html` | ✅ `md-to-word` (OMath opt-in) | · | · | · | · | | **HTML** | ✅ `html-to-md` | — | ✅ `html-to-word` | · | · | · | · | | **Word (.docx)** | ✅ `word-to-md` | ✅ `word-to-html` | — | · | · | · | · | | **PDF** | ✅ `pdf-to-md` | · | ✅ `pdf-to-docx` | ✅ `pdf-to-latex` | · | — | · | @@ -34,10 +34,16 @@ | PDF → Markdown | `pdf-to-md-swift` | ✅ implemented | direct path via PDFKit, heading/list heuristics | | Word → HTML | `word-to-html-swift` | ✅ implemented | direct path preserves Word semantics | | HTML → Word | `html-to-word-swift` | ✅ implemented | SwiftSoup → OOXML writer | -| Markdown → Word | `md-to-word-swift` | ✅ implemented | swift-markdown AST → OOXML writer | +| Markdown → Word | `md-to-word-swift` | ✅ implemented | swift-markdown AST → OOXML writer; native OMath is opt-in with `macdoc convert input.md --to docx --math omath --output output.docx` (`literal` is the default) | | PDF → DOCX | `pdf-to-docx-swift` | ✅ implemented | PDFKit text extraction → OOXML writer | | Note → HTML | `note-to-html-swift` | ✅ implemented | Notability .note → interactive HTML player with audio-synced stroke replay | +### Markdown → Word native math boundary + +Native Word OMath applies only to the Markdown → Word (`.docx`) route and must be enabled with `--math omath`. Without `--math`, or with `--math literal`, dollar-delimited formulas remain literal Markdown text. + +OMath mode supports the [versioned `latex-math-swift` macro subset](https://github.com/PsychQuant/latex-math-swift#supported-macros): fractions and radicals, subscript and superscript, accents, delimiters, n-ary operators, functions, limits, text, Greek symbols, and common operators. Full TeX support and Pandoc texmath parity are outside this capability. Every other conversion route rejects `--math omath`. + ## Rules - Open **one issue per converter** before writing code. diff --git a/Package.resolved b/Package.resolved index c4627984..b51b71f8 100644 --- a/Package.resolved +++ b/Package.resolved @@ -18,6 +18,15 @@ "version" : "1.4.1" } }, + { + "identity" : "latex-math-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/PsychQuant/latex-math-swift.git", + "state" : { + "revision" : "e222b567db1fb23fcd4816471964aad049b9079c", + "version" : "0.2.0" + } + }, { "identity" : "markdown-swift", "kind" : "remoteSourceControl", diff --git a/README.md b/README.md index be23f5b7..a23cf04e 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,9 @@ cp .build/release/macdoc ~/bin/macdoc macdoc convert --to md file.docx macdoc convert --to docx file.md +# Markdown → Word：選擇性將行內 $...$ 與獨立段落 $$...$$ 轉為原生 OMath +macdoc convert --to docx file.md --math omath + # Word ↔ HTML macdoc convert --to html file.docx macdoc convert --to docx file.html @@ -109,6 +112,15 @@ macdoc convert --to html notes.note --full macdoc convert --to html notes.note --full --css dark ``` +Markdown → Word 的數學模式預設為 `literal`，因此未指定 `--math omath` 時， +`$...$` 與 `$$...$$` 會保留為一般 Markdown 文字。原生 OMath 轉換採選擇性啟用， +且只適用於 Markdown → DOCX 路由。 + +`omath` 模式採用 [`latex-math-swift`](https://github.com/PsychQuant/latex-math-swift) +定義的支援子集：分數與根號、上下標、重音符號、成對分隔符號、求和／積分／乘積、 +函數與極限、`\text{}`、希臘字母及常用運算子。這項功能不等同完整 TeX 引擎， +也不追求與 Pandoc texmath 相同的語法涵蓋範圍；子集以連結套件的版本化文件為準。 + 常用選項： | 選項 | 說明 | @@ -117,6 +129,7 @@ macdoc convert --to html notes.note --full --css dark | `--stdout` | 輸出到 stdout | | `--frontmatter` | 含 YAML frontmatter（Word → MD） | | `--html-extensions` | 保留 `///`（→ MD） | +| `--math literal\|omath` | Markdown → DOCX 數學模式（預設 `literal`；`omath` 為原生 Word 數學） | | `--full` | 輸出完整 HTML 文件 | | `--css dark\|light` | SRT 主題 | | `--css minimal\|web` | Bib 樣式 | diff --git a/Sources/MacDocCLI/MacDoc+Convert.swift b/Sources/MacDocCLI/MacDoc+Convert.swift index 21d72814..9071c26c 100644 --- a/Sources/MacDocCLI/MacDoc+Convert.swift +++ b/Sources/MacDocCLI/MacDoc+Convert.swift @@ -21,6 +21,18 @@ import NoteToPDF // MARK: - Convert 子命令（textutil-compatible 統一入口） extension MacDoc { struct Convert: AsyncParsableCommand { + private enum MathOption: String, CaseIterable, ExpressibleByArgument { + case literal + case omath + + var converterMode: MarkdownMathMode { + switch self { + case .literal: .literal + case .omath: .omath + } + } + } + static let configuration = CommandConfiguration( commandName: "convert", abstract: "Convert documents between formats (textutil-compatible)" @@ -50,6 +62,9 @@ extension MacDoc { @Flag(name: .long, help: "Preserve /// as raw HTML in Markdown") var htmlExtensions: Bool = false + @Option(name: .long, help: "Markdown math mode: literal|omath (Markdown to DOCX only; default: literal)") + private var math: MathOption? + @Argument(help: "Input file") var input: String @@ -59,6 +74,10 @@ extension MacDoc { let ext = inputURL.pathExtension.lowercased() let target = to.lowercased() + if math != nil && !(["md", "markdown"].contains(ext) && target == "docx") { + throw ValidationError("--math 只支援 Markdown 轉 DOCX") + } + switch (ext, target) { case ("docx", "md"): try convertWordToMD(inputURL: inputURL) @@ -290,7 +309,8 @@ extension MacDoc { options.hardLineBreaks = hardBreaks let outputURL = try resolveDocxOutputURL(inputURL: inputURL) - try MarkdownToWordConverter().convertToFile(input: inputURL, output: outputURL, options: options) + let converter = MarkdownToWordConverter(mathMode: (math ?? .literal).converterMode) + try converter.convertToFile(input: inputURL, output: outputURL, options: options) FileHandle.standardError.write(Data("已寫入: \(outputURL.path)\n".utf8)) } diff --git a/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift b/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift new file mode 100644 index 00000000..d5f32cde --- /dev/null +++ b/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift @@ -0,0 +1,174 @@ +import Foundation +import Testing + +struct MarkdownOMathRouteTests { + @Test("md to docx defaults to literal dollar text") + func defaultModeIsLiteral() throws { + let workspace = try makeWorkspace() + defer { try? FileManager.default.removeItem(at: workspace) } + let input = workspace.appendingPathComponent("fixture.md") + let output = workspace.appendingPathComponent("fixture.docx") + try "Before $x^2$ after".write(to: input, atomically: true, encoding: .utf8) + + let result = try CLITestHelper.convert( + to: "docx", + input: input.path, + flags: ["--output", output.path] + ) + + #expect(result.succeeded, "stderr: \(result.stderr)") + let xml = try archiveEntry(named: "word/document.xml", in: output) + #expect(xml.contains("Before $x^2$ after")) + #expect(!xml.contains("")) + } + + @Test("explicit literal mode preserves dollar text") + func explicitLiteralMode() throws { + let workspace = try makeWorkspace() + defer { try? FileManager.default.removeItem(at: workspace) } + let input = workspace.appendingPathComponent("fixture.markdown") + let output = workspace.appendingPathComponent("fixture.docx") + try "$x$".write(to: input, atomically: true, encoding: .utf8) + + let result = try CLITestHelper.convert( + to: "docx", + input: input.path, + flags: ["--math", "literal", "--output", output.path] + ) + + #expect(result.succeeded, "stderr: \(result.stderr)") + let xml = try archiveEntry(named: "word/document.xml", in: output) + #expect(xml.contains("$x$")) + #expect(!xml.contains("")) + } + + @Test("omath mode emits inline native Word math") + func inlineOMathMode() throws { + let workspace = try makeWorkspace() + defer { try? FileManager.default.removeItem(at: workspace) } + let input = workspace.appendingPathComponent("fixture.md") + let output = workspace.appendingPathComponent("fixture.docx") + try "Before $x^2$ after".write(to: input, atomically: true, encoding: .utf8) + + let result = try CLITestHelper.convert( + to: "docx", + input: input.path, + flags: ["--math", "omath", "--output", output.path] + ) + + #expect(result.succeeded, "stderr: \(result.stderr)") + #expect(result.stdout.isEmpty) + let xml = try archiveEntry(named: "word/document.xml", in: output) + #expect(xml.components(separatedBy: "").count - 1 == 1) + #expect(!xml.contains("$x^2$")) + } + + @Test("omath mode emits display native Word math") + func displayOMathMode() throws { + let workspace = try makeWorkspace() + defer { try? FileManager.default.removeItem(at: workspace) } + let input = workspace.appendingPathComponent("fixture.md") + let output = workspace.appendingPathComponent("fixture.docx") + try #"$$\frac{a}{b}$$"#.write(to: input, atomically: true, encoding: .utf8) + + let result = try CLITestHelper.convert( + to: "docx", + input: input.path, + flags: ["--math", "omath", "--output", output.path] + ) + + #expect(result.succeeded, "stderr: \(result.stderr)") + let xml = try archiveEntry(named: "word/document.xml", in: output) + #expect(xml.components(separatedBy: "").count - 1 == 1) + #expect(xml.components(separatedBy: "").count - 1 == 1) + } + + @Test("invalid math value fails before replacing destination") + func invalidMathValuePreservesDestination() throws { + let workspace = try makeWorkspace() + defer { try? FileManager.default.removeItem(at: workspace) } + let input = workspace.appendingPathComponent("fixture.md") + let output = workspace.appendingPathComponent("fixture.docx") + let sentinel = Data("KEEP".utf8) + try "$x$".write(to: input, atomically: true, encoding: .utf8) + try sentinel.write(to: output) + + let result = try CLITestHelper.convert( + to: "docx", + input: input.path, + flags: ["--math", "unknown", "--output", output.path] + ) + + #expect(!result.succeeded) + #expect(result.stdout.isEmpty) + #expect(try Data(contentsOf: output) == sentinel) + } + + @Test("omath is rejected outside the Markdown to docx route") + func incompatibleRoutePreservesDestination() throws { + let workspace = try makeWorkspace() + defer { try? FileManager.default.removeItem(at: workspace) } + let input = workspace.appendingPathComponent("fixture.md") + let output = workspace.appendingPathComponent("fixture.html") + let sentinel = Data("KEEP".utf8) + try "$x$".write(to: input, atomically: true, encoding: .utf8) + try sentinel.write(to: output) + + let result = try CLITestHelper.convert( + to: "html", + input: input.path, + flags: ["--math", "omath", "--output", output.path] + ) + + #expect(!result.succeeded) + #expect(result.stdout.isEmpty) + #expect(try Data(contentsOf: output) == sentinel) + } + + @Test("formula error fails before replacing destination") + func formulaErrorPreservesDestination() throws { + let workspace = try makeWorkspace() + defer { try? FileManager.default.removeItem(at: workspace) } + let input = workspace.appendingPathComponent("fixture.md") + let output = workspace.appendingPathComponent("fixture.docx") + let sentinel = Data("KEEP".utf8) + try #"$\overbrace{x}$"#.write(to: input, atomically: true, encoding: .utf8) + try sentinel.write(to: output) + + let result = try CLITestHelper.convert( + to: "docx", + input: input.path, + flags: ["--math", "omath", "--output", output.path] + ) + + #expect(!result.succeeded) + #expect(result.stdout.isEmpty) + #expect(result.stderr.contains("line 1, column 1")) + #expect(try Data(contentsOf: output) == sentinel) + } + + private func makeWorkspace() throws -> URL { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("macdoc-markdown-omath-cli-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + return directory + } + + private func archiveEntry(named path: String, in archiveURL: URL) throws -> String { + let result = try CLITestHelper.runProcess( + executableURL: URL(fileURLWithPath: "/usr/bin/unzip"), + arguments: ["-p", archiveURL.path, path], + currentDirectory: nil, + timeout: 10 + ) + guard result.succeeded else { + throw ArchiveReadError(path: path, diagnostic: result.stderr) + } + return result.stdout + } +} + +private struct ArchiveReadError: Error { + let path: String + let diagnostic: String +} diff --git a/openspec/changes/markdown-omath-conversion/.openspec.yaml b/openspec/changes/markdown-omath-conversion/.openspec.yaml new file mode 100644 index 00000000..85347286 --- /dev/null +++ b/openspec/changes/markdown-omath-conversion/.openspec.yaml @@ -0,0 +1,4 @@ +schema: spec-driven +created: 2026-08-14 +created_by: che cheng +created_with: Codex diff --git a/openspec/changes/markdown-omath-conversion/design.md b/openspec/changes/markdown-omath-conversion/design.md new file mode 100644 index 00000000..f2a0d74b --- /dev/null +++ b/openspec/changes/markdown-omath-conversion/design.md @@ -0,0 +1,134 @@ +## Context + +`MarkdownToWordConverter` currently parses Markdown with `swift-markdown` and turns inline AST nodes into `OOXMLSwift.Run` values. Dollar-delimited math has no dedicated AST node in this parser configuration, so delimiters remain ordinary text. The repository already maintains two suitable lower layers: `latex-math-swift` v0.2.0 parses a frozen LaTeX subset into `[MathComponent]`, and `ooxml-swift` serializes those components as OMML. + +The integration must preserve the existing output for callers that do not opt in, must not parse Markdown code or link destinations as equations, and must fail before replacing an output file when a recognized formula is unsupported. + +## Goals / Non-Goals + +**Goals:** + +- Add a stable public `MarkdownMathMode` with `.literal` and `.omath` cases. +- Convert conservative inline and display dollar delimiters to the correct native Word OMML carriers. +- Reuse the versioned `LaTeXMathParser` and expose stable MDToWord-layer errors with source locations. +- Preserve Markdown structure around formulas and exclude code, destinations, and escaped dollars. +- Prove package API and compiled CLI behavior, including destination preservation on formula failure. + +**Non-Goals:** + +- Full TeX, MathJax, KaTeX, or Pandoc texmath compatibility. +- A new LaTeX parser or changes to the existing `latex-math-parsing` contract. +- Math conversion in Markdown-to-HTML, PDF, or unrelated routes. +- Formulas crossing fenced-code, inline-code, link-destination, HTML-tag, or Markdown block boundaries. +- Automatic fallback from an invalid recognized formula to literal text in `.omath` mode. + +## Decisions + +### Public math mode is converter configuration with a literal default + +`MDToWord` will expose: + +```swift +public enum MarkdownMathMode: String, Sendable { + case literal + case omath +} + +public init(mathMode: MarkdownMathMode = .literal) +``` + +The mode belongs to `MarkdownToWordConverter`, not shared `ConversionOptions`, because it is input-format-specific and no other converter consumes it. Existing initializers and calls retain literal behavior. The CLI defines an ArgumentParser-facing enum with the same two raw values and passes it to the converter only for Markdown-to-DOCX. + +Alternatives rejected: + +- Adding math fields to `CommonConverterSwift.ConversionOptions` would impose an irrelevant public option on every converter. +- Enabling OMath by default would change existing documents and turn previously harmless dollar text into parse errors. + +### A conservative lexical tokenizer runs before the Markdown AST builder + +An internal `MarkdownMathScanner` will produce a transformed Markdown string plus a token table. It replaces recognized math spans with collision-resistant placeholders before `swift-markdown` parses the document. This preserves surrounding emphasis, headings, lists, quotes, and tables while preventing the Markdown parser from interpreting LaTeX punctuation. + +The scanner is a single forward pass with explicit states for fenced code, equal-length backtick code spans, escaped characters, angle-bracket HTML/autolink regions, and Markdown link/image destinations. It recognizes: + +- inline math: one unescaped `$`, non-whitespace first and last formula characters, no newline, and a valid closing `$`; +- display math: `$$...$$` occupying an entire logical paragraph, either on one trimmed line or with standalone opening and closing delimiter lines. + +Unmatched dollars remain literal. A matched display token found alongside non-whitespace paragraph content is rejected rather than silently converted inline. A token placeholder is generated only after proving it does not already occur in the source. + +Alternatives rejected: + +- A regular expression cannot reliably exclude code, escapes, or link destinations. +- Scanning decoded `Text.string` values loses enough source spelling to distinguish escaped dollars. +- Invoking Pandoc introduces a subprocess and a second conversion pipeline instead of using the maintained Swift parser. + +### Tokens become direct paragraph children with carrier-specific wrappers + +Inline tokens are emitted as `Run.rawXML` containing one `...` fragment. Since `Run.rawXML` is serialized verbatim as a paragraph child, it is not wrapped in an invalid `` container. + +Display tokens create a paragraph whose `unrecognizedChildren` contains one direct `...` child and no synthetic text run. Both wrappers use `MathComponent.toOMML()` for their interior. The streaming XML route adds the standard math namespace whenever generated body XML contains an `m:` element; the DOCX writer route uses the OOXML writer's namespace-aware document serializer. + +Surrounding Markdown run properties do not alter OMML nodes in v1. A formula remains structurally positioned inside the surrounding paragraph, but bold or italic Markdown wrappers do not rewrite `m:rPr`. + +### Recognized formula failures are stable, located, and pre-write + +`MDToWord` will expose a public `MarkdownMathConversionError` that reports the one-based line and column of a recognized formula and a stable reason category: malformed or unsupported. Internal `LaTeXParseError` details are normalized into this type so callers do not depend on transitive parser implementation. + +The converter scans and parses every recognized formula while building the complete in-memory `WordDocument`. `convertToFile` calls `DocxWriter` only after that succeeds. Therefore a formula error creates no destination and leaves an existing destination byte-identical. Unmatched currency-like dollar text is not an error because it is not a recognized formula span. + +### CLI option is route-scoped and fail-loud + +`macdoc convert` accepts `--math literal|omath`, defaulting to `literal`. `--math omath` is valid only for Markdown or `.markdown` input with DOCX output. Invalid enum values and use on any other route fail argument validation before conversion and do not create or replace a destination. + +The success path continues to write the normal one-line destination message to stderr. Formula errors are rendered through ArgumentParser's existing diagnostic path and do not write formula output to stdout. + +## Implementation Contract + +### Behavior + +- Existing `MarkdownToWordConverter()` and CLI calls without `--math` preserve literal dollar text. +- `.omath` converts `$x^2$` to one inline `` and converts a standalone `$$\frac{a}{b}$$` block to one `` containing one ``. +- `\$5`, inline/fenced code, autolinks, link destinations, and unmatched dollars remain non-math content. +- A recognized but unsupported expression, such as `$\overbrace{x}$`, fails the entire conversion without partial output. + +### Interface and data shape + +- Public library types: `MarkdownMathMode` and `MarkdownMathConversionError`. +- Public converter initializer: `MarkdownToWordConverter(mathMode:)`, with `.literal` default. +- CLI surface: `macdoc convert --to docx --math literal|omath --output `. +- Output remains a DOCX package; native math appears only in `word/document.xml` using the standard OMML namespace. + +### Failure modes + +- Invalid CLI mode or incompatible route: argument-validation failure, non-zero exit, empty stdout, destination unchanged. +- Matched unsupported or malformed formula: typed library error and non-zero CLI exit; source location is included; destination unchanged. +- Unmatched delimiter: preserved as literal text, not treated as a parser failure. +- Scanner placeholder collision: scanner selects another placeholder; caller input is never overwritten by a reserved literal. + +### Acceptance criteria + +- Package tests inspect generated XML for exact inline/display wrapper counts and no surviving formula delimiters. +- Boundary tables cover escaped dollar, currency-like text, code span, fenced code, link destination, unmatched delimiter, and malformed supported-span cases. +- Public API type-checks from a client target without `@testable` imports. +- Compiled CLI tests cover default literal, opt-in inline/display OMath, invalid route, invalid value, and existing-destination preservation on parser failure. +- ~~`swift test` passes in `packages/md-to-word-swift` and at the repository root, and Spectra validation has no Critical or Warning findings.~~ +- Apply-time baseline qualification: a clean `origin/main` checkout executes 89 package tests with 42 pre-existing `E2ETests`/`RoundTripTests` failures. This change adds focused math coverage while retaining exactly those same 42 failures. Acceptance therefore requires every new and directly affected test to pass, the full package run to introduce zero failures beyond that documented clean baseline, the root suite to introduce no regression, and Spectra validation to have no Critical or Warning findings. Repairing the unrelated round-trip baseline is outside this change. + +### Scope boundaries + +In scope: Markdown-to-DOCX library and CLI behavior, parser dependency wiring, package/root tests, route documentation, and specs. Out of scope: parser macro expansion, OOXML math model changes, Pandoc parity, other output formats, downstream bestOCR changes, and formula styling beyond native default math formatting. + +## Risks / Trade-offs + +- [A custom scanner can drift from CommonMark edge cases] → Keep recognition conservative, use a single forward state machine, and lock exclusions with table-driven tests. +- [Placeholder text could collide with user content] → Generate a per-conversion marker and verify absence before substitution. +- [Parser and OOXML dependency versions can select incompatible `MathComponent` definitions] → Pin `latex-math-swift` from v0.2.0 and verify SwiftPM resolves one compatible `ooxml-swift` graph. +- [Raw OMML could be emitted under an undeclared prefix] → Test both streaming XML and archived `word/document.xml` with XML parsing and the standard math namespace. +- [Fail-loud behavior is stricter than literal fallback] → Make it opt-in only and preserve the literal default. + +## Migration Plan + +No caller migration is required because the default remains `.literal`. Release notes document the opt-in mode and supported subset. Rollback removes the CLI option and dependency while existing literal calls remain source-compatible. + +## Open Questions + +(none) diff --git a/openspec/changes/markdown-omath-conversion/proposal.md b/openspec/changes/markdown-omath-conversion/proposal.md new file mode 100644 index 00000000..ad4265bd --- /dev/null +++ b/openspec/changes/markdown-omath-conversion/proposal.md @@ -0,0 +1,47 @@ +## Why + +The Markdown-to-DOCX route currently serializes `$...$` and `$$...$$` as literal Word text, so academic Markdown loses native equation semantics. The repository already has a versioned LaTeX-subset parser and OMML emitters; this change connects them through an explicit, backward-compatible conversion mode. + +## What Changes + +- Add a public `MDToWord` math mode whose default preserves the current literal behavior and whose opt-in OMath mode converts supported inline and display LaTeX delimiters to native Word OMML. +- Add a delimiter scanner that excludes escaped dollar signs, inline code, fenced code, and formulas crossing Markdown formatting-node boundaries. +- Reuse `LaTeXMathParser` from `latex-math-swift`; unsupported or malformed formulas fail loudly before any destination DOCX is replaced. +- Add `macdoc convert --to docx ... --math literal|omath`, with route validation that rejects the option on incompatible source/target pairs. +- Expand package-level and compiled-CLI acceptance coverage to inspect `word/document.xml`, distinguish inline from display carriers, and prove failure leaves an existing destination unchanged. +- Document the supported LaTeX subset and the intentional gap from complete TeX or Pandoc texmath compatibility. + +## Non-Goals + +- Implementing a second LaTeX parser inside macdoc or `md-to-word-swift`. +- Claiming complete TeX, MathJax, KaTeX, or Pandoc texmath compatibility. +- Converting math embedded in code spans, fenced code, URLs, image destinations, or delimiters split across separate Markdown AST text nodes. +- Changing the default output of existing `MarkdownToWordConverter()` or `macdoc convert --to docx` calls. +- Adding OMath support to non-DOCX conversion routes. + +## Capabilities + +### New Capabilities + +- `markdown-omath-conversion`: Public Markdown-to-DOCX math-mode semantics, delimiter recognition, OMML carrier rules, fail-loud behavior, and package-level acceptance. + +### Modified Capabilities + +- `e2e-conversion-routes`: Add compiled CLI coverage for the `--math` flag, native OMML output, route validation, and destination-preserving failures. + +## Impact + +- Affected specs: `markdown-omath-conversion`, `e2e-conversion-routes` +- Affected code: + - New: + - `packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift` + - `packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift` + - `Tests/MacDocCLITests/MarkdownOMathRouteTests.swift` + - Modified: + - `packages/md-to-word-swift/Sources/MDToWord/MarkdownToWordConverter.swift` + - `packages/md-to-word-swift/Package.swift` + - `Sources/MacDocCLI/MacDoc+Convert.swift` + - `Package.resolved` + - `README.md` + - `CONVERSIONS.md` + - Removed: (none) diff --git a/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md b/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md new file mode 100644 index 00000000..60a04426 --- /dev/null +++ b/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md @@ -0,0 +1,43 @@ +## ADDED Requirements + +### Requirement: Compiled Markdown OMath route coverage + +The compiled E2E suite SHALL exercise `macdoc convert` through the production command parser for default literal mode, opt-in inline OMath, opt-in display OMath, invalid math values, incompatible routes, and formula failures. Binary-output assertions SHALL inspect `word/document.xml`, not only destination existence. + +#### Scenario: Default CLI mode remains literal + +- **WHEN** the compiled binary runs `macdoc convert --to docx --output ` for source `Before $x^2$ after` +- **THEN** exit code is 0 +- **AND** `word/document.xml` contains literal `$x^2$` and no ` --math omath --output ` for a fixture containing one inline and one display formula +- **THEN** exit code is 0 and stdout is empty +- **AND** `word/document.xml` contains one inline `` plus one `` with a nested `` +- **AND** the standard Office Math namespace is bound + +#### Scenario: Invalid math value is rejected + +- **WHEN** the compiled binary runs with `--math unknown` +- **THEN** exit code is non-zero and stdout is empty +- **AND** stderr identifies the invalid `--math` value +- **AND** the destination is not created + +#### Scenario: OMath mode is rejected on an incompatible route + +- **WHEN** the compiled binary runs `macdoc convert --to html --math omath` +- **THEN** exit code is non-zero and stdout contains no converted document +- **AND** stderr states that OMath mode is available only for Markdown-to-DOCX + +#### Scenario: Formula failure preserves existing CLI destination + +- **GIVEN** an existing output file containing sentinel bytes `KEEP` +- **WHEN** the compiled binary converts Markdown containing `$\\overbrace{x}$` with `--math omath` to that output path +- **THEN** exit code is non-zero and stdout is empty +- **AND** the output file remains byte-identical to `KEEP` + +#### Scenario: Literal flag is accepted explicitly + +- **WHEN** the compiled binary runs a Markdown-to-DOCX conversion with `--math literal` +- **THEN** exit code is 0 and delimiters remain literal Word text diff --git a/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md b/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md new file mode 100644 index 00000000..a4535ea8 --- /dev/null +++ b/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md @@ -0,0 +1,134 @@ +## ADDED Requirements + +### Requirement: Markdown-to-Word exposes an opt-in native math mode + +The `MDToWord` library SHALL expose a public `MarkdownMathMode` enum with raw string values `literal` and `omath`. `MarkdownToWordConverter` SHALL provide `init(mathMode: MarkdownMathMode = .literal)`. The default initializer and every conversion using `.literal` SHALL preserve dollar-delimited input as ordinary Markdown text and SHALL NOT invoke LaTeX parsing. + +#### Scenario: Default converter preserves delimiters + +- **WHEN** `MarkdownToWordConverter()` converts `Before $x^2$ after` +- **THEN** the generated document contains the literal text `$x^2$` +- **AND** `word/document.xml` contains no `` containing the parsed OMML components. A display formula SHALL serialize as one direct paragraph child `` containing one `` and SHALL NOT generate a synthetic `` run for the formula. Generated XML containing either carrier SHALL bind prefix `m` to `http://schemas.openxmlformats.org/officeDocument/2006/math`. + +#### Scenario: Mixed text and inline formula preserve source order + +- **WHEN** `.omath` conversion receives `Before $x^2$ after` +- **THEN** `word/document.xml` orders a text run containing `Before `, one ``, and a text run containing ` after` +- **AND** no literal `$x^2$` remains + +##### Example: Inline carrier sequence + +- **GIVEN** source `Before $x^2$ after` +- **WHEN** it is converted in `.omath` mode +- **THEN** the paragraph child sequence is `w:r`, `m:oMath`, `w:r` + +#### Scenario: Display formula uses oMathPara + +- **WHEN** `.omath` conversion receives a standalone display formula `$$\\frac{a}{b}$$` +- **THEN** its paragraph contains exactly one `` and exactly one nested `` +- **AND** the paragraph contains no literal delimiters and no synthetic `` for the formula + +#### Scenario: Streaming XML declares the math namespace + +- **WHEN** the `DocumentConverter.convert` streaming surface emits a document containing inline or display math +- **THEN** the document root binds `xmlns:m` to the standard Office Math namespace +- **AND** an XML parser accepts the emitted document + +### Requirement: Formula failures occur before destination replacement + +`convertMarkdown`, `convertToDocument`, and `convertToFile` SHALL parse every recognized formula before returning success. `convertToFile` SHALL NOT invoke the DOCX writer when scanning or formula parsing fails. On failure, a nonexistent destination SHALL remain absent and an existing destination SHALL remain byte-identical. + +#### Scenario: Unsupported formula leaves absent destination absent + +- **WHEN** `convertToFile` receives `$\\overbrace{x}$` in `.omath` mode and the destination does not exist +- **THEN** conversion throws `MarkdownMathConversionError.unsupportedFormula` +- **AND** the destination is not created + +#### Scenario: Malformed formula preserves existing destination + +- **GIVEN** an existing destination containing sentinel bytes `KEEP` +- **WHEN** `convertToFile` receives `$\\frac{a}{b$` in `.omath` mode +- **THEN** conversion throws `MarkdownMathConversionError.malformedFormula` +- **AND** the destination bytes remain exactly `KEEP` + +### Requirement: Documentation states the native math boundary + +The route documentation SHALL state that native OMath is opt-in, SHALL name `literal` as the default, SHALL list the supported macro families by linking to `latex-math-swift`, and SHALL state that full TeX and Pandoc texmath parity are outside this capability. + +#### Scenario: Conversion documentation identifies mode and subset + +- **WHEN** a user reads the Markdown-to-DOCX route documentation +- **THEN** the documentation shows `--math omath`, the literal default, and the parser-subset boundary diff --git a/openspec/changes/markdown-omath-conversion/tasks.md b/openspec/changes/markdown-omath-conversion/tasks.md new file mode 100644 index 00000000..36dc1fd2 --- /dev/null +++ b/openspec/changes/markdown-omath-conversion/tasks.md @@ -0,0 +1,33 @@ +## 1. Public mode and dependency contract + +- [x] 1.1 Add RED package tests for **Markdown-to-Word exposes an opt-in native math mode** and **Public math mode is converter configuration with a literal default**: an ordinary `import MDToWord` client can construct `.omath`, while default and explicit `.literal` preserve `$x^2$`; verify the focused `MarkdownOMathConversionTests` first fails for missing public symbols rather than test syntax. +- [x] 1.2 Wire `latex-math-swift` v0.2.0 into `packages/md-to-word-swift/Package.swift`, add `MarkdownMathMode` and `MarkdownToWordConverter(mathMode:)`, and make the task 1.1 tests green while `swift package show-dependencies` proves one compatible `OOXMLSwift` resolution. + +## 2. Conservative source tokenizer + +- [x] 2.1 Add RED table-driven tests for **OMath mode recognizes conservative dollar delimiters** and **A conservative lexical tokenizer runs before the Markdown AST builder**, covering inline/display recognition, escaped dollars, currency-like text, unmatched delimiters, code spans, fenced code, autolinks, link/image destinations, mixed display content, and source locations; verify the focused scanner tests fail before implementation. +- [x] 2.2 Implement the single-pass `MarkdownMathScanner` with collision-checked placeholders and explicit lexical states so every task 2.1 boundary becomes green without changing `.literal` input; verify focused scanner tests and existing `MarkdownToWordConverterTests` pass. + +## 3. Parser and native OMML carriers + +- [x] 3.1 Add RED conversion tests for **Recognized formulas use the versioned LaTeX parser**, **Inline and display math use native Word carriers**, and **Tokens become direct paragraph children with carrier-specific wrappers**, asserting parser-subset fraction output, source-order `w:r/m:oMath/w:r`, display `m:oMathPara`, exact carrier counts, no delimiters, and valid `xmlns:m`; verify failures precede production emission changes. +- [x] 3.2 Connect scanned tokens to `LaTeXMathParser`, `MathComponent.toOMML()`, inline `Run.rawXML`, display `Paragraph.unrecognizedChildren`, and conditional streaming namespace emission; verify task 3.1 tests parse both streaming XML and archived `word/document.xml` successfully. + +## 4. Stable errors and destination safety + +- [x] 4.1 Add RED tests for **Formula failures occur before destination replacement** and **Recognized formula failures are stable, located, and pre-write**, covering unsupported and malformed expressions, one-based line/column, absent output, and an existing `KEEP` destination; verify focused tests fail for missing typed errors or unsafe behavior. +- [x] 4.2 Implement public `MarkdownMathConversionError` normalization and complete all scanning/parsing before `DocxWriter` is called, so unsupported/malformed/misplaced display errors are stable and both absent/existing destination assertions pass; verify task 4.1 tests are green and the full package run has exactly the clean `origin/main` baseline's 42 pre-existing `E2ETests`/`RoundTripTests` failures with zero new failures. + +## 5. Route-scoped compiled CLI + +- [x] 5.1 Add RED compiled-binary tests for **Compiled Markdown OMath route coverage** and **CLI option is route-scoped and fail-loud**, covering default literal, inline/display `--math omath`, explicit literal, invalid value, incompatible route, empty stdout on failure, and `KEEP` destination preservation; verify the focused CLI suite fails because the option is not implemented. +- [x] 5.2 Add the ArgumentParser `--math literal|omath` surface and Markdown-to-DOCX route validation in `Sources/MacDocCLI/MacDoc+Convert.swift`, then update `Package.resolved`; verify all task 5.1 compiled invocations pass and incompatible routes never call the converter. + +## 6. User-facing boundary documentation + +- [x] [P] 6.1 Fulfill **Documentation states the native math boundary** in `README.md`: show opt-in `--math omath`, literal default, supported parser-subset link, and explicit non-parity with full TeX/Pandoc; verify a content assertion finds all four claims. +- [x] [P] 6.2 Fulfill **Documentation states the native math boundary** in `CONVERSIONS.md`: annotate Markdown→Word native math mode and subset/non-goals without marking other routes as supported; verify the conversion matrix and notes contain the scoped command and no OMath claim on HTML/PDF routes. + +## 7. Contract and regression verification + +- [x] 7.1 Verify the design **Behavior**, **Interface and data shape**, **Failure modes**, **Acceptance criteria**, and **Scope boundaries** end to end: run focused scanner/converter/CLI tests, full `packages/md-to-word-swift` and root `swift test`, a non-`@testable` client type-check, dependency inspection, `git diff --check`, `spectra analyze markdown-omath-conversion --json`, and `spectra validate markdown-omath-conversion`; every new/focused test passes, full runs introduce zero failures beyond the documented clean baseline, and the analyzer has no Critical or Warning findings. diff --git a/packages/md-to-word-swift/Package.swift b/packages/md-to-word-swift/Package.swift index f8b8fab8..7804a1b0 100644 --- a/packages/md-to-word-swift/Package.swift +++ b/packages/md-to-word-swift/Package.swift @@ -10,7 +10,8 @@ let package = Package( dependencies: [ .package(url: "https://github.com/apple/swift-markdown.git", from: "0.4.0"), .package(url: "https://github.com/PsychQuant/common-converter-swift.git", from: "0.4.0"), - .package(url: "https://github.com/PsychQuant/ooxml-swift.git", from: "0.5.1"), + .package(url: "https://github.com/PsychQuant/ooxml-swift.git", from: "1.4.0"), + .package(url: "https://github.com/PsychQuant/latex-math-swift.git", from: "0.2.0"), .package(url: "https://github.com/jpsim/Yams.git", from: "5.0.0"), .package(url: "https://github.com/PsychQuant/word-to-md-swift.git", from: "0.4.0"), ], @@ -21,6 +22,7 @@ let package = Package( .product(name: "Markdown", package: "swift-markdown"), .product(name: "CommonConverterSwift", package: "common-converter-swift"), .product(name: "OOXMLSwift", package: "ooxml-swift"), + .product(name: "LaTeXMathSwift", package: "latex-math-swift"), .product(name: "Yams", package: "Yams"), ] ), diff --git a/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathConversionError.swift b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathConversionError.swift new file mode 100644 index 00000000..3fd34ca6 --- /dev/null +++ b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathConversionError.swift @@ -0,0 +1,22 @@ +import Foundation + +/// Stable Markdown-to-Word errors for recognized math spans that cannot be +/// converted to native Office Math. +public enum MarkdownMathConversionError: Error, Equatable, Sendable { + case malformedFormula(line: Int, column: Int) + case unsupportedFormula(token: String, line: Int, column: Int) + case misplacedDisplayFormula(line: Int, column: Int) +} + +extension MarkdownMathConversionError: LocalizedError { + public var errorDescription: String? { + switch self { + case .malformedFormula(let line, let column): + return "Malformed Markdown math formula at line \(line), column \(column)." + case .unsupportedFormula(let token, let line, let column): + return "Unsupported LaTeX token \(token) at line \(line), column \(column)." + case .misplacedDisplayFormula(let line, let column): + return "Display math must occupy its own paragraph at line \(line), column \(column)." + } + } +} diff --git a/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift new file mode 100644 index 00000000..77c36fb4 --- /dev/null +++ b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift @@ -0,0 +1,604 @@ +import Foundation + +struct MarkdownMathScanner { + enum TokenKind: Equatable { + case inline + case display + } + + struct Token: Equatable { + let placeholder: String + let latex: String + let kind: TokenKind + let line: Int + let column: Int + } + + struct Result: Equatable { + let markdown: String + let tokens: [Token] + } + + enum ScanError: Error, Equatable { + case misplacedDisplayFormula(line: Int, column: Int) + } + + private let markerPrefix: String + private let markerNonce: String + + init( + markerPrefix: String = "MDTOWORDMATHPLACEHOLDER", + markerNonce: String = UUID().uuidString.replacingOccurrences(of: "-", with: "") + ) { + self.markerPrefix = markerPrefix + self.markerNonce = markerNonce + } + + func scan(_ source: String) throws -> Result { + let characters = Array(source) + var output = "" + var tokens: [Token] = [] + var nextMarkerIndex = 0 + var index = 0 + var line = 1 + var column = 1 + var lineStart = 0 + var fence: (character: Character, length: Int)? + + func marker() -> String { + var candidate: String + repeat { + candidate = "\(markerPrefix)\(markerNonce)\(nextMarkerIndex)TOKEN" + nextMarkerIndex += 1 + } while source.contains(candidate) || tokens.contains(where: { $0.placeholder == candidate }) + return candidate + } + + while index < characters.count { + if index == lineStart { + let end = Self.lineEnd(in: characters, from: index) + if let activeFence = fence { + let contentEnd = Self.contentEndBeforeNewline(in: characters, lineEnd: end) + if Self.isFenceClosingLine( + characters, + start: index, + end: contentEnd, + fence: activeFence + ) { + fence = nil + } + output += String(characters[index.. DisplayMatch? { + let openingLineEnd = contentEndBeforeNewline( + in: characters, + lineEnd: lineEnd(in: characters, from: opening) + ) + let prefixIsWhitespace = characters[lineStart.. Int? { + var index = opening + 1 + while index < characters.count { + if characters[index].isNewline { + return nil + } + if characters[index] == "\\" { + index = min(index + 2, characters.count) + continue + } + if characters[index] == "$" { + return index + } + index += 1 + } + return nil + } + + private static func doubleDollarClosing( + in characters: [Character], + from start: Int, + before end: Int + ) -> Int? { + var index = start + while index + 1 < end { + if characters[index] == "\\" { + index += 2 + continue + } + if characters[index] == "$", characters[index + 1] == "$" { + return index + } + index += 1 + } + return nil + } + + private static func fenceOpening( + _ characters: [Character], + start: Int, + end: Int + ) -> (character: Character, length: Int)? { + var index = start + var indentation = 0 + while index < end, characters[index] == " ", indentation < 4 { + index += 1 + indentation += 1 + } + guard indentation <= 3, index < end else { return nil } + let character = characters[index] + guard character == "`" || character == "~" else { return nil } + let length = runLength(of: character, in: characters, from: index) + guard length >= 3 else { return nil } + return (character, length) + } + + private static func isReferenceDefinitionLine( + _ characters: [Character], + start: Int, + end: Int + ) -> Bool { + var index = start + var indentation = 0 + while index < end, characters[index] == " ", indentation < 4 { + index += 1 + indentation += 1 + } + guard indentation <= 3, index < end, characters[index] == "[" else { + return false + } + + index += 1 + let labelStart = index + while index < end { + if characters[index] == "\\" { + index = min(index + 2, end) + continue + } + if characters[index] == "]" { + guard index > labelStart else { return false } + let colon = index + 1 + return colon < end && characters[colon] == ":" + } + index += 1 + } + return false + } + + private static func isFenceClosingLine( + _ characters: [Character], + start: Int, + end: Int, + fence: (character: Character, length: Int) + ) -> Bool { + var index = start + var indentation = 0 + while index < end, characters[index] == " ", indentation < 4 { + index += 1 + indentation += 1 + } + guard indentation <= 3, index < end else { return false } + let length = runLength(of: fence.character, in: characters, from: index) + guard length >= fence.length else { return false } + return characters[(index + length).. Int? { + var index = start + while index < characters.count { + if characters[index] == "`" { + let length = runLength(of: "`", in: characters, from: index) + if length == delimiterLength { + return index + length + } + index += length + continue + } + index += 1 + } + return nil + } + + private static func angleRegionEnd( + in characters: [Character], + from start: Int + ) -> Int? { + var index = start + 1 + while index < characters.count { + if characters[index] == ">" { + return index + 1 + } + if characters[index].isNewline { + return nil + } + index += 1 + } + return nil + } + + private static func linkDestinationEnd( + in characters: [Character], + from openingParenthesis: Int + ) -> Int? { + var depth = 0 + var index = openingParenthesis + while index < characters.count { + if characters[index] == "\\" { + index = min(index + 2, characters.count) + continue + } + if characters[index] == "(" { + depth += 1 + } else if characters[index] == ")" { + depth -= 1 + if depth == 0 { + return index + 1 + } + } else if characters[index].isNewline { + return nil + } + index += 1 + } + return nil + } + + private static func runLength( + of character: Character, + in characters: [Character], + from start: Int + ) -> Int { + var end = start + while end < characters.count, characters[end] == character { + end += 1 + } + return end - start + } + + private static func lineEnd(in characters: [Character], from start: Int) -> Int { + var end = start + while end < characters.count { + if characters[end].isNewline { + return end + 1 + } + end += 1 + } + return end + } + + private static func contentEndBeforeNewline( + in characters: [Character], + lineEnd: Int + ) -> Int { + guard lineEnd > 0, lineEnd <= characters.count else { return lineEnd } + var end = lineEnd + if end > 0, characters[end - 1].isNewline { + end -= 1 + } + return end + } + + private static func advance( + _ characters: [Character], + from start: Int, + to end: Int, + line: inout Int, + column: inout Int, + lineStart: inout Int + ) { + guard start < end else { return } + for index in start..( input: URL, @@ -53,29 +70,69 @@ public struct MarkdownToWordConverter: DocumentConverter { options: ConversionOptions = .default ) throws -> WordDocument { let extracted = FrontmatterExtractor.extract(from: source) + let markdown: String + let mathTokens: [RenderedMarkdownMathToken] + if mathMode == .omath { + let scanned: MarkdownMathScanner.Result + do { + scanned = try MarkdownMathScanner().scan(extracted.body) + } catch let error as MarkdownMathScanner.ScanError { + switch error { + case .misplacedDisplayFormula(let line, let column): + throw MarkdownMathConversionError.misplacedDisplayFormula( + line: line + extracted.bodyStartLine - 1, + column: column + ) + } + } + markdown = scanned.markdown + mathTokens = try scanned.tokens.map { token in + do { + let components = try LaTeXMathParser.parse(token.latex) + return RenderedMarkdownMathToken( + placeholder: token.placeholder, + omml: components.map { $0.toOMML() }.joined(), + kind: token.kind + ) + } catch let error as LaTeXParseError { + switch error { + case .unrecognizedToken(let unsupportedToken): + throw MarkdownMathConversionError.unsupportedFormula( + token: unsupportedToken, + line: token.line + extracted.bodyStartLine - 1, + column: token.column + ) + case .empty, .malformed: + throw MarkdownMathConversionError.malformedFormula( + line: token.line + extracted.bodyStartLine - 1, + column: token.column + ) + } + } + } + } else { + markdown = extracted.body + mathTokens = [] + } var builder = MarkdownWordBuilder( options: options, baseURL: baseURL, sourceName: sourceName, - frontmatter: extracted.metadata + frontmatter: extracted.metadata, + mathTokens: mathTokens ) - return try builder.build(markdown: extracted.body) + return try builder.build(markdown: markdown) } private func renderDocumentXML(_ document: WordDocument) -> String { - var xml = """ - - - - """ + var bodyXML = "" for child in document.body.children { switch child { case .paragraph(let paragraph): - xml += paragraph.toXML() + bodyXML += paragraph.toXML() case .table(let table): - xml += table.toXML() + bodyXML += table.toXML() default: // BodyChild added .contentControl / .bookmarkMarker / // .rawBlockElement in newer ooxml-swift. Skip silently — @@ -84,9 +141,17 @@ public struct MarkdownToWordConverter: DocumentConverter { } } - xml += renderSectionPropertiesXML(document.sectionProperties) - xml += "" - return xml + bodyXML += renderSectionPropertiesXML(document.sectionProperties) + let mathNamespace = bodyXML.contains(" + + \(bodyXML) + """ } private func renderSectionPropertiesXML(_ section: SectionProperties) -> String { @@ -146,18 +211,21 @@ private struct MarkdownWordBuilder { private let baseURL: URL? private let sourceName: String? private let frontmatter: [String: String] + private let mathTokens: [RenderedMarkdownMathToken] private var inferredTitle = false init( options: ConversionOptions, baseURL: URL?, sourceName: String?, - frontmatter: [String: String] + frontmatter: [String: String], + mathTokens: [RenderedMarkdownMathToken] ) { self.options = options self.baseURL = baseURL self.sourceName = sourceName self.frontmatter = frontmatter + self.mathTokens = mathTokens } mutating func build(markdown: String) throws -> WordDocument { @@ -376,6 +444,30 @@ private struct MarkdownWordBuilder { extraIndentLevels: Int = 0, style: String? = nil ) throws -> WordParagraph? { + if let token = displayToken(in: children) { + var paragraph = WordParagraph(runs: []) + paragraph.unrecognizedChildren = [ + UnrecognizedChild( + name: "oMathPara", + rawXML: "\(token.omml)", + position: 0 + ) + ] + paragraph.properties.style = style + paragraph.properties.numbering = numbering + paragraph.properties.spacing = Spacing( + after: numbering == nil ? 200 : 80, + line: 276, + lineRule: .auto + ) + applyQuoteStyle( + to: ¶graph.properties, + quoteDepth: quoteDepth, + extraIndentLevels: extraIndentLevels + ) + return paragraph + } + var runs: [Run] = [] for child in children { try appendInline(from: child, into: &runs, properties: RunProperties()) @@ -408,7 +500,7 @@ private struct MarkdownWordBuilder { switch markup { case let text as Text: guard !text.string.isEmpty else { return } - runs.append(Run(text: text.string, properties: properties)) + appendText(text.string, into: &runs, properties: properties) case let emphasis as Emphasis: var next = properties @@ -479,7 +571,7 @@ private struct MarkdownWordBuilder { : "[Image: \(image.plainText)]" var next = properties next.italic = true - runs.append(Run(text: fallback, properties: next)) + appendText(fallback, into: &runs, properties: next) case let inlineHTML as InlineHTML: let stripped = stripHTML(from: inlineHTML.rawHTML) @@ -585,14 +677,7 @@ private struct MarkdownWordBuilder { private func makeExternalHyperlinkXML(text: String, relationshipId: String) -> String { """ - - - - - - - \(escapeXML(text)) - + \(hyperlinkContentXML(text)) """ } @@ -600,18 +685,41 @@ private struct MarkdownWordBuilder { private func makeInternalHyperlinkXML(text: String, anchor: String) -> String { """ - - - - - - - \(escapeXML(text)) - + \(hyperlinkContentXML(text)) """ } + private func hyperlinkContentXML(_ text: String) -> String { + var xml = "" + var cursor = text.startIndex + while cursor < text.endIndex { + guard let match = nextInlineMathMatch(in: text, from: cursor) else { + xml += hyperlinkTextRunXML(String(text[cursor...])) + break + } + if cursor < match.range.lowerBound { + xml += hyperlinkTextRunXML(String(text[cursor.. String { + """ + + + + + + + \(escapeXML(text)) + + """ + } + private func makeBreakRun() -> Run { makeRawRun("") } @@ -622,6 +730,59 @@ private struct MarkdownWordBuilder { return run } + private func appendText( + _ text: String, + into runs: inout [Run], + properties: RunProperties + ) { + var cursor = text.startIndex + while cursor < text.endIndex { + guard let match = nextInlineMathMatch(in: text, from: cursor) else { + runs.append(Run(text: String(text[cursor...]), properties: properties)) + return + } + if cursor < match.range.lowerBound { + runs.append( + Run( + text: String(text[cursor..\(match.token.omml)")) + cursor = match.range.upperBound + } + } + + private func nextInlineMathMatch( + in text: String, + from cursor: String.Index + ) -> (range: Range, token: RenderedMarkdownMathToken)? { + var match: (range: Range, token: RenderedMarkdownMathToken)? + for token in mathTokens where token.kind == .inline { + guard let range = text.range( + of: token.placeholder, + range: cursor.. RenderedMarkdownMathToken? { + guard children.count == 1, + let text = children[0] as? Text else { + return nil + } + return mathTokens.first { + $0.kind == .display && $0.placeholder == text.string + } + } + private func plainText(from markup: Markup) -> String { switch markup { case let text as Text: @@ -718,18 +879,24 @@ private struct MarkdownWordBuilder { } private enum FrontmatterExtractor { - static func extract(from source: String) -> (metadata: [String: String], body: String) { + struct Result { + let metadata: [String: String] + let body: String + let bodyStartLine: Int + } + + static func extract(from source: String) -> Result { let normalized = source .replacingOccurrences(of: "\r\n", with: "\n") .replacingOccurrences(of: "\r", with: "\n") guard normalized.hasPrefix("---\n") else { - return ([:], normalized) + return Result(metadata: [:], body: normalized, bodyStartLine: 1) } let remainder = String(normalized.dropFirst(4)) guard let closingRange = remainder.range(of: "\n---\n") else { - return ([:], normalized) + return Result(metadata: [:], body: normalized, bodyStartLine: 1) } let rawMetadata = String(remainder[..", 0, ["$x$"]), + ("[ref]: https://example.com/$x$\n\n[link][ref]", 0, ["https://example.com/$x$"]), + ("Unmatched $x", 0, ["$x"]), + ("$ spaced $", 0, ["$ spaced $"]), + ("Price $5.00", 0, ["$5.00"]), + ] + + for testCase in cases { + let result = try MarkdownMathScanner().scan(testCase.source) + XCTAssertEqual(result.tokens.count, testCase.count, "Source: \(testCase.source)") + for text in testCase.retained { + XCTAssertTrue( + result.markdown.contains(text), + "Expected retained text \(text) in \(result.markdown)" + ) + } + } + } + + func testInlineTokenRecordsBodyKindAndOneBasedLocation() throws { + let result = try MarkdownMathScanner().scan("First\n\nA $x^2$ B") + + XCTAssertEqual(result.tokens.count, 1) + XCTAssertEqual(result.tokens[0].latex, "x^2") + XCTAssertEqual(result.tokens[0].kind, .inline) + XCTAssertEqual(result.tokens[0].line, 3) + XCTAssertEqual(result.tokens[0].column, 3) + XCTAssertEqual(result.markdown, "First\n\nA \(result.tokens[0].placeholder) B") + } + + func testCRLFCountsAsOneLogicalLine() throws { + let result = try MarkdownMathScanner().scan("First\r\nA $x$ B") + + XCTAssertEqual(result.tokens.count, 1) + XCTAssertEqual(result.tokens[0].line, 2) + XCTAssertEqual(result.tokens[0].column, 3) + } + + func testOneLineDisplayFormulaIsRecognized() throws { + let result = try MarkdownMathScanner().scan(#" $$\frac{a}{b}$$ "#) + + XCTAssertEqual(result.tokens.count, 1) + XCTAssertEqual(result.tokens[0].kind, .display) + XCTAssertEqual(result.tokens[0].latex, #"\frac{a}{b}"#) + XCTAssertEqual(result.tokens[0].line, 1) + XCTAssertEqual(result.tokens[0].column, 3) + } + + func testMultilineDisplayFormulaIsRecognized() throws { + let source = #""" + $$ + \frac{a}{b} + $$ + """# + let result = try MarkdownMathScanner().scan(source) + + XCTAssertEqual(result.tokens.count, 1) + XCTAssertEqual(result.tokens[0].kind, .display) + XCTAssertEqual(result.tokens[0].latex, #"\frac{a}{b}"#) + XCTAssertEqual(result.tokens[0].line, 1) + XCTAssertEqual(result.tokens[0].column, 1) + XCTAssertEqual(result.markdown, result.tokens[0].placeholder) + } + + func testMultilineDisplayPreservesFollowingParagraphBoundary() throws { + let source = "Before\n\n$$\nx\n$$\n\nAfter" + let result = try MarkdownMathScanner().scan(source) + + XCTAssertEqual(result.tokens.count, 1) + XCTAssertEqual( + result.markdown, + "Before\n\n\(result.tokens[0].placeholder)\n\nAfter" + ) + } + + func testFencedCodeRemainsNonMath() throws { + let source = """ + ```swift + let inline = "$x$" + let display = "$$y$$" + ``` + """ + let result = try MarkdownMathScanner().scan(source) + + XCTAssertTrue(result.tokens.isEmpty) + XCTAssertEqual(result.markdown, source) + } + + func testDisplayDelimiterMixedWithTextIsRejected() { + XCTAssertThrowsError(try MarkdownMathScanner().scan("before $$x$$ after")) { error in + XCTAssertEqual( + error as? MarkdownMathScanner.ScanError, + .misplacedDisplayFormula(line: 1, column: 8) + ) + } + } + + func testPlaceholderNeverOverwritesCallerText() throws { + let scanner = MarkdownMathScanner( + markerPrefix: "MDTOWORDMATHPLACEHOLDER", + markerNonce: "FIXED" + ) + let source = "Keep MDTOWORDMATHPLACEHOLDERFIXED0TOKEN and convert $x$" + let result = try scanner.scan(source) + + XCTAssertEqual(result.tokens.count, 1) + XCTAssertNotEqual(result.tokens[0].placeholder, "MDTOWORDMATHPLACEHOLDERFIXED0TOKEN") + XCTAssertTrue(result.markdown.contains("MDTOWORDMATHPLACEHOLDERFIXED0TOKEN")) + XCTAssertTrue(result.markdown.contains(result.tokens[0].placeholder)) + } +} +#endif diff --git a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift new file mode 100644 index 00000000..72041834 --- /dev/null +++ b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift @@ -0,0 +1,288 @@ +#if canImport(XCTest) +import Foundation +import XCTest +import MDToWord +import CommonConverterSwift + +final class MarkdownOMathConversionTests: XCTestCase { + func testPublicClientCanConstructOMathConverter() { + let converter = MarkdownToWordConverter(mathMode: .omath) + _ = converter + } + + func testDefaultConverterPreservesDollarDelimitedText() throws { + let xml = try documentXML( + markdown: "Before $x^2$ after", + converter: MarkdownToWordConverter() + ) + + XCTAssertTrue(xml.contains("Before $x^2$ after"), "Got: \(xml)") + XCTAssertFalse(xml.contains("", in: xml), 1, "Got: \(xml)") + XCTAssertEqual(count("", in: xml), 1, "Got: \(xml)") + XCTAssertTrue(xml.contains(""), "Got: \(xml)") + XCTAssertTrue(xml.contains(""), "Got: \(xml)") + XCTAssertFalse(xml.contains(#"$\frac{a}{b}$"#), "Got: \(xml)") + } + + func testInlineOMathIsDirectParagraphChildInSourceOrder() throws { + let xml = try documentXML( + markdown: "Before $x^2$ after", + converter: MarkdownToWordConverter(mathMode: .omath) + ) + + let before = try XCTUnwrap(xml.range(of: "Before ")?.lowerBound) + let math = try XCTUnwrap(xml.range(of: "")?.lowerBound) + let after = try XCTUnwrap(xml.range(of: " after")?.lowerBound) + XCTAssertLessThan(before, math) + XCTAssertLessThan(math, after) + XCTAssertEqual(count("", in: xml), 1, "Got: \(xml)") + XCTAssertFalse(xml.contains(""), "Got: \(xml)") + XCTAssertFalse(xml.contains("$x^2$"), "Got: \(xml)") + } + + func testDecodedCallerTextCannotCollideWithMathPlaceholder() throws { + let xml = try documentXML( + markdown: "Keep MDTOWORDMATHPLACEHOLDER0TOKEN and $x$", + converter: MarkdownToWordConverter(mathMode: .omath) + ) + + XCTAssertTrue(xml.contains("MDTOWORDMATHPLACEHOLDER0TOKEN"), "Got: \(xml)") + XCTAssertEqual(count("", in: xml), 1, "Got: \(xml)") + } + + func testInlineMathInLinkLabelPreservesMathAndDestination() throws { + let markdown = "See [$x$](https://example.com/math)" + let converter = MarkdownToWordConverter(mathMode: .omath) + let xml = try documentXML(markdown: markdown, converter: converter) + let document = try converter.convertMarkdown(markdown) + + XCTAssertEqual(count("", in: xml), 1, "Got: \(xml)") + XCTAssertFalse(xml.contains("MDTOWORDMATHPLACEHOLDER"), "Got: \(xml)") + XCTAssertTrue(xml.contains("x"), "Got: \(xml)") + XCTAssertEqual(document.hyperlinkReferences.first?.url, "https://example.com/math") + } + + func testInlineMathInImageAltTextDoesNotLeakPlaceholder() throws { + let xml = try documentXML( + markdown: "![Plot $x$](missing.png)", + converter: MarkdownToWordConverter(mathMode: .omath) + ) + + XCTAssertEqual(count("", in: xml), 1, "Got: \(xml)") + XCTAssertFalse(xml.contains("MDTOWORDMATHPLACEHOLDER"), "Got: \(xml)") + XCTAssertTrue(xml.contains("Plot "), "Got: \(xml)") + } + + func testDisplayFormulaUsesOMathParaWithoutSyntheticTextRun() throws { + let xml = try documentXML( + markdown: #"$$\frac{a}{b}$$"#, + converter: MarkdownToWordConverter(mathMode: .omath) + ) + + XCTAssertEqual(count("", in: xml), 1, "Got: \(xml)") + XCTAssertEqual(count("", in: xml), 1, "Got: \(xml)") + XCTAssertTrue(xml.contains(""), "Got: \(xml)") + XCTAssertFalse(xml.contains(#"$$\frac{a}{b}$$"#), "Got: \(xml)") + + let mathParagraphStart = try XCTUnwrap(xml.range(of: "")?.lowerBound) + let paragraphEnd = try XCTUnwrap(xml.range(of: "", range: mathParagraphStart..", in: output.content), 1) + XCTAssertNoThrow( + try XMLDocument( + data: Data(output.content.utf8), + options: [.nodePreserveAll] + ) + ) + } + + func testUnsupportedFormulaIsNormalizedWithSourceLocation() { + XCTAssertThrowsError( + try MarkdownToWordConverter(mathMode: .omath) + .convertMarkdown("First\n\n $\\overbrace{x}$") + ) { error in + XCTAssertEqual( + error as? MarkdownMathConversionError, + .unsupportedFormula(token: #"\overbrace"#, line: 3, column: 3) + ) + } + } + + func testFormulaLocationIncludesFrontmatterLines() { + let markdown = """ + --- + title: Math + --- + $\\overbrace{x}$ + """ + + XCTAssertThrowsError( + try MarkdownToWordConverter(mathMode: .omath).convertMarkdown(markdown) + ) { error in + XCTAssertEqual( + error as? MarkdownMathConversionError, + .unsupportedFormula(token: #"\overbrace"#, line: 4, column: 1) + ) + } + } + + func testMalformedFormulaIsNormalizedWithSourceLocation() { + XCTAssertThrowsError( + try MarkdownToWordConverter(mathMode: .omath) + .convertMarkdown(#"before $\frac{a}{b$"#) + ) { error in + XCTAssertEqual( + error as? MarkdownMathConversionError, + .malformedFormula(line: 1, column: 8) + ) + } + } + + func testMisplacedDisplayFormulaIsNormalizedWithSourceLocation() { + XCTAssertThrowsError( + try MarkdownToWordConverter(mathMode: .omath) + .convertMarkdown("before $$x$$ after") + ) { error in + XCTAssertEqual( + error as? MarkdownMathConversionError, + .misplacedDisplayFormula(line: 1, column: 8) + ) + } + } + + func testUnsupportedFormulaLeavesAbsentDestinationAbsent() throws { + let directory = try makeWorkspace(prefix: "markdown-omath-absent-output") + defer { try? FileManager.default.removeItem(at: directory) } + let input = directory.appendingPathComponent("fixture.md") + let output = directory.appendingPathComponent("fixture.docx") + try #"$\overbrace{x}$"#.write(to: input, atomically: true, encoding: .utf8) + + XCTAssertThrowsError( + try MarkdownToWordConverter(mathMode: .omath) + .convertToFile(input: input, output: output) + ) { error in + XCTAssertEqual( + error as? MarkdownMathConversionError, + .unsupportedFormula(token: #"\overbrace"#, line: 1, column: 1) + ) + } + XCTAssertFalse(FileManager.default.fileExists(atPath: output.path)) + } + + func testMalformedFormulaPreservesExistingDestination() throws { + let directory = try makeWorkspace(prefix: "markdown-omath-existing-output") + defer { try? FileManager.default.removeItem(at: directory) } + let input = directory.appendingPathComponent("fixture.md") + let output = directory.appendingPathComponent("fixture.docx") + let sentinel = Data("KEEP".utf8) + try #"$\frac{a}{b$"#.write(to: input, atomically: true, encoding: .utf8) + try sentinel.write(to: output) + + XCTAssertThrowsError( + try MarkdownToWordConverter(mathMode: .omath) + .convertToFile(input: input, output: output) + ) { error in + XCTAssertEqual( + error as? MarkdownMathConversionError, + .malformedFormula(line: 1, column: 1) + ) + } + XCTAssertEqual(try Data(contentsOf: output), sentinel) + } + + private func documentXML( + markdown: String, + converter: MarkdownToWordConverter + ) throws -> String { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("markdown-omath-mode-tests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let input = directory.appendingPathComponent("fixture.md") + let output = directory.appendingPathComponent("fixture.docx") + try markdown.write(to: input, atomically: true, encoding: .utf8) + try converter.convertToFile(input: input, output: output) + + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/unzip") + process.arguments = ["-p", output.path, "word/document.xml"] + let stdout = Pipe() + let stderr = Pipe() + process.standardOutput = stdout + process.standardError = stderr + try process.run() + process.waitUntilExit() + + let data = stdout.fileHandleForReading.readDataToEndOfFile() + let errorData = stderr.fileHandleForReading.readDataToEndOfFile() + guard process.terminationStatus == 0 else { + XCTFail("Failed to read document.xml: \(String(decoding: errorData, as: UTF8.self))") + return "" + } + return String(decoding: data, as: UTF8.self) + } + + private func makeWorkspace(prefix: String) throws -> URL { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("\(prefix)-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + return directory + } + + private func count(_ needle: String, in haystack: String) -> Int { + haystack.components(separatedBy: needle).count - 1 + } +} +#endif From 3365faf08db6ee400d791e684d9ef181461edea8 Mon Sep 17 00:00:00 2001 From: che cheng Date: Fri, 14 Aug 2026 02:07:17 +0800 Subject: [PATCH 2/7] refactor: harden Markdown math boundaries (#141) --- .../MarkdownOMathRouteTests.swift | 28 ++ .../markdown-omath-conversion/tasks.md | 3 +- .../MDToWord/MarkdownMathScanner.swift | 245 ++++++++++++++++-- .../MDToWord/MarkdownToWordConverter.swift | 44 +++- .../MarkdownMathScannerTests.swift | 45 ++++ .../MarkdownOMathConversionTests.swift | 68 +++++ 6 files changed, 402 insertions(+), 31 deletions(-) diff --git a/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift b/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift index d5f32cde..79d9f622 100644 --- a/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift +++ b/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift @@ -61,6 +61,12 @@ struct MarkdownOMathRouteTests { let xml = try archiveEntry(named: "word/document.xml", in: output) #expect(xml.components(separatedBy: "").count - 1 == 1) #expect(!xml.contains("$x^2$")) + #expect( + xml.contains( + "xmlns:m=\"http://schemas.openxmlformats.org/officeDocument/2006/math\"" + ) + ) + _ = try XMLDocument(data: Data(xml.utf8), options: [.nodePreserveAll]) } @Test("omath mode emits display native Word math") @@ -147,6 +153,28 @@ struct MarkdownOMathRouteTests { #expect(try Data(contentsOf: output) == sentinel) } + @Test("display math sharing a logical paragraph fails before replacing destination") + func mixedLogicalParagraphDisplayPreservesDestination() throws { + let workspace = try makeWorkspace() + defer { try? FileManager.default.removeItem(at: workspace) } + let input = workspace.appendingPathComponent("fixture.md") + let output = workspace.appendingPathComponent("fixture.docx") + let sentinel = Data("KEEP".utf8) + try "Before\n$$x$$\nAfter".write(to: input, atomically: true, encoding: .utf8) + try sentinel.write(to: output) + + let result = try CLITestHelper.convert( + to: "docx", + input: input.path, + flags: ["--math", "omath", "--output", output.path] + ) + + #expect(!result.succeeded) + #expect(result.stdout.isEmpty) + #expect(result.stderr.contains("line 2, column 1")) + #expect(try Data(contentsOf: output) == sentinel) + } + private func makeWorkspace() throws -> URL { let directory = FileManager.default.temporaryDirectory .appendingPathComponent("macdoc-markdown-omath-cli-\(UUID().uuidString)", isDirectory: true) diff --git a/openspec/changes/markdown-omath-conversion/tasks.md b/openspec/changes/markdown-omath-conversion/tasks.md index 36dc1fd2..f442b468 100644 --- a/openspec/changes/markdown-omath-conversion/tasks.md +++ b/openspec/changes/markdown-omath-conversion/tasks.md @@ -30,4 +30,5 @@ ## 7. Contract and regression verification -- [x] 7.1 Verify the design **Behavior**, **Interface and data shape**, **Failure modes**, **Acceptance criteria**, and **Scope boundaries** end to end: run focused scanner/converter/CLI tests, full `packages/md-to-word-swift` and root `swift test`, a non-`@testable` client type-check, dependency inspection, `git diff --check`, `spectra analyze markdown-omath-conversion --json`, and `spectra validate markdown-omath-conversion`; every new/focused test passes, full runs introduce zero failures beyond the documented clean baseline, and the analyzer has no Critical or Warning findings. +- [x] 7.1 Verify the design **Behavior**, **Interface and data shape**, **Failure modes**, **Acceptance criteria**, and **Scope boundaries** end to end: run focused scanner/converter/CLI tests, full `packages/md-to-word-swift` and root `swift test`, a non-`@testable` client build, dependency inspection, the repository's CRLF-aware `git -c core.whitespace=cr-at-eol diff --check`, `spectra analyze markdown-omath-conversion --json`, and `spectra validate markdown-omath-conversion`; every new/focused test passes, full runs introduce zero failures beyond the documented clean baseline, and the analyzer has no Critical or Warning findings. Plain `git diff --check` is not the gate for the pre-existing CRLF-formatted `Sources/MacDocCLI/MacDoc+Convert.swift`, because Git otherwise reports every newly added CRLF line as trailing whitespace. +- [x] 7.2 Close the first verification round's blockers with RED-then-GREEN regressions: archived OMath now declares `xmlns:m`; display math sharing a CommonMark logical paragraph fails before replacing the destination; multiline/container reference destinations and container/indented code remain literal without generated placeholders; dense 400-formula conversion is linear enough for the release regression budget. Final evidence: focused package 37/37, compiled CLI 8/8, root 53 total with 0 failures and 3 environment skips, package 121 total with only the documented 42 parent-baseline failures, release dense conversion 0.013 seconds, external non-`@testable` client build PASS, Spectra 0 Critical/Warning. diff --git a/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift index 77c36fb4..2482a5ce 100644 --- a/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift +++ b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift @@ -1,6 +1,8 @@ import Foundation struct MarkdownMathScanner { + static let defaultMarkerPrefix = "MDTOWORDMATHPLACEHOLDER" + enum TokenKind: Equatable { case inline case display @@ -23,11 +25,16 @@ struct MarkdownMathScanner { case misplacedDisplayFormula(line: Int, column: Int) } + private enum ReferenceContinuation: Equatable { + case destination + case optionalTitle + } + private let markerPrefix: String private let markerNonce: String init( - markerPrefix: String = "MDTOWORDMATHPLACEHOLDER", + markerPrefix: String = Self.defaultMarkerPrefix, markerNonce: String = UUID().uuidString.replacingOccurrences(of: "-", with: "") ) { self.markerPrefix = markerPrefix @@ -44,6 +51,7 @@ struct MarkdownMathScanner { var column = 1 var lineStart = 0 var fence: (character: Character, length: Int)? + var referenceContinuation: ReferenceContinuation? func marker() -> String { var candidate: String @@ -57,11 +65,16 @@ struct MarkdownMathScanner { while index < characters.count { if index == lineStart { let end = Self.lineEnd(in: characters, from: index) + let contentEnd = Self.contentEndBeforeNewline(in: characters, lineEnd: end) + let context = Self.lineContext( + characters, + start: index, + end: contentEnd + ) if let activeFence = fence { - let contentEnd = Self.contentEndBeforeNewline(in: characters, lineEnd: end) if Self.isFenceClosingLine( characters, - start: index, + start: context.contentStart, end: contentEnd, fence: activeFence ) { @@ -80,10 +93,23 @@ struct MarkdownMathScanner { continue } - let contentEnd = Self.contentEndBeforeNewline(in: characters, lineEnd: end) + if context.isIndentedCode { + output += String(characters[index.. 0 { + referenceContinuation = .optionalTitle + output += String(characters[index.. LineContext { + var index = start + var indentation = 0 + + func consumeIndentation() { + indentation = 0 + while index < end { + if characters[index] == " " { + indentation += 1 + index += 1 + } else if characters[index] == "\t" { + indentation += 4 + index += 1 + } else { + break + } + } + } + + consumeIndentation() + if indentation >= 4 { + return LineContext( + contentStart: index, + indentation: indentation, + isIndentedCode: true, + isBlank: index == end + ) + } + + while index < end { + if characters[index] == ">" { + index += 1 + if index < end, characters[index] == " " || characters[index] == "\t" { + index += 1 + } + consumeIndentation() + if indentation >= 4 { + return LineContext( + contentStart: index, + indentation: indentation, + isIndentedCode: true, + isBlank: index == end + ) + } + continue + } + + if let markerEnd = listMarkerEnd( + in: characters, + from: index, + before: end + ) { + index = markerEnd + consumeIndentation() + continue + } + break + } + + return LineContext( + contentStart: index, + indentation: indentation, + isIndentedCode: false, + isBlank: characters[index.. Int? { + guard start < end else { return nil } + if characters[start] == "-" || characters[start] == "+" || characters[start] == "*" { + let after = start + 1 + guard after < end, characters[after].isWhitespace else { return nil } + return after + } + + var index = start + var digits = 0 + while index < end, characters[index].isNumber, digits < 9 { + index += 1 + digits += 1 + } + guard digits > 0, + index < end, + characters[index] == "." || characters[index] == ")" else { + return nil + } + let after = index + 1 + guard after < end, characters[after].isWhitespace else { return nil } + return after + } + + private static func adjacentPreviousLineHasContent( + in characters: [Character], + before lineStart: Int + ) -> Bool { + guard lineStart > 0 else { return false } + var end = lineStart + if characters[end - 1].isNewline { + end -= 1 + } + var start = end + while start > 0, !characters[start - 1].isNewline { + start -= 1 + } + return lineHasLogicalContent(characters, start: start, end: end) + } + + private static func adjacentNextLineHasContent( + in characters: [Character], + after consumedEnd: Int + ) -> Bool { + var start = consumedEnd + if start > 0, !characters[start - 1].isNewline { + start = lineEnd(in: characters, from: start) + } + guard start < characters.count else { return false } + let end = contentEndBeforeNewline( + in: characters, + lineEnd: lineEnd(in: characters, from: start) + ) + return lineHasLogicalContent(characters, start: start, end: end) + } + + private static func lineHasLogicalContent( _ characters: [Character], start: Int, end: Int ) -> Bool { + let context = lineContext(characters, start: start, end: end) + return !context.isBlank + } + + private static func referenceDefinitionNeedsDestination( + _ characters: [Character], + start: Int, + end: Int + ) -> Bool? { var index = start var indentation = 0 while index < end, characters[index] == " ", indentation < 4 { @@ -446,7 +658,7 @@ struct MarkdownMathScanner { indentation += 1 } guard indentation <= 3, index < end, characters[index] == "[" else { - return false + return nil } index += 1 @@ -457,13 +669,14 @@ struct MarkdownMathScanner { continue } if characters[index] == "]" { - guard index > labelStart else { return false } + guard index > labelStart else { return nil } let colon = index + 1 - return colon < end && characters[colon] == ":" + guard colon < end, characters[colon] == ":" else { return nil } + return characters[(colon + 1).. String { @@ -212,6 +217,7 @@ private struct MarkdownWordBuilder { private let sourceName: String? private let frontmatter: [String: String] private let mathTokens: [RenderedMarkdownMathToken] + private let mathTokensByPlaceholder: [String: RenderedMarkdownMathToken] private var inferredTitle = false init( @@ -226,6 +232,9 @@ private struct MarkdownWordBuilder { self.sourceName = sourceName self.frontmatter = frontmatter self.mathTokens = mathTokens + self.mathTokensByPlaceholder = Dictionary( + uniqueKeysWithValues: mathTokens.map { ($0.placeholder, $0) } + ) } mutating func build(markdown: String) throws -> WordDocument { @@ -758,19 +767,25 @@ private struct MarkdownWordBuilder { in text: String, from cursor: String.Index ) -> (range: Range, token: RenderedMarkdownMathToken)? { - var match: (range: Range, token: RenderedMarkdownMathToken)? - for token in mathTokens where token.kind == .inline { - guard let range = text.range( - of: token.placeholder, - range: cursor.. RenderedMarkdownMathToken? { @@ -778,9 +793,10 @@ private struct MarkdownWordBuilder { let text = children[0] as? Text else { return nil } - return mathTokens.first { - $0.kind == .display && $0.placeholder == text.string + guard let token = mathTokensByPlaceholder[text.string], token.kind == .display else { + return nil } + return token } private func plainText(from markup: Markup) -> String { diff --git a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift index 1e2c10af..087f8e89 100644 --- a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift +++ b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift @@ -120,5 +120,50 @@ final class MarkdownMathScannerTests: XCTestCase { XCTAssertTrue(result.markdown.contains("MDTOWORDMATHPLACEHOLDERFIXED0TOKEN")) XCTAssertTrue(result.markdown.contains(result.tokens[0].placeholder)) } + + func testDisplayFormulaCannotShareALogicalParagraphWithText() { + let sources = [ + "Before\n$$x$$\nAfter", + "Before\n$$\nx\n$$\nAfter", + ] + + for source in sources { + XCTAssertThrowsError(try MarkdownMathScanner().scan(source)) { error in + XCTAssertEqual( + error as? MarkdownMathScanner.ScanError, + .misplacedDisplayFormula(line: 2, column: 1), + "Source: \(source)" + ) + } + } + } + + func testReferenceDefinitionDestinationsRemainLiteralInContainersAndContinuations() throws { + let sources = [ + "[ref]:\n https://example.com/$x$\n\n[link][ref]", + "> [ref]: https://example.com/$x$\n>\n> [link][ref]", + "> [ref]:\n> https://example.com/$x$\n>\n> [link][ref]", + ] + + for source in sources { + let result = try MarkdownMathScanner().scan(source) + XCTAssertTrue(result.tokens.isEmpty, "Source: \(source)") + XCTAssertEqual(result.markdown, source) + } + } + + func testContainerFencesAndIndentedCodeRemainLiteral() throws { + let sources = [ + "> ~~~text\n> $x$\n> ~~~", + "- ~~~text\n $x$\n ~~~", + " $x$", + ] + + for source in sources { + let result = try MarkdownMathScanner().scan(source) + XCTAssertTrue(result.tokens.isEmpty, "Source: \(source)") + XCTAssertEqual(result.markdown, source) + } + } } #endif diff --git a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift index 72041834..12b50349 100644 --- a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift +++ b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift @@ -147,6 +147,74 @@ final class MarkdownOMathConversionTests: XCTestCase { ) } + func testArchivedOMathDeclaresMathNamespaceAndParses() throws { + let xml = try documentXML( + markdown: "Before $x$ after", + converter: MarkdownToWordConverter(mathMode: .omath) + ) + + XCTAssertTrue( + xml.contains( + "xmlns:m=\"http://schemas.openxmlformats.org/officeDocument/2006/math\"" + ), + "Got: \(xml)" + ) + XCTAssertNoThrow( + try XMLDocument( + data: Data(xml.utf8), + options: [.nodePreserveAll] + ) + ) + } + + func testReferenceDefinitionContinuationPreservesRelationshipTarget() throws { + let markdown = "[link][ref]\n\n[ref]:\n https://example.com/$x$" + let document = try MarkdownToWordConverter(mathMode: .omath) + .convertMarkdown(markdown) + + XCTAssertEqual( + document.hyperlinkReferences.first?.url, + "https://example.com/$x$" + ) + } + + func testCodeBlocksNeverLeakGeneratedPlaceholders() throws { + let sources = [ + "> ~~~text\n> $x$\n> ~~~", + "- ~~~text\n $x$\n ~~~", + " $x$", + ] + + for source in sources { + let xml = try documentXML( + markdown: source, + converter: MarkdownToWordConverter(mathMode: .omath) + ) + XCTAssertTrue(xml.contains("$x$"), "Source: \(source); XML: \(xml)") + XCTAssertFalse( + xml.contains("MDTOWORDMATHPLACEHOLDER"), + "Source: \(source); XML: \(xml)" + ) + XCTAssertFalse(xml.contains("", in: xml), 400) + XCTAssertLessThan( + elapsed, + 5, + "Dense inline conversion took \(elapsed) seconds" + ) + } + func testUnsupportedFormulaIsNormalizedWithSourceLocation() { XCTAssertThrowsError( try MarkdownToWordConverter(mathMode: .omath) From 68c7403b39f3fad5d5da551f02fcc5c9f51bcb8a Mon Sep 17 00:00:00 2001 From: che cheng Date: Fri, 14 Aug 2026 02:39:49 +0800 Subject: [PATCH 3/7] refactor: align Markdown math with CommonMark (#141) --- .../MarkdownOMathRouteTests.swift | 51 +++ .../markdown-omath-conversion/tasks.md | 1 + .../MDToWord/MarkdownMathScanner.swift | 322 +++++++++++++----- .../MDToWord/MarkdownToWordConverter.swift | 36 +- .../MarkdownMathScannerTests.swift | 59 +++- .../MarkdownOMathConversionTests.swift | 65 ++++ 6 files changed, 438 insertions(+), 96 deletions(-) diff --git a/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift b/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift index 79d9f622..be251b5a 100644 --- a/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift +++ b/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift @@ -175,6 +175,57 @@ struct MarkdownOMathRouteTests { #expect(try Data(contentsOf: output) == sentinel) } + @Test("multiline link destination remains byte-exact in relationship") + func multilineLinkDestinationIsNotMath() throws { + let workspace = try makeWorkspace() + defer { try? FileManager.default.removeItem(at: workspace) } + let input = workspace.appendingPathComponent("fixture.md") + let output = workspace.appendingPathComponent("fixture.docx") + try "[link](\nhttps://example.com/$x$\n)".write( + to: input, + atomically: true, + encoding: .utf8 + ) + + let result = try CLITestHelper.convert( + to: "docx", + input: input.path, + flags: ["--math", "omath", "--output", output.path] + ) + + #expect(result.succeeded, "stderr: \(result.stderr)") + let relationships = try archiveEntry( + named: "word/_rels/document.xml.rels", + in: output + ) + #expect(relationships.contains("Target=\"https://example.com/$x$\"")) + #expect(!relationships.contains("MDTOWORDMATHPLACEHOLDER")) + } + + @Test("standalone display next to other CommonMark blocks succeeds") + func displayUsesParsedBlockBoundaries() throws { + let workspace = try makeWorkspace() + defer { try? FileManager.default.removeItem(at: workspace) } + let input = workspace.appendingPathComponent("fixture.md") + let output = workspace.appendingPathComponent("fixture.docx") + try "# Heading\n$$x$$\n\nAfter".write( + to: input, + atomically: true, + encoding: .utf8 + ) + + let result = try CLITestHelper.convert( + to: "docx", + input: input.path, + flags: ["--math", "omath", "--output", output.path] + ) + + #expect(result.succeeded, "stderr: \(result.stderr)") + let xml = try archiveEntry(named: "word/document.xml", in: output) + #expect(xml.components(separatedBy: "").count - 1 == 1) + #expect(!xml.contains("MDTOWORDMATHPLACEHOLDER")) + } + private func makeWorkspace() throws -> URL { let directory = FileManager.default.temporaryDirectory .appendingPathComponent("macdoc-markdown-omath-cli-\(UUID().uuidString)", isDirectory: true) diff --git a/openspec/changes/markdown-omath-conversion/tasks.md b/openspec/changes/markdown-omath-conversion/tasks.md index f442b468..ea8f29d1 100644 --- a/openspec/changes/markdown-omath-conversion/tasks.md +++ b/openspec/changes/markdown-omath-conversion/tasks.md @@ -32,3 +32,4 @@ - [x] 7.1 Verify the design **Behavior**, **Interface and data shape**, **Failure modes**, **Acceptance criteria**, and **Scope boundaries** end to end: run focused scanner/converter/CLI tests, full `packages/md-to-word-swift` and root `swift test`, a non-`@testable` client build, dependency inspection, the repository's CRLF-aware `git -c core.whitespace=cr-at-eol diff --check`, `spectra analyze markdown-omath-conversion --json`, and `spectra validate markdown-omath-conversion`; every new/focused test passes, full runs introduce zero failures beyond the documented clean baseline, and the analyzer has no Critical or Warning findings. Plain `git diff --check` is not the gate for the pre-existing CRLF-formatted `Sources/MacDocCLI/MacDoc+Convert.swift`, because Git otherwise reports every newly added CRLF line as trailing whitespace. - [x] 7.2 Close the first verification round's blockers with RED-then-GREEN regressions: archived OMath now declares `xmlns:m`; display math sharing a CommonMark logical paragraph fails before replacing the destination; multiline/container reference destinations and container/indented code remain literal without generated placeholders; dense 400-formula conversion is linear enough for the release regression budget. Final evidence: focused package 37/37, compiled CLI 8/8, root 53 total with 0 failures and 3 environment skips, package 121 total with only the documented 42 parent-baseline failures, release dense conversion 0.013 seconds, external non-`@testable` client build PASS, Spectra 0 Critical/Warning. +- [x] 7.3 Close the second verification round's blockers with RED-then-GREEN regressions: multiline inline-link and image destinations plus reference-definition titles remain literal; display placement is decided from the parsed CommonMark paragraph tree rather than adjacent physical lines; marker collision avoidance pre-indexes caller text once instead of rescanning it per token. Final evidence: focused package 38/38, compiled CLI 10/10, root 55 total with 0 failures and 3 environment skips, package 127 total with only the documented 42 parent-baseline failures, release 10,000-marker and 400-formula regressions both 0.004 seconds, external non-`@testable` type-check PASS, dependency graph resolves one OOXMLSwift 1.5.0 identity, both configured and ordinary diff checks PASS, and Spectra reports 0 Critical/Warning with 1 Suggestion. diff --git a/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift index 2482a5ce..ddf15da7 100644 --- a/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift +++ b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift @@ -28,6 +28,11 @@ struct MarkdownMathScanner { private enum ReferenceContinuation: Equatable { case destination case optionalTitle + case title(closing: Character) + } + + private struct ReferenceDefinitionMatch { + let continuation: ReferenceContinuation? } private let markerPrefix: String @@ -43,6 +48,11 @@ struct MarkdownMathScanner { func scan(_ source: String) throws -> Result { let characters = Array(source) + let markerStem = markerPrefix + markerNonce + let reservedMarkerIndices = Self.reservedMarkerIndices( + in: characters, + markerStem: Array(markerStem) + ) var output = "" var tokens: [Token] = [] var nextMarkerIndex = 0 @@ -50,16 +60,16 @@ struct MarkdownMathScanner { var line = 1 var column = 1 var lineStart = 0 + var logicalLineStart = 0 var fence: (character: Character, length: Int)? var referenceContinuation: ReferenceContinuation? func marker() -> String { - var candidate: String - repeat { - candidate = "\(markerPrefix)\(markerNonce)\(nextMarkerIndex)TOKEN" + while reservedMarkerIndices.contains(nextMarkerIndex) { nextMarkerIndex += 1 - } while source.contains(candidate) || tokens.contains(where: { $0.placeholder == candidate }) - return candidate + } + defer { nextMarkerIndex += 1 } + return "\(markerStem)\(nextMarkerIndex)TOKEN" } while index < characters.count { @@ -71,6 +81,7 @@ struct MarkdownMathScanner { start: index, end: contentEnd ) + logicalLineStart = context.contentStart if let activeFence = fence { if Self.isFenceClosingLine( characters, @@ -129,9 +140,12 @@ struct MarkdownMathScanner { if let activeReferenceContinuation = referenceContinuation { if context.isBlank { referenceContinuation = nil - } else if activeReferenceContinuation == .destination - || context.indentation > 0 { - referenceContinuation = .optionalTitle + } else if activeReferenceContinuation == .destination { + referenceContinuation = Self.continuationAfterDestination( + characters, + start: context.contentStart, + end: contentEnd + ) output += String(characters[index.. Bool { - guard lineStart > 0 else { return false } - var end = lineStart - if characters[end - 1].isNewline { - end -= 1 - } - var start = end - while start > 0, !characters[start - 1].isNewline { - start -= 1 - } - return lineHasLogicalContent(characters, start: start, end: end) - } - - private static func adjacentNextLineHasContent( - in characters: [Character], - after consumedEnd: Int - ) -> Bool { - var start = consumedEnd - if start > 0, !characters[start - 1].isNewline { - start = lineEnd(in: characters, from: start) - } - guard start < characters.count else { return false } - let end = contentEndBeforeNewline( - in: characters, - lineEnd: lineEnd(in: characters, from: start) - ) - return lineHasLogicalContent(characters, start: start, end: end) - } - - private static func lineHasLogicalContent( - _ characters: [Character], - start: Int, - end: Int - ) -> Bool { - let context = lineContext(characters, start: start, end: end) - return !context.isBlank - } - - private static func referenceDefinitionNeedsDestination( + private static func referenceDefinition( _ characters: [Character], start: Int, end: Int - ) -> Bool? { + ) -> ReferenceDefinitionMatch? { var index = start var indentation = 0 while index < end, characters[index] == " ", indentation < 4 { @@ -672,13 +668,119 @@ struct MarkdownMathScanner { guard index > labelStart else { return nil } let colon = index + 1 guard colon < end, characters[colon] == ":" else { return nil } - return characters[(colon + 1).. ReferenceContinuation? { + var index = start + while index < end, characters[index].isWhitespace { + index += 1 + } + guard index < end else { return .destination } + + if characters[index] == "<" { + index += 1 + while index < end { + if characters[index] == "\\" { + index = min(index + 2, end) + continue + } + if characters[index] == ">" { + index += 1 + break + } + index += 1 + } + } else { + while index < end, !characters[index].isWhitespace { + index += 1 + } + } + + while index < end, characters[index].isWhitespace { + index += 1 + } + guard index < end else { return .optionalTitle } + return titleContinuation(characters, opening: index, end: end) + } + + private static func optionalTitleLine( + _ characters: [Character], + start: Int, + end: Int + ) -> ReferenceDefinitionMatch? { + var index = start + while index < end, characters[index].isWhitespace { + index += 1 + } + guard index < end else { return nil } + let character = characters[index] + guard character == "\"" || character == "'" || character == "(" else { + return nil + } + return ReferenceDefinitionMatch( + continuation: titleContinuation(characters, opening: index, end: end) + ) + } + + private static func titleContinuation( + _ characters: [Character], + opening: Int, + end: Int + ) -> ReferenceContinuation? { + let closing: Character + switch characters[opening] { + case "\"": closing = "\"" + case "'": closing = "'" + case "(": closing = ")" + default: return nil + } + return containsUnescaped( + closing, + in: characters, + start: opening + 1, + end: end + ) ? nil : .title(closing: closing) + } + + private static func containsUnescaped( + _ character: Character, + in characters: [Character], + start: Int, + end: Int + ) -> Bool { + var index = start + while index < end { + if characters[index] == "\\" { + index = min(index + 2, end) + continue + } + if characters[index] == character { + return true + } + index += 1 + } + return false + } + private static func isFenceClosingLine( _ characters: [Character], start: Int, @@ -740,12 +842,21 @@ struct MarkdownMathScanner { ) -> Int? { var depth = 0 var index = openingParenthesis + var quote: Character? while index < characters.count { if characters[index] == "\\" { index = min(index + 2, characters.count) continue } - if characters[index] == "(" { + if let activeQuote = quote { + if characters[index] == activeQuote { + quote = nil + } + } else if (characters[index] == "\"" || characters[index] == "'"), + index > openingParenthesis, + characters[index - 1].isWhitespace { + quote = characters[index] + } else if characters[index] == "(" { depth += 1 } else if characters[index] == ")" { depth -= 1 @@ -753,13 +864,50 @@ struct MarkdownMathScanner { return index + 1 } } else if characters[index].isNewline { - return nil + var next = index + 1 + while next < characters.count, + characters[next].isWhitespace, + !characters[next].isNewline { + next += 1 + } + if next < characters.count, characters[next].isNewline { + return nil + } } index += 1 } return nil } + private static func reservedMarkerIndices( + in characters: [Character], + markerStem: [Character] + ) -> Set { + guard !markerStem.isEmpty else { return [] } + let suffix = Array("TOKEN") + var reserved: Set = [] + var index = 0 + + while index + markerStem.count < characters.count { + let stemEnd = index + markerStem.count + if characters[index.. stemEnd, + suffixEnd <= characters.count, + characters[digitsEnd.. = [] + + func collect(from markup: Markup) { + if let paragraph = markup as? Markdown.Paragraph { + let children = Array(paragraph.children) + if children.count == 1, + let text = children[0] as? Text, + let token = mathTokensByPlaceholder[text.string], + token.kind == .display { + standaloneDisplayPlaceholders.insert(token.placeholder) + } + } + for child in markup.children { + collect(from: child) + } + } + + collect(from: parsed) + for token in mathTokens where token.kind == .display { + guard standaloneDisplayPlaceholders.contains(token.placeholder) else { + throw MarkdownMathConversionError.misplacedDisplayFormula( + line: token.line, + column: token.column + ) + } + } + } + private mutating func applyDocumentMetadata() { document.properties.creator = frontmatter["author"] ?? frontmatter["creator"] ?? "macdoc" document.properties.subject = frontmatter["subject"] diff --git a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift index 087f8e89..fcbb66d2 100644 --- a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift +++ b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift @@ -121,20 +121,23 @@ final class MarkdownMathScannerTests: XCTestCase { XCTAssertTrue(result.markdown.contains(result.tokens[0].placeholder)) } - func testDisplayFormulaCannotShareALogicalParagraphWithText() { + func testScannerDefersLogicalParagraphValidationToCommonMarkTree() throws { let sources = [ "Before\n$$x$$\nAfter", "Before\n$$\nx\n$$\nAfter", + "# Heading\n$$x$$\n\nAfter", + "Before\n\n$$x$$\n# After", + "- Before\n- $$x$$\n- After", + "- $$\n x\n $$", + "> $$\n> x\n> $$", ] for source in sources { - XCTAssertThrowsError(try MarkdownMathScanner().scan(source)) { error in - XCTAssertEqual( - error as? MarkdownMathScanner.ScanError, - .misplacedDisplayFormula(line: 2, column: 1), - "Source: \(source)" - ) - } + let result = try MarkdownMathScanner().scan(source) + XCTAssertEqual(result.tokens.count, 1, "Source: \(source)") + guard result.tokens.count == 1 else { continue } + XCTAssertEqual(result.tokens[0].kind, .display, "Source: \(source)") + XCTAssertEqual(result.tokens[0].latex, "x", "Source: \(source)") } } @@ -165,5 +168,45 @@ final class MarkdownMathScannerTests: XCTestCase { XCTAssertEqual(result.markdown, source) } } + + func testMultilineInlineLinkAndImageDestinationsRemainLiteral() throws { + let sources = [ + "[link](\nhttps://example.com/$x$\n)", + "[link]( /uri\n \"title $x$\" )", + "[link](https://example.com/it's/$x$)", + "![alt](\nimages/$x$.png\n)", + ] + + for source in sources { + let result = try MarkdownMathScanner().scan(source) + XCTAssertTrue(result.tokens.isEmpty, "Source: \(source)") + XCTAssertEqual(result.markdown, source) + } + } + + func testReferenceDefinitionOptionalTitlesRemainLiteral() throws { + let sources = [ + "[ref]: https://example.com\n\"$x$\"", + "[ref]: https://example.com\n'$x$'", + "[ref]: https://example.com\n($x$)", + "[ref]: https://example.com '\n$title$\n'", + ] + + for source in sources { + let result = try MarkdownMathScanner().scan(source) + XCTAssertTrue(result.tokens.isEmpty, "Source: \(source)") + XCTAssertEqual(result.markdown, source) + } + } + + func testDenseMarkerAllocationDoesNotRescanSourcePerToken() throws { + let source = Array(repeating: "$x$", count: 10_000).joined(separator: " ") + let started = Date() + let result = try MarkdownMathScanner(markerNonce: "PERF").scan(source) + let elapsed = Date().timeIntervalSince(started) + + XCTAssertEqual(result.tokens.count, 10_000) + XCTAssertLessThan(elapsed, 1.5, "Dense marker allocation took \(elapsed) seconds") + } } #endif diff --git a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift index 12b50349..faab2fd9 100644 --- a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift +++ b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift @@ -178,6 +178,71 @@ final class MarkdownOMathConversionTests: XCTestCase { ) } + func testMultilineInlineLinkDestinationPreservesRelationshipTarget() throws { + let markdown = "[link](\nhttps://example.com/$x$\n)" + let converter = MarkdownToWordConverter(mathMode: .omath) + let document = try converter.convertMarkdown(markdown) + let xml = try documentXML(markdown: markdown, converter: converter) + + XCTAssertEqual( + document.hyperlinkReferences.first?.url, + "https://example.com/$x$" + ) + XCTAssertFalse(xml.contains("MDTOWORDMATHPLACEHOLDER"), "Got: \(xml)") + XCTAssertFalse(xml.contains(" $$\n> x\n> $$", + ] + for source in accepted { + let xml = try documentXML( + markdown: source, + converter: MarkdownToWordConverter(mathMode: .omath) + ) + XCTAssertEqual(count("", in: xml), 1, "Source: \(source)") + XCTAssertFalse(xml.contains("MDTOWORDMATHPLACEHOLDER"), "Source: \(source)") + } + } + func testCodeBlocksNeverLeakGeneratedPlaceholders() throws { let sources = [ "> ~~~text\n> $x$\n> ~~~", From 1da703a5bde4a2b7e0f1b0bbbb75542f1484c604 Mon Sep 17 00:00:00 2001 From: che cheng Date: Fri, 14 Aug 2026 03:32:08 +0800 Subject: [PATCH 4/7] spectra(markdown-omath-conversion): harden CommonMark math boundaries (#141) Change: markdown-omath-conversion Tasks: 19/19 complete --- .../MarkdownOMathRouteTests.swift | 95 +++ .../markdown-omath-conversion/design.md | 31 +- .../markdown-omath-conversion/proposal.md | 2 + .../specs/e2e-conversion-routes/spec.md | 13 + .../specs/markdown-omath-conversion/spec.md | 40 +- .../markdown-omath-conversion/tasks.md | 9 +- .../MarkdownMathConversionError.swift | 3 + .../MDToWord/MarkdownMathScanner.swift | 804 +++++++----------- .../MDToWord/MarkdownToWordConverter.swift | 26 +- .../MarkdownMathScannerTests.swift | 89 +- .../MarkdownOMathConversionTests.swift | 79 +- 11 files changed, 695 insertions(+), 496 deletions(-) diff --git a/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift b/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift index be251b5a..5c81c2f8 100644 --- a/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift +++ b/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift @@ -226,6 +226,101 @@ struct MarkdownOMathRouteTests { #expect(!xml.contains("MDTOWORDMATHPLACEHOLDER")) } + @Test("display math cannot cross original CommonMark blocks") + func crossBlockDisplayPreservesDestination() throws { + let sources = [ + "$$\n\nx\n\n$$", + "- $$\n- x\n- $$", + "> $$\nx\n> $$", + ] + + for source in sources { + let workspace = try makeWorkspace() + defer { try? FileManager.default.removeItem(at: workspace) } + let input = workspace.appendingPathComponent("fixture.md") + let output = workspace.appendingPathComponent("fixture.docx") + let sentinel = Data("KEEP".utf8) + try source.write(to: input, atomically: true, encoding: .utf8) + try sentinel.write(to: output) + + let result = try CLITestHelper.convert( + to: "docx", + input: input.path, + flags: ["--math", "omath", "--output", output.path] + ) + + #expect(!result.succeeded, "Source unexpectedly succeeded: \(source)") + #expect(result.stdout.isEmpty) + #expect(try Data(contentsOf: output) == sentinel) + } + } + + @Test("complex destinations remain byte-exact and placeholder-free") + func complexDestinationsRemainLiteral() throws { + let cases: [(source: String, target: String)] = [ + ( + "Use [ref].\n\n[\nref\n]: https://example.com/$x$", + "https://example.com/$x$" + ), + ( + "[link]()", + "https://example.com/a)$x$" + ), + ] + + for testCase in cases { + let workspace = try makeWorkspace() + defer { try? FileManager.default.removeItem(at: workspace) } + let input = workspace.appendingPathComponent("fixture.md") + let output = workspace.appendingPathComponent("fixture.docx") + try testCase.source.write(to: input, atomically: true, encoding: .utf8) + + let result = try CLITestHelper.convert( + to: "docx", + input: input.path, + flags: ["--math", "omath", "--output", output.path] + ) + + #expect(result.succeeded, "stderr: \(result.stderr)") + let relationships = try archiveEntry( + named: "word/_rels/document.xml.rels", + in: output + ) + let xml = try archiveEntry(named: "word/document.xml", in: output) + #expect(relationships.contains("Target=\"\(testCase.target)\"")) + #expect(!relationships.contains("MDTOWORDMATHPLACEHOLDER")) + #expect(!xml.contains("MDTOWORDMATHPLACEHOLDER")) + } + } + + @Test("HTML and formatting boundaries remain placeholder-free") + func htmlAndFormattingBoundariesRemainLiteral() throws { + let sources = [ + "\nVisible", + #" $x$\">text"#, + "$*x*$", + ] + + for source in sources { + let workspace = try makeWorkspace() + defer { try? FileManager.default.removeItem(at: workspace) } + let input = workspace.appendingPathComponent("fixture.md") + let output = workspace.appendingPathComponent("fixture.docx") + try source.write(to: input, atomically: true, encoding: .utf8) + + let result = try CLITestHelper.convert( + to: "docx", + input: input.path, + flags: ["--math", "omath", "--output", output.path] + ) + + #expect(result.succeeded, "Source: \(source); stderr: \(result.stderr)") + let xml = try archiveEntry(named: "word/document.xml", in: output) + #expect(!xml.contains("MDTOWORDMATHPLACEHOLDER")) + #expect(!xml.contains(" URL { let directory = FileManager.default.temporaryDirectory .appendingPathComponent("macdoc-markdown-omath-cli-\(UUID().uuidString)", isDirectory: true) diff --git a/openspec/changes/markdown-omath-conversion/design.md b/openspec/changes/markdown-omath-conversion/design.md index f2a0d74b..bf602a85 100644 --- a/openspec/changes/markdown-omath-conversion/design.md +++ b/openspec/changes/markdown-omath-conversion/design.md @@ -44,21 +44,24 @@ Alternatives rejected: - Adding math fields to `CommonConverterSwift.ConversionOptions` would impose an irrelevant public option on every converter. - Enabling OMath by default would change existing documents and turn previously harmless dollar text into parse errors. -### A conservative lexical tokenizer runs before the Markdown AST builder +### A conservative lexical tokenizer uses the original Markdown AST An internal `MarkdownMathScanner` will produce a transformed Markdown string plus a token table. It replaces recognized math spans with collision-resistant placeholders before `swift-markdown` parses the document. This preserves surrounding emphasis, headings, lists, quotes, and tables while preventing the Markdown parser from interpreting LaTeX punctuation. -The scanner is a single forward pass with explicit states for fenced code, equal-length backtick code spans, escaped characters, angle-bracket HTML/autolink regions, and Markdown link/image destinations. It recognizes: +The converter first parses the original source with `swift-markdown` and converts source ranges into character offsets using the parser's UTF-8 line/column semantics. Text-node ranges define where inline math is eligible. When cmark normalizes visible text and reports a child range that does not cover its raw spelling, the scanner recovers only exact dollar-span signatures present in that normalized `Text`, matched from the suffix of the same plain-text paragraph; it never makes the whole paragraph or a metadata prefix eligible. Paragraph ranges whose descendants contain only plain text and line-break nodes define where display math is eligible. Code, HTML, autolink, destination, reference metadata, and formatting-container ranges are therefore opaque without duplicating CommonMark grammar inside the scanner. + +The scanner then performs a bounded linear replacement pass over source characters. It recognizes: - inline math: one unescaped `$`, non-whitespace first and last formula characters, no newline, and a valid closing `$`; - display math: `$$...$$` occupying an entire logical paragraph, either on one trimmed line or with standalone opening and closing delimiter lines. -Unmatched dollars remain literal. A matched display token found alongside non-whitespace paragraph content is rejected rather than silently converted inline. A token placeholder is generated only after proving it does not already occur in the source. +Unmatched dollars remain literal. A matched display token found alongside non-whitespace paragraph content or spanning more than one original eligible paragraph is rejected rather than silently converted inline. A token placeholder is generated only after a linear pre-index proves its numeric suffix is absent from caller input. Alternatives rejected: - A regular expression cannot reliably exclude code, escapes, or link destinations. -- Scanning decoded `Text.string` values loses enough source spelling to distinguish escaped dollars. +- Scanning only decoded `Text.string` values loses enough source spelling to distinguish escaped dollars; source ranges retain the raw spelling while the AST supplies eligibility. +- Reimplementing reference definitions, HTML blocks, nested destinations, and container identity as scanner states drifts from CommonMark and permits placeholders to reach metadata. - Invoking Pandoc introduces a subprocess and a second conversion pipeline instead of using the maintained Swift parser. ### Tokens become direct paragraph children with carrier-specific wrappers @@ -69,6 +72,17 @@ Display tokens create a paragraph whose `unrecognizedChildren` contains one dire Surrounding Markdown run properties do not alter OMML nodes in v1. A formula remains structurally positioned inside the surrounding paragraph, but bold or italic Markdown wrappers do not rewrite `m:rPr`. +### Original CommonMark ranges gate every token and placeholder + +Inline delimiter pairs are accepted only when the entire raw span belongs to one original eligible `Text` node. Display delimiter pairs are accepted only when the entire raw span belongs to one original eligible `Paragraph` and the transformed paragraph contains no other visible content. Delimiters split by `Emphasis`, `Strong`, links, HTML, or another inline node remain literal. + +After building the transformed CommonMark tree, the converter records placeholder consumption. Success requires a bijection between scanned tokens and allowed output carriers: each inline token appears exactly once in visible text processing, and each display token appears exactly once as the sole content of one paragraph. A placeholder in a relationship target, HTML node, reference metadata, or unconsumed location is an integrity failure before the DOCX writer is invoked. + +Alternatives rejected: + +- Validating only the transformed tree cannot recover original block boundaries after a scanner has collapsed them. +- Treating missing tokens as harmless would allow hidden HTML, destinations, or malformed metadata to suppress formulas or leak internal placeholders. + ### Recognized formula failures are stable, located, and pre-write `MDToWord` will expose a public `MarkdownMathConversionError` that reports the one-based line and column of a recognized formula and a stable reason category: malformed or unsupported. Internal `LaTeXParseError` details are normalized into this type so callers do not depend on transitive parser implementation. @@ -87,7 +101,7 @@ The success path continues to write the normal one-line destination message to s - Existing `MarkdownToWordConverter()` and CLI calls without `--math` preserve literal dollar text. - `.omath` converts `$x^2$` to one inline `` and converts a standalone `$$\frac{a}{b}$$` block to one `` containing one ``. -- `\$5`, inline/fenced code, autolinks, link destinations, and unmatched dollars remain non-math content. +- `\$5`, inline/fenced code, HTML, autolinks, link/reference/image destinations, formatting-node-spanning delimiters, and unmatched dollars remain non-math content. - A recognized but unsupported expression, such as `$\overbrace{x}$`, fails the entire conversion without partial output. ### Interface and data shape @@ -103,11 +117,14 @@ The success path continues to write the normal one-line destination message to s - Matched unsupported or malformed formula: typed library error and non-zero CLI exit; source location is included; destination unchanged. - Unmatched delimiter: preserved as literal text, not treated as a parser failure. - Scanner placeholder collision: scanner selects another placeholder; caller input is never overwritten by a reserved literal. +- Original range or placeholder-consumption mismatch: conversion fails before writing and never emits a generated placeholder or modifies an existing destination. ### Acceptance criteria - Package tests inspect generated XML for exact inline/display wrapper counts and no surviving formula delimiters. -- Boundary tables cover escaped dollar, currency-like text, code span, fenced code, link destination, unmatched delimiter, and malformed supported-span cases. +- Boundary tables cover escaped dollar, currency-like text, code span, fenced code, HTML blocks/tags, inline/reference/image destinations, formatting-node spans, unmatched delimiters, and malformed supported-span cases. +- Negative display tests cover blank-separated paragraphs, different list items, and blockquote/container changes; all fail before destination replacement. +- Relationship assertions cover multiline reference labels and angle-bracket destinations containing `)` and prove generated placeholders never enter targets. - Public API type-checks from a client target without `@testable` imports. - Compiled CLI tests cover default literal, opt-in inline/display OMath, invalid route, invalid value, and existing-destination preservation on parser failure. - ~~`swift test` passes in `packages/md-to-word-swift` and at the repository root, and Spectra validation has no Critical or Warning findings.~~ @@ -119,7 +136,7 @@ In scope: Markdown-to-DOCX library and CLI behavior, parser dependency wiring, p ## Risks / Trade-offs -- [A custom scanner can drift from CommonMark edge cases] → Keep recognition conservative, use a single forward state machine, and lock exclusions with table-driven tests. +- [A custom scanner can drift from CommonMark edge cases] → Use original CommonMark source ranges as the eligibility authority, keep replacement to bounded linear passes, and lock exclusions with table-driven tests. - [Placeholder text could collide with user content] → Generate a per-conversion marker and verify absence before substitution. - [Parser and OOXML dependency versions can select incompatible `MathComponent` definitions] → Pin `latex-math-swift` from v0.2.0 and verify SwiftPM resolves one compatible `ooxml-swift` graph. - [Raw OMML could be emitted under an undeclared prefix] → Test both streaming XML and archived `word/document.xml` with XML parsing and the standard math namespace. diff --git a/openspec/changes/markdown-omath-conversion/proposal.md b/openspec/changes/markdown-omath-conversion/proposal.md index ad4265bd..3f6d415e 100644 --- a/openspec/changes/markdown-omath-conversion/proposal.md +++ b/openspec/changes/markdown-omath-conversion/proposal.md @@ -6,6 +6,8 @@ The Markdown-to-DOCX route currently serializes `$...$` and `$$...$$` as literal - Add a public `MDToWord` math mode whose default preserves the current literal behavior and whose opt-in OMath mode converts supported inline and display LaTeX delimiters to native Word OMML. - Add a delimiter scanner that excludes escaped dollar signs, inline code, fenced code, and formulas crossing Markdown formatting-node boundaries. +- Derive formula-eligible source ranges from the original CommonMark tree before substitution, so block/container boundaries, HTML regions, code, destinations, reference metadata, and formatting-node splits cannot be erased by placeholders. +- Require every generated placeholder to be consumed exactly once by an allowed visible text or display carrier before conversion can succeed. - Reuse `LaTeXMathParser` from `latex-math-swift`; unsupported or malformed formulas fail loudly before any destination DOCX is replaced. - Add `macdoc convert --to docx ... --math literal|omath`, with route validation that rejects the option on incompatible source/target pairs. - Expand package-level and compiled-CLI acceptance coverage to inspect `word/document.xml`, distinguish inline from display carriers, and prove failure leaves an existing destination unchanged. diff --git a/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md b/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md index 60a04426..4d0c67df 100644 --- a/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md +++ b/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md @@ -41,3 +41,16 @@ The compiled E2E suite SHALL exercise `macdoc convert` through the production co - **WHEN** the compiled binary runs a Markdown-to-DOCX conversion with `--math literal` - **THEN** exit code is 0 and delimiters remain literal Word text + +#### Scenario: Cross-block display failure preserves the compiled CLI destination + +- **GIVEN** an existing output file containing sentinel bytes `KEEP` +- **WHEN** the compiled binary receives display delimiters in different list items, blank-separated paragraphs, or different blockquote containers with `--math omath` +- **THEN** exit code is non-zero and stdout is empty +- **AND** the output file remains byte-identical to `KEEP` + +#### Scenario: Compiled route never writes placeholders into relationships or visible text + +- **WHEN** the compiled binary receives multiline reference labels, angle-bracket destinations containing `)`, HTML comments/blocks, quoted HTML attributes, or formatting-node-spanning delimiters +- **THEN** exit code and formula behavior follow the library boundary contract +- **AND** neither `word/document.xml` nor `word/_rels/document.xml.rels` contains `MDTOWORDMATHPLACEHOLDER` diff --git a/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md b/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md index a4535ea8..ef1e736a 100644 --- a/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md +++ b/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md @@ -24,7 +24,9 @@ The `MDToWord` library SHALL expose a public `MarkdownMathMode` enum with raw st In `.omath` mode, the converter SHALL recognize inline math only from a pair of unescaped single-dollar delimiters within non-code Markdown content. The first and last formula characters SHALL be non-whitespace, and an inline formula SHALL NOT contain a newline. The converter SHALL recognize display math only when a `$$...$$` pair occupies an entire logical paragraph, either on one trimmed line or between standalone opening and closing delimiter lines. Unmatched dollars SHALL remain literal. -The scanner SHALL exclude escaped dollar signs, equal-length backtick code spans, fenced code blocks, angle-bracket HTML or autolink regions, Markdown link destinations, and image destinations. It SHALL perform one forward scan over source characters. A placeholder SHALL be selected only after confirming that the placeholder does not occur in caller input. +The converter SHALL derive formula-eligible ranges from the original CommonMark tree before replacing source text. Inline formulas SHALL fit within one eligible original `Text` range. Display formulas SHALL fit within one eligible original `Paragraph` range whose inline descendants are limited to text and line breaks. Code, HTML, autolink, link/reference/image destinations, reference metadata, and delimiters split across formatting nodes SHALL remain non-math content. Source-range indexing, collision indexing, and replacement SHALL use a bounded number of linear passes over source characters. A placeholder SHALL be selected only after confirming that its numeric suffix does not occur in caller input. + +Every generated placeholder SHALL be consumed exactly once by an allowed visible inline or display carrier. A placeholder that is absent, repeated, or present in HTML, metadata, a relationship target, or any other non-carrier location SHALL cause conversion to fail before destination replacement. #### Scenario: Inline recognition boundary table @@ -53,6 +55,42 @@ The scanner SHALL exclude escaped dollar signs, equal-length backtick code spans - **WHEN** `.omath` conversion receives a paragraph containing standalone `$$`, then `\\frac{a}{b}`, then standalone `$$` - **THEN** exactly one display formula is recognized with source body `\\frac{a}{b}` +#### Scenario: Display formula cannot cross original CommonMark boundaries + +- **WHEN** `.omath` conversion receives standalone `$$` delimiters separated by blank paragraphs, different list items, or a blockquote/container change +- **THEN** conversion throws `MarkdownMathConversionError.misplacedDisplayFormula` at the opening delimiter +- **AND** no destination is created or replaced + +#### Scenario: HTML and formatting-node spans remain non-math + +- **WHEN** `.omath` conversion receives dollar delimiters inside an HTML comment, HTML block, quoted HTML attribute, or split around an `Emphasis` or `Strong` node +- **THEN** no formula token is created from those delimiters +- **AND** no generated placeholder appears in visible document text + +#### Scenario: CommonMark destinations and reference metadata are byte-preserving + +- **WHEN** `.omath` conversion receives a multiline reference label, an angle-bracket destination containing `)`, a multiline inline or image destination, or a valid optional reference title containing dollar delimiters +- **THEN** every parsed relationship target remains byte-exact +- **AND** no generated placeholder appears in a target or reference metadata + +#### Scenario: Visible text after invalid reference-like syntax remains eligible + +- **WHEN** a line beginning with reference-like or title-like syntax is parsed by CommonMark as visible text containing `$x$` +- **THEN** `.omath` mode converts `$x$` as ordinary visible inline math + +#### Scenario: Every token has one allowed consumer + +- **WHEN** scanning produces formula tokens and the transformed CommonMark tree is built +- **THEN** each token is consumed exactly once by one visible inline text carrier or one standalone display paragraph +- **AND** any missing, duplicate, metadata, HTML, or relationship consumer causes a pre-write conversion failure + +##### Example: One inline and one display carrier + +- **GIVEN** source `Before $x$` followed by a separate `$$y$$` paragraph +- **WHEN** conversion succeeds in `.omath` mode +- **THEN** the inline token is consumed once by `` and the display token once by `` +- **AND** neither placeholder remains in document XML or relationships + #### Scenario: Display delimiter mixed with paragraph text is rejected - **WHEN** `.omath` conversion receives `before $$x$$ after` in one paragraph diff --git a/openspec/changes/markdown-omath-conversion/tasks.md b/openspec/changes/markdown-omath-conversion/tasks.md index ea8f29d1..de9925bf 100644 --- a/openspec/changes/markdown-omath-conversion/tasks.md +++ b/openspec/changes/markdown-omath-conversion/tasks.md @@ -32,4 +32,11 @@ - [x] 7.1 Verify the design **Behavior**, **Interface and data shape**, **Failure modes**, **Acceptance criteria**, and **Scope boundaries** end to end: run focused scanner/converter/CLI tests, full `packages/md-to-word-swift` and root `swift test`, a non-`@testable` client build, dependency inspection, the repository's CRLF-aware `git -c core.whitespace=cr-at-eol diff --check`, `spectra analyze markdown-omath-conversion --json`, and `spectra validate markdown-omath-conversion`; every new/focused test passes, full runs introduce zero failures beyond the documented clean baseline, and the analyzer has no Critical or Warning findings. Plain `git diff --check` is not the gate for the pre-existing CRLF-formatted `Sources/MacDocCLI/MacDoc+Convert.swift`, because Git otherwise reports every newly added CRLF line as trailing whitespace. - [x] 7.2 Close the first verification round's blockers with RED-then-GREEN regressions: archived OMath now declares `xmlns:m`; display math sharing a CommonMark logical paragraph fails before replacing the destination; multiline/container reference destinations and container/indented code remain literal without generated placeholders; dense 400-formula conversion is linear enough for the release regression budget. Final evidence: focused package 37/37, compiled CLI 8/8, root 53 total with 0 failures and 3 environment skips, package 121 total with only the documented 42 parent-baseline failures, release dense conversion 0.013 seconds, external non-`@testable` client build PASS, Spectra 0 Critical/Warning. -- [x] 7.3 Close the second verification round's blockers with RED-then-GREEN regressions: multiline inline-link and image destinations plus reference-definition titles remain literal; display placement is decided from the parsed CommonMark paragraph tree rather than adjacent physical lines; marker collision avoidance pre-indexes caller text once instead of rescanning it per token. Final evidence: focused package 38/38, compiled CLI 10/10, root 55 total with 0 failures and 3 environment skips, package 127 total with only the documented 42 parent-baseline failures, release 10,000-marker and 400-formula regressions both 0.004 seconds, external non-`@testable` type-check PASS, dependency graph resolves one OOXMLSwift 1.5.0 identity, both configured and ordinary diff checks PASS, and Spectra reports 0 Critical/Warning with 1 Suggestion. +- [x] 7.3 Close the second verification round's blockers with RED-then-GREEN regressions: multiline inline-link and image destinations plus reference-definition titles remain literal; display placement is decided from the parsed CommonMark paragraph tree rather than adjacent physical lines; marker collision avoidance pre-indexes caller text once instead of rescanning it per token. Final evidence: focused package 38/38, compiled CLI 10/10, root 55 total with 0 failures and 3 environment skips, package 127 total with only the documented 42 parent-baseline failures, release 10,000-marker and 400-formula regressions both 0.004 seconds, external non-`@testable` type-check PASS, dependency graph resolves one OOXMLSwift 1.5.0 identity, the task 7.1 CRLF-aware diff gate PASS, and Spectra reports 0 Critical/Warning with 1 Suggestion. Any earlier ordinary `git diff --check` evidence is informational and not the repository's authoritative CRLF gate. + +## 8. Original CommonMark range and carrier integrity + +- [x] 8.1 Add RED tests for **OMath mode recognizes conservative dollar delimiters** and **Original CommonMark ranges gate every token and placeholder**, covering blank-separated display paragraphs, three distinct list items, blockquote/container changes, multiline reference labels, angle-bracket destinations containing `)`, valid and invalid optional-title syntax, multiline HTML comments/blocks, quoted `>` HTML attributes, `$*x*$`/`$**x**$`, and missing or non-carrier placeholder consumption; verify package and compiled CLI tests fail against `68c7403b` for the documented reason while existing `KEEP` destinations remain byte-identical on errors. +- [x] 8.2 Implement **A conservative lexical tokenizer uses the original Markdown AST** and original-AST source-range eligibility plus exact token-consumption reconciliation so inline formulas fit one eligible `Text`, display formulas fit one eligible `Paragraph` and container, relationship/HTML/reference/code spans remain opaque, formatting-node-spanning delimiters remain literal, and every placeholder has exactly one allowed carrier. GREEN evidence: scanner 20/20 and scanner plus converter 47/47; release 10,000-marker allocation completed in 0.011 seconds and release 400-formula conversion in 0.005 seconds. +- [x] 8.3 Extend **Compiled Markdown OMath route coverage** with the R3 production counterexamples and prove compiled output never contains `MDTOWORDMATHPLACEHOLDER` in `word/document.xml` or relationships, all reject cases preserve `KEEP`, focused package/CLI/root tests pass, the full package adds no failure beyond #155's 42-test baseline, external API/dependency/Spectra checks pass, and the CRLF-aware diff gate passes. Final evidence: compiled CLI 13/13; root 58 total with 0 failures and 3 environment skips; package 136 total with exactly the documented 42 baseline failures; external non-`@testable` client build PASS; one OOXMLSwift 1.5.0 identity; CRLF-aware diff gate PASS; Spectra 0 Critical/Warning. +- [x] 8.4 Record the R3 evidence correction and final frozen proof: task 7.3's ordinary `git diff --check` claim is superseded because task 7.1 defines the CRLF-aware command as the repository gate; implementation now uses bounded O(N) source-location, eligibility, collision, and consumption indexing plus replacement rather than claiming a literal one-pass architecture. Final Spectra validation PASS with 0 Critical, 0 Warning, and 2 non-blocking Suggestions. diff --git a/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathConversionError.swift b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathConversionError.swift index 3fd34ca6..9bb9c35a 100644 --- a/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathConversionError.swift +++ b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathConversionError.swift @@ -6,6 +6,7 @@ public enum MarkdownMathConversionError: Error, Equatable, Sendable { case malformedFormula(line: Int, column: Int) case unsupportedFormula(token: String, line: Int, column: Int) case misplacedDisplayFormula(line: Int, column: Int) + case formulaPlacementMismatch(line: Int, column: Int) } extension MarkdownMathConversionError: LocalizedError { @@ -17,6 +18,8 @@ extension MarkdownMathConversionError: LocalizedError { return "Unsupported LaTeX token \(token) at line \(line), column \(column)." case .misplacedDisplayFormula(let line, let column): return "Display math must occupy its own paragraph at line \(line), column \(column)." + case .formulaPlacementMismatch(let line, let column): + return "Math formula could not be placed exactly once at line \(line), column \(column)." } } } diff --git a/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift index ddf15da7..555efe20 100644 --- a/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift +++ b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift @@ -1,4 +1,5 @@ import Foundation +import Markdown struct MarkdownMathScanner { static let defaultMarkerPrefix = "MDTOWORDMATHPLACEHOLDER" @@ -25,19 +26,259 @@ struct MarkdownMathScanner { case misplacedDisplayFormula(line: Int, column: Int) } - private enum ReferenceContinuation: Equatable { - case destination - case optionalTitle - case title(closing: Character) - } - - private struct ReferenceDefinitionMatch { - let continuation: ReferenceContinuation? - } - private let markerPrefix: String private let markerNonce: String + /// Original CommonMark source ranges are the authority for math eligibility. + /// A narrow raw-angle shield supplements ranges when cmark normalizes quoted tag spelling. + private struct SourceEligibility { + private struct Point: Hashable { + let line: Int + let column: Int + } + + private let inlineIneligiblePrefix: [Int] + private let visibleOffsets: [Bool] + private let paragraphRanges: Set> + private let displayRanges: Set> + + init(source: String, characters: [Character]) { + let offsets = Self.sourceOffsets(for: characters) + let document = Document(parsing: source) + var inlineRanges: [Range] = [] + var fallbackInlineRanges: [Range] = [] + var paragraphRanges: Set> = [] + var displayRanges: Set> = [] + + func offsetRange(for markup: Markup) -> Range? { + guard let range = markup.range, + let lower = offsets[ + Point(line: range.lowerBound.line, column: range.lowerBound.column) + ], + let upper = offsets[ + Point(line: range.upperBound.line, column: range.upperBound.column) + ], + lower <= upper else { + return nil + } + return lower..) -> Range { + var lower = range.lowerBound + var upper = range.upperBound + while lower < upper, characters[lower].isWhitespace { + lower += 1 + } + while lower < upper, characters[upper - 1].isWhitespace { + upper -= 1 + } + return lower.. Bool { + guard let range = offsetRange(for: link), + range.lowerBound < range.upperBound else { + return false + } + return characters[range.lowerBound] == "<" + && characters[range.upperBound - 1] == ">" + } + + func paragraphAllowsDisplay(_ paragraph: Markdown.Paragraph) -> Bool { + paragraph.children.allSatisfy { child in + child is Text || child is SoftBreak || child is LineBreak + } + } + + func recoverNormalizedTextRanges( + in paragraph: Markdown.Paragraph, + sourceRange: Range + ) { + guard paragraphAllowsDisplay(paragraph) else { return } + var needed: [String: Int] = [:] + for child in paragraph.children { + guard let text = child as? Text else { continue } + for candidate in Self.inlineCandidates(in: Array(text.string)) { + needed[candidate.signature, default: 0] += 1 + } + } + guard !needed.isEmpty else { return } + + let rawCandidates = Self.inlineCandidates( + in: characters, + range: sourceRange + ) + for candidate in rawCandidates.reversed() + where needed[candidate.signature, default: 0] > 0 { + fallbackInlineRanges.append(candidate.range) + needed[candidate.signature, default: 0] -= 1 + } + } + + func collect(_ markup: Markup, insideAutolink: Bool = false) { + if let paragraph = markup as? Markdown.Paragraph, + let range = offsetRange(for: paragraph) { + let contentRange = trimmingWhitespace(range) + paragraphRanges.insert(contentRange) + if paragraphAllowsDisplay(paragraph) { + displayRanges.insert(contentRange) + recoverNormalizedTextRanges(in: paragraph, sourceRange: range) + } + } + + let childrenInsideAutolink = insideAutolink + || ((markup as? Markdown.Link).map(isAutolink) ?? false) + if markup is Text, !insideAutolink, let range = offsetRange(for: markup) { + inlineRanges.append(range) + } + for child in markup.children { + collect(child, insideAutolink: childrenInsideAutolink) + } + } + + collect(document) + var visibleOffsets = [Bool](repeating: false, count: characters.count) + for range in inlineRanges + fallbackInlineRanges { + for offset in range where offset < visibleOffsets.count { + visibleOffsets[offset] = true + } + } + for range in Self.angleRanges(in: characters) { + for offset in range where offset < visibleOffsets.count { + visibleOffsets[offset] = false + } + } + var inlineIneligiblePrefix = [Int](repeating: 0, count: characters.count + 1) + for offset in characters.indices { + inlineIneligiblePrefix[offset + 1] = inlineIneligiblePrefix[offset] + + (visibleOffsets[offset] ? 0 : 1) + } + self.inlineIneligiblePrefix = inlineIneligiblePrefix + self.visibleOffsets = visibleOffsets + self.paragraphRanges = paragraphRanges + self.displayRanges = displayRanges + } + + func containsInline(_ range: Range) -> Bool { + guard !range.isEmpty, + 0 <= range.lowerBound, + range.upperBound < inlineIneligiblePrefix.count else { + return false + } + return inlineIneligiblePrefix[range.upperBound] + == inlineIneligiblePrefix[range.lowerBound] + } + + func containsVisibleText(at offset: Int) -> Bool { + visibleOffsets.indices.contains(offset) && visibleOffsets[offset] + } + + func isParagraph(_ range: Range) -> Bool { + paragraphRanges.contains(range) + } + + func allowsDisplay(_ range: Range) -> Bool { + displayRanges.contains(range) + } + + private static func sourceOffsets(for characters: [Character]) -> [Point: Int] { + var result: [Point: Int] = [:] + var line = 1 + var column = 1 + for (offset, character) in characters.enumerated() { + result[Point(line: line, column: column)] = offset + if character.isNewline { + line += 1 + column = 1 + } else { + column += String(character).utf8.count + } + } + result[Point(line: line, column: column)] = characters.count + return result + } + + private static func angleRanges(in characters: [Character]) -> [Range] { + var result: [Range] = [] + var index = 0 + while index < characters.count { + guard characters[index] == "<", index + 1 < characters.count else { + index += 1 + continue + } + let next = characters[index + 1] + guard next.isLetter || next == "/" || next == "!" || next == "?" else { + index += 1 + continue + } + + var cursor = index + 2 + var quote: Character? + while cursor < characters.count, !characters[cursor].isNewline { + let character = characters[cursor] + if let activeQuote = quote { + if character == activeQuote { + quote = nil + } + } else if character == "\"" || character == "'" { + quote = character + } else if character == ">" { + result.append(index..<(cursor + 1)) + index = cursor + break + } + cursor += 1 + } + index += 1 + } + return result + } + + private static func inlineCandidates( + in characters: [Character], + range: Range? = nil + ) -> [(range: Range, signature: String)] { + let bounds = range ?? characters.startIndex.., signature: String)] = [] + var index = bounds.lowerBound + while index < bounds.upperBound { + guard characters[index] == "$" else { + index += 1 + continue + } + var closing = index + 1 + while closing < bounds.upperBound, + !characters[closing].isNewline, + characters[closing] != "$" { + if characters[closing] == "\\" { + closing = min(closing + 2, bounds.upperBound) + } else { + closing += 1 + } + } + guard closing < bounds.upperBound, characters[closing] == "$" else { + index += 1 + continue + } + let bodyRange = (index + 1).. Result { let characters = Array(source) + let eligibility = SourceEligibility(source: source, characters: characters) let markerStem = markerPrefix + markerNonce let reservedMarkerIndices = Self.reservedMarkerIndices( in: characters, @@ -61,8 +303,7 @@ struct MarkdownMathScanner { var column = 1 var lineStart = 0 var logicalLineStart = 0 - var fence: (character: Character, length: Int)? - var referenceContinuation: ReferenceContinuation? + var logicalQuoteDepth = 0 func marker() -> String { while reservedMarkerIndices.contains(nextMarkerIndex) { @@ -82,88 +323,39 @@ struct MarkdownMathScanner { end: contentEnd ) logicalLineStart = context.contentStart - if let activeFence = fence { - if Self.isFenceClosingLine( - characters, - start: context.contentStart, - end: contentEnd, - fence: activeFence - ) { - fence = nil - } - output += String(characters[index.. DisplayMatch? { let openingLineEnd = contentEndBeforeNewline( in: characters, @@ -449,6 +554,9 @@ struct MarkdownMathScanner { start: search, end: candidateLineEnd ) + guard context.quoteDepth == openingQuoteDepth else { + return nil + } let contentStart = context.contentStart if contentStart + 2 <= candidateLineEnd, characters[contentStart] == "$", @@ -517,30 +625,27 @@ struct MarkdownMathScanner { return nil } - private static func fenceOpening( - _ characters: [Character], - start: Int, - end: Int - ) -> (character: Character, length: Int)? { + private static func laterDoubleDollarClosing( + in characters: [Character], + after start: Int + ) -> Int? { var index = start - var indentation = 0 - while index < end, characters[index] == " ", indentation < 4 { + while index + 1 < characters.count { + if characters[index] == "\\" { + index += 2 + continue + } + if characters[index] == "$", characters[index + 1] == "$" { + return index + } index += 1 - indentation += 1 } - guard indentation <= 3, index < end else { return nil } - let character = characters[index] - guard character == "`" || character == "~" else { return nil } - let length = runLength(of: character, in: characters, from: index) - guard length >= 3 else { return nil } - return (character, length) + return nil } private struct LineContext { let contentStart: Int - let indentation: Int - let isIndentedCode: Bool - let isBlank: Bool + let quoteDepth: Int } private static func lineContext( @@ -550,6 +655,7 @@ struct MarkdownMathScanner { ) -> LineContext { var index = start var indentation = 0 + var quoteDepth = 0 func consumeIndentation() { indentation = 0 @@ -570,14 +676,13 @@ struct MarkdownMathScanner { if indentation >= 4 { return LineContext( contentStart: index, - indentation: indentation, - isIndentedCode: true, - isBlank: index == end + quoteDepth: quoteDepth ) } while index < end { if characters[index] == ">" { + quoteDepth += 1 index += 1 if index < end, characters[index] == " " || characters[index] == "\t" { index += 1 @@ -586,9 +691,7 @@ struct MarkdownMathScanner { if indentation >= 4 { return LineContext( contentStart: index, - indentation: indentation, - isIndentedCode: true, - isBlank: index == end + quoteDepth: quoteDepth ) } continue @@ -608,9 +711,7 @@ struct MarkdownMathScanner { return LineContext( contentStart: index, - indentation: indentation, - isIndentedCode: false, - isBlank: characters[index.. ReferenceDefinitionMatch? { - var index = start - var indentation = 0 - while index < end, characters[index] == " ", indentation < 4 { - index += 1 - indentation += 1 - } - guard indentation <= 3, index < end, characters[index] == "[" else { - return nil - } - - index += 1 - let labelStart = index - while index < end { - if characters[index] == "\\" { - index = min(index + 2, end) - continue - } - if characters[index] == "]" { - guard index > labelStart else { return nil } - let colon = index + 1 - guard colon < end, characters[colon] == ":" else { return nil } - let contentStart = colon + 1 - if characters[contentStart.. ReferenceContinuation? { - var index = start - while index < end, characters[index].isWhitespace { - index += 1 - } - guard index < end else { return .destination } - - if characters[index] == "<" { - index += 1 - while index < end { - if characters[index] == "\\" { - index = min(index + 2, end) - continue - } - if characters[index] == ">" { - index += 1 - break - } - index += 1 - } - } else { - while index < end, !characters[index].isWhitespace { - index += 1 - } - } - - while index < end, characters[index].isWhitespace { - index += 1 - } - guard index < end else { return .optionalTitle } - return titleContinuation(characters, opening: index, end: end) - } - - private static func optionalTitleLine( - _ characters: [Character], - start: Int, - end: Int - ) -> ReferenceDefinitionMatch? { - var index = start - while index < end, characters[index].isWhitespace { - index += 1 - } - guard index < end else { return nil } - let character = characters[index] - guard character == "\"" || character == "'" || character == "(" else { - return nil - } - return ReferenceDefinitionMatch( - continuation: titleContinuation(characters, opening: index, end: end) - ) - } - - private static func titleContinuation( - _ characters: [Character], - opening: Int, - end: Int - ) -> ReferenceContinuation? { - let closing: Character - switch characters[opening] { - case "\"": closing = "\"" - case "'": closing = "'" - case "(": closing = ")" - default: return nil - } - return containsUnescaped( - closing, - in: characters, - start: opening + 1, - end: end - ) ? nil : .title(closing: closing) - } - - private static func containsUnescaped( - _ character: Character, - in characters: [Character], - start: Int, - end: Int - ) -> Bool { - var index = start - while index < end { - if characters[index] == "\\" { - index = min(index + 2, end) - continue - } - if characters[index] == character { - return true - } - index += 1 - } - return false - } - - private static func isFenceClosingLine( - _ characters: [Character], - start: Int, - end: Int, - fence: (character: Character, length: Int) - ) -> Bool { - var index = start - var indentation = 0 - while index < end, characters[index] == " ", indentation < 4 { - index += 1 - indentation += 1 - } - guard indentation <= 3, index < end else { return false } - let length = runLength(of: fence.character, in: characters, from: index) - guard length >= fence.length else { return false } - return characters[(index + length).. Int? { - var index = start - while index < characters.count { - if characters[index] == "`" { - let length = runLength(of: "`", in: characters, from: index) - if length == delimiterLength { - return index + length - } - index += length - continue - } - index += 1 - } - return nil - } - - private static func angleRegionEnd( - in characters: [Character], - from start: Int - ) -> Int? { - var index = start + 1 - while index < characters.count { - if characters[index] == ">" { - return index + 1 - } - if characters[index].isNewline { - return nil - } - index += 1 - } - return nil - } - - private static func linkDestinationEnd( - in characters: [Character], - from openingParenthesis: Int - ) -> Int? { - var depth = 0 - var index = openingParenthesis - var quote: Character? - while index < characters.count { - if characters[index] == "\\" { - index = min(index + 2, characters.count) - continue - } - if let activeQuote = quote { - if characters[index] == activeQuote { - quote = nil - } - } else if (characters[index] == "\"" || characters[index] == "'"), - index > openingParenthesis, - characters[index - 1].isWhitespace { - quote = characters[index] - } else if characters[index] == "(" { - depth += 1 - } else if characters[index] == ")" { - depth -= 1 - if depth == 0 { - return index + 1 - } - } else if characters[index].isNewline { - var next = index + 1 - while next < characters.count, - characters[next].isWhitespace, - !characters[next].isNewline { - next += 1 - } - if next < characters.count, characters[next].isNewline { - return nil - } - } - index += 1 - } - return nil - } - private static func reservedMarkerIndices( in characters: [Character], markerStem: [Character] @@ -908,18 +772,6 @@ struct MarkdownMathScanner { return reserved } - private static func runLength( - of character: Character, - in characters: [Character], - from start: Int - ) -> Int { - var end = start - while end < characters.count, characters[end] == character { - end += 1 - } - return end - start - } - private static func lineEnd(in characters: [Character], from start: Int) -> Int { var end = start while end < characters.count { diff --git a/packages/md-to-word-swift/Sources/MDToWord/MarkdownToWordConverter.swift b/packages/md-to-word-swift/Sources/MDToWord/MarkdownToWordConverter.swift index c1a53256..e1579e2b 100644 --- a/packages/md-to-word-swift/Sources/MDToWord/MarkdownToWordConverter.swift +++ b/packages/md-to-word-swift/Sources/MDToWord/MarkdownToWordConverter.swift @@ -222,6 +222,7 @@ private struct MarkdownWordBuilder { private let frontmatter: [String: String] private let mathTokens: [RenderedMarkdownMathToken] private let mathTokensByPlaceholder: [String: RenderedMarkdownMathToken] + private var consumedMathPlaceholders: [String: Int] = [:] private var inferredTitle = false init( @@ -249,6 +250,7 @@ private struct MarkdownWordBuilder { for child in parsed.children { try appendBlock(child, quoteDepth: 0) } + try validateMathTokenConsumption() if document.body.children.isEmpty { document.appendParagraph(WordParagraph(text: "")) @@ -257,6 +259,19 @@ private struct MarkdownWordBuilder { return document } + private func validateMathTokenConsumption() throws { + for token in mathTokens where consumedMathPlaceholders[token.placeholder, default: 0] != 1 { + throw MarkdownMathConversionError.formulaPlacementMismatch( + line: token.line, + column: token.column + ) + } + } + + private mutating func markMathTokenConsumed(_ token: RenderedMarkdownMathToken) { + consumedMathPlaceholders[token.placeholder, default: 0] += 1 + } + private func validateDisplayMathPlacement(in parsed: Document) throws { var standaloneDisplayPlaceholders: Set = [] @@ -488,6 +503,7 @@ private struct MarkdownWordBuilder { style: String? = nil ) throws -> WordParagraph? { if let token = displayToken(in: children) { + markMathTokenConsumed(token) var paragraph = WordParagraph(runs: []) paragraph.unrecognizedChildren = [ UnrecognizedChild( @@ -717,7 +733,7 @@ private struct MarkdownWordBuilder { return "rId\(baseID + usedCount)" } - private func makeExternalHyperlinkXML(text: String, relationshipId: String) -> String { + private mutating func makeExternalHyperlinkXML(text: String, relationshipId: String) -> String { """ \(hyperlinkContentXML(text)) @@ -725,7 +741,7 @@ private struct MarkdownWordBuilder { """ } - private func makeInternalHyperlinkXML(text: String, anchor: String) -> String { + private mutating func makeInternalHyperlinkXML(text: String, anchor: String) -> String { """ \(hyperlinkContentXML(text)) @@ -733,7 +749,7 @@ private struct MarkdownWordBuilder { """ } - private func hyperlinkContentXML(_ text: String) -> String { + private mutating func hyperlinkContentXML(_ text: String) -> String { var xml = "" var cursor = text.startIndex while cursor < text.endIndex { @@ -745,6 +761,7 @@ private struct MarkdownWordBuilder { xml += hyperlinkTextRunXML(String(text[cursor..\(match.token.omml)")) + markMathTokenConsumed(match.token) cursor = match.range.upperBound } } diff --git a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift index fcbb66d2..08da33b8 100644 --- a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift +++ b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift @@ -98,6 +98,14 @@ final class MarkdownMathScannerTests: XCTestCase { XCTAssertEqual(result.markdown, source) } + func testUnmatchedVisibleDisplayDoesNotPairWithOpaqueCodeDelimiter() throws { + let source = "Unmatched $$\n\n```text\n$$\n```" + let result = try MarkdownMathScanner().scan(source) + + XCTAssertTrue(result.tokens.isEmpty) + XCTAssertEqual(result.markdown, source) + } + func testDisplayDelimiterMixedWithTextIsRejected() { XCTAssertThrowsError(try MarkdownMathScanner().scan("before $$x$$ after")) { error in XCTAssertEqual( @@ -121,10 +129,26 @@ final class MarkdownMathScannerTests: XCTestCase { XCTAssertTrue(result.markdown.contains(result.tokens[0].placeholder)) } - func testScannerDefersLogicalParagraphValidationToCommonMarkTree() throws { - let sources = [ - "Before\n$$x$$\nAfter", - "Before\n$$\nx\n$$\nAfter", + func testOriginalCommonMarkParagraphGatesDisplayRecognition() throws { + let rejected: [(source: String, line: Int, column: Int)] = [ + ("Before\n$$x$$\nAfter", 2, 1), + ("Before\n$$\nx\n$$\nAfter", 2, 1), + ("$$\n\nx\n\n$$", 1, 1), + ("- $$\n- x\n- $$", 1, 3), + ("> $$\nx\n> $$", 1, 3), + ] + + for testCase in rejected { + XCTAssertThrowsError(try MarkdownMathScanner().scan(testCase.source)) { error in + XCTAssertEqual( + error as? MarkdownMathScanner.ScanError, + .misplacedDisplayFormula(line: testCase.line, column: testCase.column), + "Source: \(testCase.source)" + ) + } + } + + let accepted = [ "# Heading\n$$x$$\n\nAfter", "Before\n\n$$x$$\n# After", "- Before\n- $$x$$\n- After", @@ -132,7 +156,7 @@ final class MarkdownMathScannerTests: XCTestCase { "> $$\n> x\n> $$", ] - for source in sources { + for source in accepted { let result = try MarkdownMathScanner().scan(source) XCTAssertEqual(result.tokens.count, 1, "Source: \(source)") guard result.tokens.count == 1 else { continue } @@ -141,6 +165,61 @@ final class MarkdownMathScannerTests: XCTestCase { } } + func testOriginalCommonMarkOpaqueAndFormattingRangesRemainLiteral() throws { + let sources = [ + "", + "
\n$x$\n
", + #" $x$\">text"#, + "$*x*$", + "$**x**$", + ] + + for source in sources { + let result = try MarkdownMathScanner().scan(source) + XCTAssertTrue(result.tokens.isEmpty, "Source: \(source)") + XCTAssertEqual(result.markdown, source) + } + } + + func testCommonMarkDestinationAndReferenceMetadataRemainLiteral() throws { + let sources = [ + "Use [ref].\n\n[\nref\n]: https://example.com/$x$", + "[link]()", + "[ref]: https://example.com\n\"$x$\"", + ] + + for source in sources { + let result = try MarkdownMathScanner().scan(source) + XCTAssertTrue(result.tokens.isEmpty, "Source: \(source)") + XCTAssertEqual(result.markdown, source) + } + } + + func testVisibleInvalidReferenceLikeTextRemainsEligible() throws { + let sources = [ + "[ref]: invalid destination $x$", + "Use [ref].\n\n[ref]: https://example.com\n\"$x$\" ok", + ] + + for source in sources { + let result = try MarkdownMathScanner().scan(source) + XCTAssertEqual(result.tokens.count, 1, "Source: \(source)") + XCTAssertEqual(result.tokens.first?.latex, "x", "Source: \(source)") + } + } + + func testInvalidReferenceTitleRecoveryNeverConsumesMatchingDestinationFormula() throws { + let source = "Use [ref].\n\n[ref]: https://example.com/$x$\n\"$x$\" ok" + let result = try MarkdownMathScanner().scan(source) + + XCTAssertEqual(result.tokens.map(\.latex), ["x"]) + XCTAssertTrue(result.markdown.contains("https://example.com/$x$")) + XCTAssertEqual( + result.markdown.components(separatedBy: MarkdownMathScanner.defaultMarkerPrefix).count - 1, + 1 + ) + } + func testReferenceDefinitionDestinationsRemainLiteralInContainersAndContinuations() throws { let sources = [ "[ref]:\n https://example.com/$x$\n\n[link][ref]", diff --git a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift index faab2fd9..3e00b43c 100644 --- a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift +++ b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift @@ -213,14 +213,19 @@ final class MarkdownOMathConversionTests: XCTestCase { let rejected = [ "Before\n$$x$$\nAfter", "Before\n$$\nx\n$$\nAfter", + "$$\n\nx\n\n$$", + "- $$\n- x\n- $$", + "> $$\nx\n> $$", ] - for source in rejected { + for (index, source) in rejected.enumerated() { XCTAssertThrowsError( try MarkdownToWordConverter(mathMode: .omath).convertMarkdown(source) ) { error in + let expectedColumn = index >= 3 ? 3 : 1 + let expectedLine = index < 2 ? 2 : 1 XCTAssertEqual( error as? MarkdownMathConversionError, - .misplacedDisplayFormula(line: 2, column: 1), + .misplacedDisplayFormula(line: expectedLine, column: expectedColumn), "Source: \(source)" ) } @@ -243,6 +248,76 @@ final class MarkdownOMathConversionTests: XCTestCase { } } + func testComplexCommonMarkDestinationsNeverReceivePlaceholders() throws { + let cases: [(source: String, target: String)] = [ + ( + "Use [ref].\n\n[\nref\n]: https://example.com/$x$", + "https://example.com/$x$" + ), + ( + "[link]()", + "https://example.com/a)$x$" + ), + ] + + for testCase in cases { + let converter = MarkdownToWordConverter(mathMode: .omath) + let document = try converter.convertMarkdown(testCase.source) + let xml = try documentXML(markdown: testCase.source, converter: converter) + + XCTAssertEqual(document.hyperlinkReferences.first?.url, testCase.target) + XCTAssertFalse(xml.contains("MDTOWORDMATHPLACEHOLDER"), "Source: \(testCase.source)") + XCTAssertFalse(xml.contains("", in: xml), 1, "Source: \(source); XML: \(xml)") + XCTAssertFalse(xml.contains("MDTOWORDMATHPLACEHOLDER"), "Source: \(source)") + } + } + + func testInvalidReferenceTitleConvertsVisibleMathWithoutChangingMatchingTarget() throws { + let source = "Use [ref].\n\n[ref]: https://example.com/$x$\n\"$x$\" ok" + let converter = MarkdownToWordConverter(mathMode: .omath) + let document = try converter.convertMarkdown(source) + let xml = try documentXML(markdown: source, converter: converter) + + XCTAssertEqual(document.hyperlinkReferences.first?.url, "https://example.com/$x$") + XCTAssertEqual(count("", in: xml), 1) + XCTAssertFalse(xml.contains("MDTOWORDMATHPLACEHOLDER")) + } + + func testHTMLAndFormattingNodeBoundariesNeverCreateMathTokens() throws { + let sources = [ + "\nVisible", + "
\n$\\overbrace{x}$\n
\nVisible", + #" $x$\">text"#, + "$*x*$", + "$**x**$", + ] + + for source in sources { + let converter = MarkdownToWordConverter(mathMode: .omath) + let document = try converter.convertMarkdown(source) + let xml = try documentXML(markdown: source, converter: converter) + + XCTAssertNil(document.documentRootAttributes["xmlns:m"], "Source: \(source)") + XCTAssertFalse(xml.contains(" ~~~text\n> $x$\n> ~~~", From d10f850205da0fadb21408f394a960ec843c3c78 Mon Sep 17 00:00:00 2001 From: che cheng Date: Fri, 14 Aug 2026 03:57:39 +0800 Subject: [PATCH 5/7] spectra(markdown-omath-conversion): harden final CommonMark edge cases (#141) --- .../MarkdownOMathRouteTests.swift | 24 ++- .../markdown-omath-conversion/design.md | 4 +- .../specs/e2e-conversion-routes/spec.md | 6 + .../specs/markdown-omath-conversion/spec.md | 6 + .../markdown-omath-conversion/tasks.md | 5 + .../MDToWord/MarkdownMathScanner.swift | 166 +++++++++++++----- .../MDToWord/MarkdownToWordConverter.swift | 37 +++- .../MarkdownMathScannerTests.swift | 44 ++++- .../MarkdownOMathConversionTests.swift | 15 +- 9 files changed, 253 insertions(+), 54 deletions(-) diff --git a/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift b/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift index 5c81c2f8..e3ec233f 100644 --- a/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift +++ b/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift @@ -231,7 +231,7 @@ struct MarkdownOMathRouteTests { let sources = [ "$$\n\nx\n\n$$", "- $$\n- x\n- $$", - "> $$\nx\n> $$", + "> $$\nx\n$$", ] for source in sources { @@ -321,6 +321,28 @@ struct MarkdownOMathRouteTests { } } + @Test("invalid HTML-like visible text remains math eligible") + func invalidHTMLLikeVisibleTextEmitsMath() throws { + let workspace = try makeWorkspace() + defer { try? FileManager.default.removeItem(at: workspace) } + let input = workspace.appendingPathComponent("fixture.md") + let output = workspace.appendingPathComponent("fixture.docx") + try "Visible tail".write(to: input, atomically: true, encoding: .utf8) + + let result = try CLITestHelper.convert( + to: "docx", + input: input.path, + flags: ["--math", "omath", "--output", output.path] + ) + + #expect(result.succeeded, "stderr: \(result.stderr)") + let xml = try archiveEntry(named: "word/document.xml", in: output) + #expect(xml.components(separatedBy: "").count - 1 == 1) + #expect(xml.contains("<span ")) + #expect(xml.contains("> tail")) + #expect(!xml.contains("MDTOWORDMATHPLACEHOLDER")) + } + private func makeWorkspace() throws -> URL { let directory = FileManager.default.temporaryDirectory .appendingPathComponent("macdoc-markdown-omath-cli-\(UUID().uuidString)", isDirectory: true) diff --git a/openspec/changes/markdown-omath-conversion/design.md b/openspec/changes/markdown-omath-conversion/design.md index bf602a85..db7936ce 100644 --- a/openspec/changes/markdown-omath-conversion/design.md +++ b/openspec/changes/markdown-omath-conversion/design.md @@ -48,14 +48,14 @@ Alternatives rejected: An internal `MarkdownMathScanner` will produce a transformed Markdown string plus a token table. It replaces recognized math spans with collision-resistant placeholders before `swift-markdown` parses the document. This preserves surrounding emphasis, headings, lists, quotes, and tables while preventing the Markdown parser from interpreting LaTeX punctuation. -The converter first parses the original source with `swift-markdown` and converts source ranges into character offsets using the parser's UTF-8 line/column semantics. Text-node ranges define where inline math is eligible. When cmark normalizes visible text and reports a child range that does not cover its raw spelling, the scanner recovers only exact dollar-span signatures present in that normalized `Text`, matched from the suffix of the same plain-text paragraph; it never makes the whole paragraph or a metadata prefix eligible. Paragraph ranges whose descendants contain only plain text and line-break nodes define where display math is eligible. Code, HTML, autolink, destination, reference metadata, and formatting-container ranges are therefore opaque without duplicating CommonMark grammar inside the scanner. +The converter first parses the original source with `swift-markdown` and converts source ranges into character offsets using the parser's UTF-8 line/column semantics. Text-node ranges define where inline math is eligible. When cmark normalizes visible text and reports a child range that does not cover its raw spelling, the scanner recovers only exact dollar-span signatures present in that normalized `Text`, matched from the suffix of the same plain-text paragraph; it never makes the whole paragraph or a metadata prefix eligible. A bounded raw pairing pass may mask an opening HTML spelling only when an original `InlineHTML` closing node anchors the same tag and the recovered dollar text lies inside a quoted attribute. Unpaired invalid HTML-like text remains governed by its original `Text` range and stays math-eligible. Paragraph ranges whose descendants contain only plain text and line-break nodes define where display math is eligible. Code, HTML, autolink, destination, reference metadata, and formatting-container ranges are therefore opaque without making a free-standing raw angle heuristic override the CommonMark tree. The scanner then performs a bounded linear replacement pass over source characters. It recognizes: - inline math: one unescaped `$`, non-whitespace first and last formula characters, no newline, and a valid closing `$`; - display math: `$$...$$` occupying an entire logical paragraph, either on one trimmed line or with standalone opening and closing delimiter lines. -Unmatched dollars remain literal. A matched display token found alongside non-whitespace paragraph content or spanning more than one original eligible paragraph is rejected rather than silently converted inline. A token placeholder is generated only after a linear pre-index proves its numeric suffix is absent from caller input. +Unmatched dollars remain literal. A matched display token found alongside non-whitespace paragraph content or spanning more than one original eligible paragraph is rejected rather than silently converted inline. A token placeholder is generated only after a linear pre-index proves its numeric suffix is absent from caller input. The placeholder is wrapped in punctuation that cannot turn invalid HTML-like visible text into an HTML attribute when the transformed source is parsed again. Alternatives rejected: diff --git a/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md b/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md index 4d0c67df..2e6723a1 100644 --- a/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md +++ b/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md @@ -54,3 +54,9 @@ The compiled E2E suite SHALL exercise `macdoc convert` through the production co - **WHEN** the compiled binary receives multiline reference labels, angle-bracket destinations containing `)`, HTML comments/blocks, quoted HTML attributes, or formatting-node-spanning delimiters - **THEN** exit code and formula behavior follow the library boundary contract - **AND** neither `word/document.xml` nor `word/_rels/document.xml.rels` contains `MDTOWORDMATHPLACEHOLDER` + +#### Scenario: Compiled route converts math in invalid HTML-like visible text + +- **WHEN** the compiled binary receives `Visible tail` with `--math omath` +- **THEN** the DOCX contains one inline OMath carrier and preserves the visible angle-bracket text +- **AND** neither document XML nor relationships contain a generated placeholder diff --git a/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md b/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md index ef1e736a..4a874b61 100644 --- a/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md +++ b/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md @@ -73,6 +73,12 @@ Every generated placeholder SHALL be consumed exactly once by an allowed visible - **THEN** every parsed relationship target remains byte-exact - **AND** no generated placeholder appears in a target or reference metadata +#### Scenario: Invalid HTML-like visible text remains math-eligible + +- **WHEN** CommonMark parses `Visible tail` as one visible `Text` node rather than HTML +- **THEN** `.omath` mode converts `$x$` to one inline OMath carrier +- **AND** the visible angle-bracket text remains present without a generated placeholder + #### Scenario: Visible text after invalid reference-like syntax remains eligible - **WHEN** a line beginning with reference-like or title-like syntax is parsed by CommonMark as visible text containing `$x$` diff --git a/openspec/changes/markdown-omath-conversion/tasks.md b/openspec/changes/markdown-omath-conversion/tasks.md index de9925bf..2bfe0bbb 100644 --- a/openspec/changes/markdown-omath-conversion/tasks.md +++ b/openspec/changes/markdown-omath-conversion/tasks.md @@ -40,3 +40,8 @@ - [x] 8.2 Implement **A conservative lexical tokenizer uses the original Markdown AST** and original-AST source-range eligibility plus exact token-consumption reconciliation so inline formulas fit one eligible `Text`, display formulas fit one eligible `Paragraph` and container, relationship/HTML/reference/code spans remain opaque, formatting-node-spanning delimiters remain literal, and every placeholder has exactly one allowed carrier. GREEN evidence: scanner 20/20 and scanner plus converter 47/47; release 10,000-marker allocation completed in 0.011 seconds and release 400-formula conversion in 0.005 seconds. - [x] 8.3 Extend **Compiled Markdown OMath route coverage** with the R3 production counterexamples and prove compiled output never contains `MDTOWORDMATHPLACEHOLDER` in `word/document.xml` or relationships, all reject cases preserve `KEEP`, focused package/CLI/root tests pass, the full package adds no failure beyond #155's 42-test baseline, external API/dependency/Spectra checks pass, and the CRLF-aware diff gate passes. Final evidence: compiled CLI 13/13; root 58 total with 0 failures and 3 environment skips; package 136 total with exactly the documented 42 baseline failures; external non-`@testable` client build PASS; one OOXMLSwift 1.5.0 identity; CRLF-aware diff gate PASS; Spectra 0 Critical/Warning. - [x] 8.4 Record the R3 evidence correction and final frozen proof: task 7.3's ordinary `git diff --check` claim is superseded because task 7.1 defines the CRLF-aware command as the repository gate; implementation now uses bounded O(N) source-location, eligibility, collision, and consumption indexing plus replacement rather than claiming a literal one-pass architecture. Final Spectra validation PASS with 0 Critical, 0 Warning, and 2 non-blocking Suggestions. + +## 9. R4 exact-counterexample closure + +- [x] 9.1 Add exact RED regressions for the original blockquote escape `> $$\nx\n$$`, invalid HTML-like visible text `Visible tail`, and zero/duplicate placeholder consumption; prove the frozen R4 candidate either overwrites `KEEP` or misses visible math before the production correction. +- [x] 9.2 Make original paragraph identity reject the exact blockquote escape, replace the global raw-angle override with AST-anchored bounded HTML pairing, use punctuation-wrapped placeholders that cannot reclassify invalid tag-like text, and validate each token has exactly one consumer. GREEN evidence: focused scanner/converter 50/50, compiled CLI 14/14, root 59 total with 0 failures and 3 environment skips, package 139 total with exactly the documented 42 baseline failures, both diff gates PASS, and Spectra validation PASS. diff --git a/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift index 555efe20..e94380ef 100644 --- a/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift +++ b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift @@ -30,7 +30,6 @@ struct MarkdownMathScanner { private let markerNonce: String /// Original CommonMark source ranges are the authority for math eligibility. - /// A narrow raw-angle shield supplements ranges when cmark normalizes quoted tag spelling. private struct SourceEligibility { private struct Point: Hashable { let line: Int @@ -39,6 +38,7 @@ struct MarkdownMathScanner { private let inlineIneligiblePrefix: [Int] private let visibleOffsets: [Bool] + private let paragraphIdentifiers: [Int?] private let paragraphRanges: Set> private let displayRanges: Set> @@ -47,6 +47,8 @@ struct MarkdownMathScanner { let document = Document(parsing: source) var inlineRanges: [Range] = [] var fallbackInlineRanges: [Range] = [] + var opaqueHTMLRanges: [Range] = [] + var closingHTMLTags: [Int: String] = [:] var paragraphRanges: Set> = [] var displayRanges: Set> = [] @@ -116,6 +118,30 @@ struct MarkdownMathScanner { } } + func recoverInlineHTMLRange(_ inlineHTML: InlineHTML) -> Range? { + guard let range = offsetRange(for: inlineHTML), + !range.isEmpty, + !inlineHTML.rawHTML.isEmpty else { + return nil + } + let raw = Array(inlineHTML.rawHTML) + guard raw.count <= range.count else { return range } + let candidate = range.lowerBound..<(range.lowerBound + raw.count) + return characters[candidate].elementsEqual(raw) ? candidate : range + } + + func closingHTMLTagName(_ inlineHTML: InlineHTML) -> String? { + let raw = Array(inlineHTML.rawHTML) + guard raw.count >= 4, raw[0] == "<", raw[1] == "/" else { return nil } + var nameEnd = 2 + while nameEnd < raw.count, + raw[nameEnd].isLetter || raw[nameEnd].isNumber || raw[nameEnd] == "-" { + nameEnd += 1 + } + guard nameEnd > 2 else { return nil } + return String(raw[2.. Bool { + guard paragraphIdentifiers.indices.contains(lhs), + paragraphIdentifiers.indices.contains(rhs), + let identifier = paragraphIdentifiers[lhs] else { + return false + } + return paragraphIdentifiers[rhs] == identifier + } + func isParagraph(_ range: Range) -> Bool { paragraphRanges.contains(range) } @@ -199,42 +252,6 @@ struct MarkdownMathScanner { return result } - private static func angleRanges(in characters: [Character]) -> [Range] { - var result: [Range] = [] - var index = 0 - while index < characters.count { - guard characters[index] == "<", index + 1 < characters.count else { - index += 1 - continue - } - let next = characters[index + 1] - guard next.isLetter || next == "/" || next == "!" || next == "?" else { - index += 1 - continue - } - - var cursor = index + 2 - var quote: Character? - while cursor < characters.count, !characters[cursor].isNewline { - let character = characters[cursor] - if let activeQuote = quote { - if character == activeQuote { - quote = nil - } - } else if character == "\"" || character == "'" { - quote = character - } else if character == ">" { - result.append(index..<(cursor + 1)) - index = cursor - break - } - cursor += 1 - } - index += 1 - } - return result - } - private static func inlineCandidates( in characters: [Character], range: Range? = nil @@ -277,6 +294,72 @@ struct MarkdownMathScanner { } return result } + + private static func normalizedOpeningHTMLRanges( + in characters: [Character], + closingTags: [Int: String] + ) -> [Range] { + struct Opening { + let range: Range + let containsQuotedDollar: Bool + } + var stacks: [String: [Opening]] = [:] + var result: [Range] = [] + var index = 0 + while index < characters.count { + guard characters[index] == "<" else { + index += 1 + continue + } + var cursor = index + 1 + let isClosing = cursor < characters.count && characters[cursor] == "/" + if isClosing { cursor += 1 } + let nameStart = cursor + while cursor < characters.count, + characters[cursor].isLetter || characters[cursor].isNumber + || characters[cursor] == "-" { + cursor += 1 + } + guard cursor > nameStart else { + index += 1 + continue + } + let tagName = String(characters[nameStart.." { + end = cursor + 1 + break + } + cursor += 1 + } + guard let end else { + index += 1 + continue + } + if isClosing { + if closingTags[index] == tagName, + let opening = stacks[tagName]?.popLast(), + opening.containsQuotedDollar { + result.append(opening.range) + } + } else if characters[max(index, end - 2)] != "/" { + stacks[tagName, default: []].append( + Opening(range: index.. text.startIndex { + let preceding = text.index(before: lowerBound) + if text[preceding] == "?" { lowerBound = preceding } + } + if upperBound < text.endIndex, text[upperBound] == "?" { + upperBound = text.index(after: upperBound) + } + let range = lowerBound.. $$\nx\n> $$", 1, 3), + ("> $$\nx\n$$", 1, 3), ] for testCase in rejected { @@ -181,6 +181,48 @@ final class MarkdownMathScannerTests: XCTestCase { } } + func testInvalidHTMLLikeVisibleTextRemainsMathEligible() throws { + let source = "Visible tail" + let result = try MarkdownMathScanner().scan(source) + + XCTAssertEqual(result.tokens.count, 1) + XCTAssertEqual(result.tokens.first?.latex, "x") + XCTAssertEqual( + result.markdown, + "Visible tail" + ) + } + + func testMathTokenConsumptionRequiresExactlyOneCarrier() throws { + let token = RenderedMarkdownMathToken( + placeholder: "?TOKEN?", + omml: "", + kind: .inline, + line: 7, + column: 9 + ) + + for counts in [[:], [token.placeholder: 2]] { + XCTAssertThrowsError( + try MarkdownMathConsumptionValidator.validate( + tokens: [token], + consumedPlaceholders: counts + ) + ) { error in + XCTAssertEqual( + error as? MarkdownMathConversionError, + .formulaPlacementMismatch(line: 7, column: 9) + ) + } + } + XCTAssertNoThrow( + try MarkdownMathConsumptionValidator.validate( + tokens: [token], + consumedPlaceholders: [token.placeholder: 1] + ) + ) + } + func testCommonMarkDestinationAndReferenceMetadataRemainLiteral() throws { let sources = [ "Use [ref].\n\n[\nref\n]: https://example.com/$x$", diff --git a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift index 3e00b43c..8659d595 100644 --- a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift +++ b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift @@ -215,7 +215,7 @@ final class MarkdownOMathConversionTests: XCTestCase { "Before\n$$\nx\n$$\nAfter", "$$\n\nx\n\n$$", "- $$\n- x\n- $$", - "> $$\nx\n> $$", + "> $$\nx\n$$", ] for (index, source) in rejected.enumerated() { XCTAssertThrowsError( @@ -271,6 +271,19 @@ final class MarkdownOMathConversionTests: XCTestCase { } } + func testInvalidHTMLLikeVisibleTextEmitsInlineMath() throws { + let source = "Visible tail" + let xml = try documentXML( + markdown: source, + converter: MarkdownToWordConverter(mathMode: .omath) + ) + + XCTAssertEqual(count("", in: xml), 1) + XCTAssertTrue(xml.contains("<span "), "Got: \(xml)") + XCTAssertTrue(xml.contains("> tail"), "Got: \(xml)") + XCTAssertFalse(xml.contains("MDTOWORDMATHPLACEHOLDER"), "Got: \(xml)") + } + func testInvalidReferenceLikeVisibleTextStillConvertsInlineMath() throws { let sources = [ "[ref]: invalid destination $x$", From a45c9316736beb3234cbb81a9671ac02c95232a2 Mon Sep 17 00:00:00 2001 From: che cheng Date: Fri, 14 Aug 2026 04:14:25 +0800 Subject: [PATCH 6/7] spectra(markdown-omath-conversion): stabilize transformed sentinels (#141) --- .../MarkdownOMathRouteTests.swift | 41 +++++++----- .../markdown-omath-conversion/design.md | 2 +- .../specs/e2e-conversion-routes/spec.md | 2 +- .../specs/markdown-omath-conversion/spec.md | 2 +- .../markdown-omath-conversion/tasks.md | 5 ++ .../MDToWord/MarkdownMathScanner.swift | 67 +++++++++++++++++-- .../MDToWord/MarkdownToWordConverter.swift | 4 +- .../MarkdownMathScannerTests.swift | 26 ++++--- .../MarkdownOMathConversionTests.swift | 30 ++++++--- 9 files changed, 132 insertions(+), 47 deletions(-) diff --git a/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift b/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift index e3ec233f..738acd5f 100644 --- a/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift +++ b/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift @@ -297,7 +297,7 @@ struct MarkdownOMathRouteTests { func htmlAndFormattingBoundariesRemainLiteral() throws { let sources = [ "\nVisible", - #" $x$\">text"#, + #"text"#, "$*x*$", ] @@ -323,24 +323,31 @@ struct MarkdownOMathRouteTests { @Test("invalid HTML-like visible text remains math eligible") func invalidHTMLLikeVisibleTextEmitsMath() throws { - let workspace = try makeWorkspace() - defer { try? FileManager.default.removeItem(at: workspace) } - let input = workspace.appendingPathComponent("fixture.md") - let output = workspace.appendingPathComponent("fixture.docx") - try "Visible tail".write(to: input, atomically: true, encoding: .utf8) + let sources = [ + "Visible tail", + "Visible <$x$> tail", + "Visible tail ", + "Visible tail ", + ] + for source in sources { + let workspace = try makeWorkspace() + defer { try? FileManager.default.removeItem(at: workspace) } + let input = workspace.appendingPathComponent("fixture.md") + let output = workspace.appendingPathComponent("fixture.docx") + try source.write(to: input, atomically: true, encoding: .utf8) - let result = try CLITestHelper.convert( - to: "docx", - input: input.path, - flags: ["--math", "omath", "--output", output.path] - ) + let result = try CLITestHelper.convert( + to: "docx", + input: input.path, + flags: ["--math", "omath", "--output", output.path] + ) - #expect(result.succeeded, "stderr: \(result.stderr)") - let xml = try archiveEntry(named: "word/document.xml", in: output) - #expect(xml.components(separatedBy: "").count - 1 == 1) - #expect(xml.contains("<span ")) - #expect(xml.contains("> tail")) - #expect(!xml.contains("MDTOWORDMATHPLACEHOLDER")) + #expect(result.succeeded, "Source: \(source); stderr: \(result.stderr)") + let xml = try archiveEntry(named: "word/document.xml", in: output) + #expect(xml.components(separatedBy: "").count - 1 == 1) + #expect(xml.contains("<")) + #expect(!xml.contains("MDTOWORDMATHPLACEHOLDER")) + } } private func makeWorkspace() throws -> URL { diff --git a/openspec/changes/markdown-omath-conversion/design.md b/openspec/changes/markdown-omath-conversion/design.md index db7936ce..914fd764 100644 --- a/openspec/changes/markdown-omath-conversion/design.md +++ b/openspec/changes/markdown-omath-conversion/design.md @@ -55,7 +55,7 @@ The scanner then performs a bounded linear replacement pass over source characte - inline math: one unescaped `$`, non-whitespace first and last formula characters, no newline, and a valid closing `$`; - display math: `$$...$$` occupying an entire logical paragraph, either on one trimmed line or with standalone opening and closing delimiter lines. -Unmatched dollars remain literal. A matched display token found alongside non-whitespace paragraph content or spanning more than one original eligible paragraph is rejected rather than silently converted inline. A token placeholder is generated only after a linear pre-index proves its numeric suffix is absent from caller input. The placeholder is wrapped in punctuation that cannot turn invalid HTML-like visible text into an HTML attribute when the transformed source is parsed again. +Unmatched dollars remain literal. A matched display token found alongside non-whitespace paragraph content or spanning more than one original eligible paragraph is rejected rather than silently converted inline. A token placeholder is generated only after a linear pre-index proves its numeric suffix is absent from caller input. The placeholder is wrapped in a private-use Unicode sentinel that cannot turn invalid HTML-like visible text into an HTML attribute, tag, autolink, or processing instruction when the transformed source is parsed again. Alternatives rejected: diff --git a/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md b/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md index 2e6723a1..6c6ca05d 100644 --- a/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md +++ b/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md @@ -57,6 +57,6 @@ The compiled E2E suite SHALL exercise `macdoc convert` through the production co #### Scenario: Compiled route converts math in invalid HTML-like visible text -- **WHEN** the compiled binary receives `Visible tail` with `--math omath` +- **WHEN** the compiled binary receives ``, `<$x$>`, or an invalid quoted opening tag containing `$x$` as visible text with `--math omath` - **THEN** the DOCX contains one inline OMath carrier and preserves the visible angle-bracket text - **AND** neither document XML nor relationships contain a generated placeholder diff --git a/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md b/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md index 4a874b61..71254825 100644 --- a/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md +++ b/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md @@ -75,7 +75,7 @@ Every generated placeholder SHALL be consumed exactly once by an allowed visible #### Scenario: Invalid HTML-like visible text remains math-eligible -- **WHEN** CommonMark parses `Visible tail` as one visible `Text` node rather than HTML +- **WHEN** CommonMark parses `$x$` as visible `Text` inside ``, `<$x$>`, or an invalid quoted opening tag followed by a closing tag - **THEN** `.omath` mode converts `$x$` to one inline OMath carrier - **AND** the visible angle-bracket text remains present without a generated placeholder diff --git a/openspec/changes/markdown-omath-conversion/tasks.md b/openspec/changes/markdown-omath-conversion/tasks.md index 2bfe0bbb..2e17c379 100644 --- a/openspec/changes/markdown-omath-conversion/tasks.md +++ b/openspec/changes/markdown-omath-conversion/tasks.md @@ -45,3 +45,8 @@ - [x] 9.1 Add exact RED regressions for the original blockquote escape `> $$\nx\n$$`, invalid HTML-like visible text `Visible tail`, and zero/duplicate placeholder consumption; prove the frozen R4 candidate either overwrites `KEEP` or misses visible math before the production correction. - [x] 9.2 Make original paragraph identity reject the exact blockquote escape, replace the global raw-angle override with AST-anchored bounded HTML pairing, use punctuation-wrapped placeholders that cannot reclassify invalid tag-like text, and validate each token has exactly one consumer. GREEN evidence: focused scanner/converter 50/50, compiled CLI 14/14, root 59 total with 0 failures and 3 environment skips, package 139 total with exactly the documented 42 baseline failures, both diff gates PASS, and Spectra validation PASS. + +## 10. R5 transformed-source lexical stability + +- [x] 10.1 Add exact RED regressions for `Visible <$x$> tail` and invalid quoted opening tags followed by a valid closer; prove the R5 question-mark wrapper either becomes a processing instruction or the permissive HTML pairing suppresses visible math. +- [x] 10.2 Wrap placeholders in a private-use Unicode sentinel and require a syntactically valid named quoted attribute before AST-anchored opening-tag masking. GREEN evidence: focused scanner/converter 50/50, compiled CLI 14/14, root 59 total with 0 failures and 3 environment skips, and package 139 total with exactly the documented 42 baseline failures. The matrix includes ``, `<$x$>`, unnamed quoted attributes, invalid attribute names, and the existing valid quoted-HTML opaque case. diff --git a/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift index e94380ef..61435106 100644 --- a/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift +++ b/packages/md-to-word-swift/Sources/MDToWord/MarkdownMathScanner.swift @@ -301,7 +301,7 @@ struct MarkdownMathScanner { ) -> [Range] { struct Opening { let range: Range - let containsQuotedDollar: Bool + let containsValidQuotedDollarAttribute: Bool } var stacks: [String: [Opening]] = [:] var result: [Range] = [] @@ -325,13 +325,12 @@ struct MarkdownMathScanner { continue } let tagName = String(characters[nameStart.. Bool { + var index = start + var containsDollar = false + while index < end { + while index < end, characters[index].isWhitespace { index += 1 } + if index >= end { break } + if characters[index] == "/" { + index += 1 + while index < end, characters[index].isWhitespace { index += 1 } + return index == end && containsDollar + } + guard characters[index].isLetter || characters[index] == "_" + || characters[index] == ":" else { + return false + } + index += 1 + while index < end, + characters[index].isLetter || characters[index].isNumber + || characters[index] == "_" || characters[index] == ":" + || characters[index] == "." || characters[index] == "-" { + index += 1 + } + while index < end, characters[index].isWhitespace { index += 1 } + guard index < end, characters[index] == "=" else { continue } + index += 1 + while index < end, characters[index].isWhitespace { index += 1 } + guard index < end, + characters[index] == "\"" || characters[index] == "'" else { + while index < end, !characters[index].isWhitespace { index += 1 } + continue + } + let quote = characters[index] + index += 1 + var attributeContainsDollar = false + while index < end, characters[index] != quote { + if characters[index] == "$" { attributeContainsDollar = true } + index += 1 + } + guard index < end else { return false } + index += 1 + containsDollar = containsDollar || attributeContainsDollar + } + return containsDollar + } } init( @@ -393,7 +448,7 @@ struct MarkdownMathScanner { nextMarkerIndex += 1 } defer { nextMarkerIndex += 1 } - return "?\(markerStem)\(nextMarkerIndex)TOKEN?" + return "\u{E000}\(markerStem)\(nextMarkerIndex)TOKEN\u{E000}" } while index < characters.count { diff --git a/packages/md-to-word-swift/Sources/MDToWord/MarkdownToWordConverter.swift b/packages/md-to-word-swift/Sources/MDToWord/MarkdownToWordConverter.swift index 117acb31..7d96bafa 100644 --- a/packages/md-to-word-swift/Sources/MDToWord/MarkdownToWordConverter.swift +++ b/packages/md-to-word-swift/Sources/MDToWord/MarkdownToWordConverter.swift @@ -845,9 +845,9 @@ private struct MarkdownWordBuilder { var upperBound = suffixRange.upperBound if lowerBound > text.startIndex { let preceding = text.index(before: lowerBound) - if text[preceding] == "?" { lowerBound = preceding } + if text[preceding] == "\u{E000}" { lowerBound = preceding } } - if upperBound < text.endIndex, text[upperBound] == "?" { + if upperBound < text.endIndex, text[upperBound] == "\u{E000}" { upperBound = text.index(after: upperBound) } let range = lowerBound..", "
\n$x$\n
", - #" $x$\">text"#, + #"text"#, "$*x*$", "$**x**$", ] @@ -182,15 +182,23 @@ final class MarkdownMathScannerTests: XCTestCase { } func testInvalidHTMLLikeVisibleTextRemainsMathEligible() throws { - let source = "Visible tail" - let result = try MarkdownMathScanner().scan(source) + let sources = [ + "Visible tail", + "Visible <$x$> tail", + "Visible tail ", + "Visible tail ", + "Visible tail ", + ] + for source in sources { + let result = try MarkdownMathScanner().scan(source) - XCTAssertEqual(result.tokens.count, 1) - XCTAssertEqual(result.tokens.first?.latex, "x") - XCTAssertEqual( - result.markdown, - "Visible tail" - ) + XCTAssertEqual(result.tokens.count, 1, "Source: \(source)") + XCTAssertEqual(result.tokens.first?.latex, "x", "Source: \(source)") + XCTAssertTrue( + result.markdown.contains(try XCTUnwrap(result.tokens.first).placeholder), + "Source: \(source)" + ) + } } func testMathTokenConsumptionRequiresExactlyOneCarrier() throws { diff --git a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift index 8659d595..50775106 100644 --- a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift +++ b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownOMathConversionTests.swift @@ -272,16 +272,26 @@ final class MarkdownOMathConversionTests: XCTestCase { } func testInvalidHTMLLikeVisibleTextEmitsInlineMath() throws { - let source = "Visible tail" - let xml = try documentXML( - markdown: source, - converter: MarkdownToWordConverter(mathMode: .omath) - ) + let sources = [ + "Visible tail", + "Visible <$x$> tail", + "Visible tail ", + "Visible tail ", + "Visible tail ", + ] + for source in sources { + let xml = try documentXML( + markdown: source, + converter: MarkdownToWordConverter(mathMode: .omath) + ) - XCTAssertEqual(count("", in: xml), 1) - XCTAssertTrue(xml.contains("<span "), "Got: \(xml)") - XCTAssertTrue(xml.contains("> tail"), "Got: \(xml)") - XCTAssertFalse(xml.contains("MDTOWORDMATHPLACEHOLDER"), "Got: \(xml)") + XCTAssertEqual(count("", in: xml), 1, "Source: \(source)") + XCTAssertTrue(xml.contains("<"), "Source: \(source); got: \(xml)") + XCTAssertFalse( + xml.contains("MDTOWORDMATHPLACEHOLDER"), + "Source: \(source); got: \(xml)" + ) + } } func testInvalidReferenceLikeVisibleTextStillConvertsInlineMath() throws { @@ -315,7 +325,7 @@ final class MarkdownOMathConversionTests: XCTestCase { let sources = [ "\nVisible", "
\n$\\overbrace{x}$\n
\nVisible", - #" $x$\">text"#, + #"text"#, "$*x*$", "$**x**$", ] From 057327cf3dfb0212cf21f3a3709fe9b2c5d7ec8f Mon Sep 17 00:00:00 2001 From: che cheng Date: Fri, 14 Aug 2026 04:35:55 +0800 Subject: [PATCH 7/7] spectra(markdown-omath-conversion): bound CommonMark recovery (#141) --- .../MarkdownOMathRouteTests.swift | 3 + .../markdown-omath-conversion/design.md | 2 +- .../specs/e2e-conversion-routes/spec.md | 2 +- .../specs/markdown-omath-conversion/spec.md | 8 +- .../markdown-omath-conversion/tasks.md | 5 + .../MDToWord/MarkdownMathScanner.swift | 108 +++++++++++++----- .../MarkdownMathScannerTests.swift | 13 +++ .../MarkdownOMathConversionTests.swift | 3 + 8 files changed, 113 insertions(+), 31 deletions(-) diff --git a/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift b/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift index 738acd5f..7415ae2f 100644 --- a/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift +++ b/Tests/MacDocCLITests/MarkdownOMathRouteTests.swift @@ -298,6 +298,7 @@ struct MarkdownOMathRouteTests { let sources = [ "\nVisible", #"text"#, + #"text"#, "$*x*$", ] @@ -328,6 +329,8 @@ struct MarkdownOMathRouteTests { "Visible <$x$> tail", "Visible tail ", "Visible tail ", + "Visible tail ", + "Visible tail ", ] for source in sources { let workspace = try makeWorkspace() diff --git a/openspec/changes/markdown-omath-conversion/design.md b/openspec/changes/markdown-omath-conversion/design.md index 914fd764..bdd7f6e9 100644 --- a/openspec/changes/markdown-omath-conversion/design.md +++ b/openspec/changes/markdown-omath-conversion/design.md @@ -48,7 +48,7 @@ Alternatives rejected: An internal `MarkdownMathScanner` will produce a transformed Markdown string plus a token table. It replaces recognized math spans with collision-resistant placeholders before `swift-markdown` parses the document. This preserves surrounding emphasis, headings, lists, quotes, and tables while preventing the Markdown parser from interpreting LaTeX punctuation. -The converter first parses the original source with `swift-markdown` and converts source ranges into character offsets using the parser's UTF-8 line/column semantics. Text-node ranges define where inline math is eligible. When cmark normalizes visible text and reports a child range that does not cover its raw spelling, the scanner recovers only exact dollar-span signatures present in that normalized `Text`, matched from the suffix of the same plain-text paragraph; it never makes the whole paragraph or a metadata prefix eligible. A bounded raw pairing pass may mask an opening HTML spelling only when an original `InlineHTML` closing node anchors the same tag and the recovered dollar text lies inside a quoted attribute. Unpaired invalid HTML-like text remains governed by its original `Text` range and stays math-eligible. Paragraph ranges whose descendants contain only plain text and line-break nodes define where display math is eligible. Code, HTML, autolink, destination, reference metadata, and formatting-container ranges are therefore opaque without making a free-standing raw angle heuristic override the CommonMark tree. +The converter first parses the original source with `swift-markdown` and converts source ranges into character offsets using the parser's UTF-8 line/column semantics. Text-node ranges define where inline math is eligible. When cmark normalizes visible text and reports a child range that does not cover its raw spelling, the scanner recovers only exact dollar-span signatures present in that normalized `Text`, matched from the suffix of the same plain-text paragraph; it never makes the whole paragraph or a metadata prefix eligible. A bounded raw pairing pass may mask an opening HTML spelling only when an original `InlineHTML` closing node anchors the same tag, the complete opening spelling satisfies the CommonMark tag-and-attribute grammar, and the recovered dollar text lies inside a quoted attribute. The pass advances monotonically to a discovered tag or physical-line boundary so malformed prefixes cannot trigger suffix rescans. Unpaired or grammatically invalid HTML-like text remains governed by its original `Text` range and stays math-eligible. Paragraph ranges whose descendants contain only plain text and line-break nodes define where display math is eligible. Code, HTML, autolink, destination, reference metadata, and formatting-container ranges are therefore opaque without making a free-standing raw angle heuristic override the CommonMark tree. The scanner then performs a bounded linear replacement pass over source characters. It recognizes: diff --git a/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md b/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md index 6c6ca05d..f18d5ef0 100644 --- a/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md +++ b/openspec/changes/markdown-omath-conversion/specs/e2e-conversion-routes/spec.md @@ -57,6 +57,6 @@ The compiled E2E suite SHALL exercise `macdoc convert` through the production co #### Scenario: Compiled route converts math in invalid HTML-like visible text -- **WHEN** the compiled binary receives ``, `<$x$>`, or an invalid quoted opening tag containing `$x$` as visible text with `--math omath` +- **WHEN** the compiled binary receives ``, `<$x$>`, or a grammatically invalid opening tag containing visible `$x$` with `--math omath` - **THEN** the DOCX contains one inline OMath carrier and preserves the visible angle-bracket text - **AND** neither document XML nor relationships contain a generated placeholder diff --git a/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md b/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md index 71254825..e69aada2 100644 --- a/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md +++ b/openspec/changes/markdown-omath-conversion/specs/markdown-omath-conversion/spec.md @@ -75,10 +75,16 @@ Every generated placeholder SHALL be consumed exactly once by an allowed visible #### Scenario: Invalid HTML-like visible text remains math-eligible -- **WHEN** CommonMark parses `$x$` as visible `Text` inside ``, `<$x$>`, or an invalid quoted opening tag followed by a closing tag +- **WHEN** CommonMark parses `$x$` as visible `Text` inside ``, `<$x$>`, an opening tag whose unquoted value contains a forbidden character, or an opening tag whose attributes lack required whitespace - **THEN** `.omath` mode converts `$x$` to one inline OMath carrier - **AND** the visible angle-bracket text remains present without a generated placeholder +#### Scenario: Unterminated HTML-like prefixes remain bounded + +- **WHEN** one physical line contains many `` +- **THEN** source eligibility visits the line suffix a bounded number of times +- **AND** conversion time does not grow quadratically with the number of prefixes + #### Scenario: Visible text after invalid reference-like syntax remains eligible - **WHEN** a line beginning with reference-like or title-like syntax is parsed by CommonMark as visible text containing `$x$` diff --git a/openspec/changes/markdown-omath-conversion/tasks.md b/openspec/changes/markdown-omath-conversion/tasks.md index 2e17c379..3a007736 100644 --- a/openspec/changes/markdown-omath-conversion/tasks.md +++ b/openspec/changes/markdown-omath-conversion/tasks.md @@ -50,3 +50,8 @@ - [x] 10.1 Add exact RED regressions for `Visible <$x$> tail` and invalid quoted opening tags followed by a valid closer; prove the R5 question-mark wrapper either becomes a processing instruction or the permissive HTML pairing suppresses visible math. - [x] 10.2 Wrap placeholders in a private-use Unicode sentinel and require a syntactically valid named quoted attribute before AST-anchored opening-tag masking. GREEN evidence: focused scanner/converter 50/50, compiled CLI 14/14, root 59 total with 0 failures and 3 environment skips, and package 139 total with exactly the documented 42 baseline failures. The matrix includes ``, `<$x$>`, unnamed quoted attributes, invalid attribute names, and the existing valid quoted-HTML opaque case. + +## 11. R6 CommonMark attribute grammar and bounded malformed input + +- [x] 11.1 Add exact RED regressions for an unquoted attribute value containing forbidden `=`, missing whitespace between attributes, and 10,000 unterminated ` nameStart else { - index += 1 - continue - } let tagName = String(characters[nameStart..= end { break } + let separatorStart = index + while index < end, isHTMLSpace(characters[index]) { index += 1 } + if index >= end { return containsDollar } if characters[index] == "/" { index += 1 - while index < end, characters[index].isWhitespace { index += 1 } return index == end && containsDollar } - guard characters[index].isLetter || characters[index] == "_" - || characters[index] == ":" else { + guard index > separatorStart, + isASCIIAttributeNameStart(characters[index]) else { return false } index += 1 while index < end, - characters[index].isLetter || characters[index].isNumber - || characters[index] == "_" || characters[index] == ":" - || characters[index] == "." || characters[index] == "-" { + isASCIIAttributeNameContinuation(characters[index]) { index += 1 } - while index < end, characters[index].isWhitespace { index += 1 } - guard index < end, characters[index] == "=" else { continue } - index += 1 - while index < end, characters[index].isWhitespace { index += 1 } - guard index < end, - characters[index] == "\"" || characters[index] == "'" else { - while index < end, !characters[index].isWhitespace { index += 1 } + + let afterName = index + while index < end, isHTMLSpace(characters[index]) { index += 1 } + guard index < end, characters[index] == "=" else { + index = afterName continue } - let quote = characters[index] index += 1 - var attributeContainsDollar = false - while index < end, characters[index] != quote { - if characters[index] == "$" { attributeContainsDollar = true } + while index < end, isHTMLSpace(characters[index]) { index += 1 } + guard index < end else { return false } + + if characters[index] == "\"" || characters[index] == "'" { + let quote = characters[index] + index += 1 + var attributeContainsDollar = false + while index < end, characters[index] != quote { + guard characters[index].asciiValue != 0 else { return false } + if characters[index] == "$" { attributeContainsDollar = true } + index += 1 + } + guard index < end else { return false } index += 1 + containsDollar = containsDollar || attributeContainsDollar + } else { + let valueStart = index + while index < end, !isHTMLSpace(characters[index]) { + guard !isForbiddenUnquotedAttributeValueCharacter(characters[index]) + else { + return false + } + index += 1 + } + guard index > valueStart else { return false } } - guard index < end else { return false } - index += 1 - containsDollar = containsDollar || attributeContainsDollar } return containsDollar } + + private static func isASCIILetter(_ character: Character) -> Bool { + guard let value = character.asciiValue else { return false } + return (65...90).contains(value) || (97...122).contains(value) + } + + private static func isASCIINumber(_ character: Character) -> Bool { + guard let value = character.asciiValue else { return false } + return (48...57).contains(value) + } + + private static func isASCIIAttributeNameStart(_ character: Character) -> Bool { + isASCIILetter(character) || character == "_" || character == ":" + } + + private static func isASCIIAttributeNameContinuation(_ character: Character) -> Bool { + isASCIIAttributeNameStart(character) || isASCIINumber(character) + || character == "." || character == "-" + } + + private static func isHTMLSpace(_ character: Character) -> Bool { + character == " " || character == "\t" || character == "\r" + || character == "\n" || character == "\u{000B}" + || character == "\u{000C}" + } + + private static func isForbiddenUnquotedAttributeValueCharacter( + _ character: Character + ) -> Bool { + character.asciiValue == 0 || character == "\"" || character == "'" + || character == "=" || character == "<" || character == ">" + || character == "`" + } } init( diff --git a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift index e178f821..f5bffcee 100644 --- a/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift +++ b/packages/md-to-word-swift/Tests/MDToWordTests/MarkdownMathScannerTests.swift @@ -188,6 +188,8 @@ final class MarkdownMathScannerTests: XCTestCase { "Visible tail ", "Visible tail ", "Visible tail ", + "Visible tail ", + "Visible tail ", ] for source in sources { let result = try MarkdownMathScanner().scan(source) @@ -201,6 +203,17 @@ final class MarkdownMathScannerTests: XCTestCase { } } + func testUnterminatedHTMLLikePrefixesDoNotRescanLineSuffixes() throws { + let source = String(repeating: " tail ", "Visible tail ", "Visible tail ", + "Visible tail ", + "Visible tail ", ] for source in sources { let xml = try documentXML( @@ -326,6 +328,7 @@ final class MarkdownOMathConversionTests: XCTestCase { "\nVisible", "
\n$\\overbrace{x}$\n
\nVisible", #"text"#, + #"text"#, "$*x*$", "$**x**$", ]