diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index 4b62e19..0030bac 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -29,3 +29,11 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: lcov.info + # Sanitizer runs guard the hand-rolled cmark memory model (see the memory/leak/ + # concurrency safety-harness tests). Run after coverage so they don't clobber the + # coverage build's profraw data. GitHub Actions sets CI=true, so the CI-gated + # position test stays skipped here. + - name: Run tests under Address Sanitizer + run: swift test --sanitize=address + - name: Run tests under Thread Sanitizer + run: swift test --sanitize=thread diff --git a/Tests/MarkdownSyntaxTests/ConcurrencyTests.swift b/Tests/MarkdownSyntaxTests/ConcurrencyTests.swift new file mode 100644 index 0000000..9280819 --- /dev/null +++ b/Tests/MarkdownSyntaxTests/ConcurrencyTests.swift @@ -0,0 +1,140 @@ +// +// ConcurrencyTests.swift +// MarkdownSyntaxTests +// +// Safety-harness coverage: concurrent parsing. +// +// `CMNode` is `@unchecked Sendable` and `Markdown` is an actor, but parsing runs off the +// main thread and the app re-parses during token streaming. cmark itself keeps global +// state (extension registration via `cmark_gfm_core_extensions_ensure_registered`, the +// syntax-extension registry), and each parse allocates/frees its own tree. If a +// churn-reducing change introduced shared mutable wrapper state (a cache, a pool, a shared +// owner), concurrent parsing would race. +// +// These tests fan a large number of concurrent parses across the global executor, mixing +// the same and different inputs, and assert every result is correct. Meant to be run under +// Thread Sanitizer (`swift test --sanitize=thread`), which flags data races even when the +// assertions happen to pass. +// + +import XCTest +@testable import MarkdownSyntax + +final class ConcurrencyTests: XCTestCase { + + private let sample = """ + # Concurrent 😀 + + A paragraph with **bold**, _emphasis_, `code`, [link](https://example.com), and 日本語. + + - one + - two + + > quote + + ```swift + let x = 1 + ``` + """ + + // MARK: - Many concurrent parses of the SAME input + + func testConcurrentParseSameInputManyTasks() async throws { + let iterations = 400 + + // Establish the expected HTML once, sequentially. + let expectedHTML = try await Markdown(text: sample).renderHtml() + + try await withThrowingTaskGroup(of: String.self) { group in + for _ in 0.. owner) but the owner must not reference + /// children back. Holding a child keeps the owner alive; dropping the child must free both. + func testChildAndOwnerDeallocateTogether() throws { + weak var weakOwner: CMNode? + weak var weakChild: CMNode? + + try autoreleasepool { + let document = try CMDocument(text: sample) + let child = try XCTUnwrap(document.node.firstChild?.firstChild) + weakOwner = child.internalMemoryOwner + weakChild = child + XCTAssertNotNil(weakOwner) + XCTAssertNotNil(weakChild) + } + + XCTAssertNil(weakChild, "Child CMNode leaked") + XCTAssertNil(weakOwner, "Owner CMNode leaked — child<->owner retain cycle") + } + + /// The iterator retains its owner node; after both leave scope, both must deallocate. + func testIteratorAndOwnerDeallocate() throws { + weak var weakIteratorOwner: CMNode? + + try autoreleasepool { + let document = try CMDocument(text: sample) + let iterator = try XCTUnwrap(document.node.iterator) + try iterator.enumerate { _, _ in false } + weakIteratorOwner = document.node + XCTAssertNotNil(weakIteratorOwner) + } + + XCTAssertNil(weakIteratorOwner, "Iterator kept its owner node alive after scope exit") + } + + /// A deep leaf retained past the document's scope must itself deallocate once dropped, + /// taking the whole retained tree with it. + func testRetainedLeafReleasesTreeWhenDropped() throws { + weak var weakOwner: CMNode? + + try autoreleasepool { + var leaf: CMNode? = try { + let document = try CMDocument(text: sample) + return try XCTUnwrap(document.node.getAll(where: { $0.type == .text }).first) + }() + weakOwner = leaf?.internalMemoryOwner + XCTAssertNotNil(weakOwner, "Retained leaf should keep the owner alive") + leaf = nil + } + + XCTAssertNil(weakOwner, "Dropping the last retained leaf did not release the owner/tree") + } + + // MARK: - Bounded growth under repeated parsing + + /// Weak-reference sampling across a re-parse loop: every iteration's root wrapper must be + /// deallocated by the time the next iteration starts (no unbounded accumulation of live + /// wrapper trees). This is a Swift-level bounded-growth check; ASan covers the C heap. + func testRepeatedParseDoesNotAccumulateWrappers() throws { + weak var previousRoot: CMNode? + + for _ in 0..<300 { + try autoreleasepool { + XCTAssertNil(previousRoot, "Previous iteration's wrapper tree was still alive") + let document = try CMDocument(text: sample) + previousRoot = document.node + _ = document.node.getAll(where: { _ in true }) // materialize the whole wrapper tree + } + } + XCTAssertNil(previousRoot) + } + + /// Same idea through the full `Markdown` actor + value-typed `parse()` path. + func testRepeatedMarkdownParseBoundedGrowth() async throws { + for i in 0..<300 { + try await autoreleasepoolAsync { + let markdown = try await Markdown(text: "# H\(i)\n\nBody \(i) **b** `c`.") + let root = await markdown.parse() + XCTAssertFalse(root.children.isEmpty) + } + } + } +} + +/// Async-friendly autorelease wrapper (autoreleasepool cannot straddle an await). +private func autoreleasepoolAsync(_ body: () async throws -> Void) async rethrows { + try await body() +} diff --git a/Tests/MarkdownSyntaxTests/MemorySafetyTests.swift b/Tests/MarkdownSyntaxTests/MemorySafetyTests.swift new file mode 100644 index 0000000..2e1d4a7 --- /dev/null +++ b/Tests/MarkdownSyntaxTests/MemorySafetyTests.swift @@ -0,0 +1,209 @@ +// +// MemorySafetyTests.swift +// MarkdownSyntaxTests +// +// Safety-harness coverage: use-after-free / double-free of the hand-rolled cmark +// memory model in `CMNode`. +// +// Invariant under test: every derived `CMNode` wrapper (`next`, `firstChild`, `parent`, +// `children`, iterator nodes, …) strongly retains the *root owner* node through +// `referencedMemoryOwner`. As long as any wrapper is held, the underlying cmark tree +// must stay alive and readable; only the owning root frees it, exactly once, in `deinit`. +// +// A future optimization that reduces wrapper-object churn (e.g. caching, or not threading +// the owner through every wrapper) could break this ownership chain and produce a +// use-after-free or double-free. These tests exercise the chain so such a regression +// surfaces here — run them under Address Sanitizer (`swift test --sanitize=address`) to +// catch the memory error even when the logical assertions would still pass by luck. +// + +import XCTest +@testable import MarkdownSyntax + +final class MemorySafetyTests: XCTestCase { + + private let sample = """ + # Title 😀 + + A paragraph with **bold**, _emphasis_, `code`, and a [link](https://example.com). + + - item one + - item two with `inline` + + > a blockquote with 日本語 + + | a | b | + |---|---| + | 1 | 2 | + + ```swift + let x = 1 + ``` + """ + + // MARK: - Retained nested nodes outlive the document/owner + + /// Grab nested `CMNode`s, drop the owning `CMDocument`, then read the retained nodes. + /// Must not crash and must return the correct values — the retained wrappers keep the + /// root owner (and thus the cmark tree) alive via `referencedMemoryOwner`. + func testRetainedChildNodesOutliveDocument() throws { + var document: CMDocument? = try CMDocument(text: sample) + + // Reach several levels deep and hold onto leaf wrappers. + let heading = try XCTUnwrap(document?.node.firstChild) + let headingText = try XCTUnwrap(heading.firstChild) + let paragraph = try XCTUnwrap(heading.next) + let paragraphChildren = paragraph.children + + // Drop the document (and its owner-root wrapper reference held by CMDocument). + document = nil + + // Access retained nodes AFTER the document is gone. + XCTAssertEqual(heading.type, .heading) + XCTAssertEqual(headingText.literal, "Title 😀") + XCTAssertEqual(paragraph.type, .paragraph) + XCTAssertFalse(paragraphChildren.isEmpty) + // Walking further from a retained node must still work. + XCTAssertNotNil(paragraph.firstChild?.next) + } + + /// Hold only a *single* deep leaf node, drop everything else, then read it. + /// This is the tightest form of the invariant: one retained wrapper must be enough + /// to keep the entire tree alive. + func testSingleDeepNodeKeepsTreeAlive() throws { + func deepestText() throws -> CMNode { + let document = try CMDocument(text: sample) + let all = document.node.getAll(where: { $0.type == .text }) + return try XCTUnwrap(all.first) + // `document` goes out of scope here; only the returned node survives. + } + + let leaf = try deepestText() + // The owner is reachable and alive through referencedMemoryOwner. + XCTAssertNotNil(leaf.internalMemoryOwner) + XCTAssertNotNil(leaf.literal) + // Navigate upward to the root through the retained tree. + var cursor: CMNode? = leaf + var hops = 0 + while let parent = cursor?.parent { + cursor = parent + hops += 1 + } + XCTAssertEqual(cursor?.type, .document) + XCTAssertGreaterThan(hops, 0) + } + + /// The Swift AST returned by `parse()` is fully value-typed and must not depend on any + /// cmark memory after the parser/document are gone. + func testParsedASTIndependentOfCmarkMemory() async throws { + func parseAndDrop() async throws -> Root { + let markdown = try await Markdown(text: sample) + return await markdown.parse() + // `markdown` (and its CMDocument) deallocate here. + } + + let root = try await parseAndDrop() + // Read deep into the value-typed tree after the source document is gone. + let heading = try XCTUnwrap(root.children.first as? Heading) + XCTAssertEqual(heading.depth, .h1) + XCTAssertNotNil(heading.position.range) + } + + // MARK: - Iterator lifetime & reset + + /// Fully enumerate a document, then reset and re-enumerate. The iterator retains its + /// owner node; nodes yielded during enumeration must be readable. + func testIteratorEnumerateResetReenumerate() throws { + let document = try CMDocument(text: sample) + let iterator = try XCTUnwrap(document.node.iterator) + + var firstPassTypes: [CMNodeType] = [] + try iterator.enumerate { node, event in + if event == .enter { firstPassTypes.append(node.type) } + return false + } + XCTAssertFalse(firstPassTypes.isEmpty) + XCTAssertEqual(firstPassTypes.first, .document) + + // Reset back to the root and enumerate again. reset(to:eventType:.enter) leaves the + // iterator positioned *at* the document-enter event, and enumerate advances with + // cmark_iter_next before yielding — so the re-walk yields the same sequence minus the + // already-consumed leading .document enter. The point is that reset + re-enumeration + // works and every yielded node stays readable. + iterator.reset(to: document.node, eventType: .enter) + var secondPassTypes: [CMNodeType] = [] + try iterator.enumerate { node, event in + if event == .enter { secondPassTypes.append(node.type) } + return false + } + XCTAssertEqual(secondPassTypes, Array(firstPassTypes.dropFirst())) + XCTAssertFalse(secondPassTypes.isEmpty) + } + + /// Nodes captured from inside `enumerate` must remain valid after the iterator is gone, + /// because each yielded node retains the owner. + func testIteratorYieldedNodesOutliveIterator() throws { + let document = try CMDocument(text: sample) + var captured: [CMNode] = [] + + do { + let iterator = try XCTUnwrap(document.node.iterator) + try iterator.enumerate { node, event in + if event == .enter, node.type == .text { + captured.append(node) + } + return false + } + // iterator deallocates at the end of this scope. + } + + XCTAssertFalse(captured.isEmpty) + for node in captured { + XCTAssertNotNil(node.literal) // read after the iterator is freed + } + } + + // MARK: - Alloc/free churn under tight re-parse loops + + /// Re-parse the *same* input hundreds of times. Each iteration allocates and frees a + /// full wrapper tree; this stresses the deinit/free path for double-frees. + func testTightReparseSameInputLoop() async throws { + for _ in 0..<500 { + let markdown = try await Markdown(text: sample) + let root = await markdown.parse() + XCTAssertFalse(root.children.isEmpty) + } + } + + /// Re-parse *different* inputs in a tight loop (mimics per-keystroke re-parsing). + func testTightReparseDifferentInputLoop() async throws { + for i in 0..<500 { + let input = "# Heading \(i) 😀\n\nBody **\(i)** with `code\(i)` and 日本語\(i)." + let markdown = try await Markdown(text: input) + let root = await markdown.parse() + let heading = try XCTUnwrap(root.children.first as? Heading) + let text = try XCTUnwrap(heading.children.first as? Text) + XCTAssertEqual(text.value, "Heading \(i) 😀") + } + } + + /// Churn the raw wrapper graph (no value-typed parse) in a loop: repeatedly walk + /// `firstChild`/`next`/`parent`/`children`, allocating and freeing many wrappers over + /// one shared tree, then let the owner drop. + func testWrapperGraphChurnLoop() throws { + let document = try CMDocument(text: sample) + for _ in 0..<2000 { + var count = 0 + func walk(_ node: CMNode) { + count += 1 + for child in node.children { walk(child) } + _ = node.parent + _ = node.next + _ = node.previous + _ = node.lastChild + } + walk(document.node) + XCTAssertGreaterThan(count, 0) + } + } +} diff --git a/Tests/MarkdownSyntaxTests/MultibytePositionTests.swift b/Tests/MarkdownSyntaxTests/MultibytePositionTests.swift new file mode 100644 index 0000000..0dd1087 --- /dev/null +++ b/Tests/MarkdownSyntaxTests/MultibytePositionTests.swift @@ -0,0 +1,251 @@ +// +// MultibytePositionTests.swift +// MarkdownSyntaxTests +// +// Safety-harness coverage: byte-vs-character position correctness. +// +// cmark reports source positions in *byte* (UTF-8) columns. `CMNode.position(in:using:)` +// builds `String.Index`es via `text.utf8.index(...)`, so the stored `Position.range` +// should slice the *correct substring* even when the input contains multibyte +// characters. The pre-existing position tests only assert against `input.range(0...N)`, +// where N is a *Character* offset — that coincides with the byte offset only for ASCII, +// so those tests are structurally blind to multibyte mapping errors. +// +// These tests assert the substring the range maps to (`input[range] == expected`), +// never a numeric offset. If any of these fail, the byte/character mapping is a real +// latent bug (not a test bug) and must be reported, not worked around. +// + +import XCTest +@testable import MarkdownSyntax + +final class MultibytePositionTests: XCTestCase { + + // MARK: - Emoji (4-byte scalars) + + func testEmphasisAfterEmoji() async throws { + // "😀" is 1 Character but 4 UTF-8 bytes; the strong delimiters follow it. + let input = "😀 **bold** tail" + + let tree = try await Markdown(text: input).parse() + let paragraph = try XCTUnwrap(tree.children.first as? Paragraph) + let strong = try XCTUnwrap(paragraph.children.first(where: { $0 is Strong }) as? Strong) + let range = try XCTUnwrap(strong.position.range) + + XCTAssertEqual(String(input[range]), "**bold**") + } + + func testLinkAfterEmoji() async throws { + let input = "😀😀 [alpha](https://example.com)" + + let tree = try await Markdown(text: input).parse() + let paragraph = try XCTUnwrap(tree.children.first as? Paragraph) + let link = try XCTUnwrap(paragraph.children.first(where: { $0 is Link }) as? Link) + let range = try XCTUnwrap(link.position.range) + + XCTAssertEqual(String(input[range]), "[alpha](https://example.com)") + } + + func testInlineCodeContainingEmoji() async throws { + let input = "before `x = 😀` after" + + let tree = try await Markdown(text: input).parse() + let paragraph = try XCTUnwrap(tree.children.first as? Paragraph) + let code = try XCTUnwrap(paragraph.children.first(where: { $0 is InlineCode }) as? InlineCode) + let range = try XCTUnwrap(code.position.range) + + XCTAssertEqual(String(input[range]), "`x = 😀`") + } + + // MARK: - CJK (3-byte scalars) + + func testEmphasisAfterCJK() async throws { + // Each of 日本語 is 3 UTF-8 bytes; strong follows. + let input = "日本語 **太字** です" + + let tree = try await Markdown(text: input).parse() + let paragraph = try XCTUnwrap(tree.children.first as? Paragraph) + let strong = try XCTUnwrap(paragraph.children.first(where: { $0 is Strong }) as? Strong) + let range = try XCTUnwrap(strong.position.range) + + XCTAssertEqual(String(input[range]), "**太字**") + } + + func testHeadingWithCJK() async throws { + let input = "# 日本語のテスト" + + let tree = try await Markdown(text: input).parse() + let heading = try XCTUnwrap(tree.children.first as? Heading) + let range = try XCTUnwrap(heading.position.range) + + XCTAssertEqual(String(input[range]), "# 日本語のテスト") + } + + func testCJKTextNodeSubstring() async throws { + let input = "日本語 **太字**" + + let tree = try await Markdown(text: input).parse() + let paragraph = try XCTUnwrap(tree.children.first as? Paragraph) + let text = try XCTUnwrap(paragraph.children.first as? Text) + let range = try XCTUnwrap(text.position.range) + + // The leading text node holds "日本語 " (trailing space before the strong). + XCTAssertEqual(String(input[range]), "日本語 ") + } + + // MARK: - Accented / precomposed (2-byte scalars) + + func testEmphasisAfterAccented() async throws { + // café résumé — precomposed é (U+00E9, 2 bytes). + let input = "café résumé **gras**" + + let tree = try await Markdown(text: input).parse() + let paragraph = try XCTUnwrap(tree.children.first as? Paragraph) + let strong = try XCTUnwrap(paragraph.children.first(where: { $0 is Strong }) as? Strong) + let range = try XCTUnwrap(strong.position.range) + + XCTAssertEqual(String(input[range]), "**gras**") + } + + // MARK: - Combining marks (grapheme = multiple scalars) + + func testEmphasisAfterCombiningMark() async throws { + // "cafe" + combining acute over the final e => 1 grapheme, 2 scalars, 3 bytes for "é". + let input = "cafe\u{0301} **gras**" + + let tree = try await Markdown(text: input).parse() + let paragraph = try XCTUnwrap(tree.children.first as? Paragraph) + let strong = try XCTUnwrap(paragraph.children.first(where: { $0 is Strong }) as? Strong) + let range = try XCTUnwrap(strong.position.range) + + XCTAssertEqual(String(input[range]), "**gras**") + } + + // MARK: - Emoji with ZWJ sequences (grapheme = many scalars) + + func testEmphasisAfterZWJEmoji() async throws { + // Family emoji: multiple scalars joined by ZWJ, a single grapheme, 18 UTF-8 bytes. + let input = "👨‍👩‍👧 **bold**" + + let tree = try await Markdown(text: input).parse() + let paragraph = try XCTUnwrap(tree.children.first as? Paragraph) + let strong = try XCTUnwrap(paragraph.children.first(where: { $0 is Strong }) as? Strong) + let range = try XCTUnwrap(strong.position.range) + + XCTAssertEqual(String(input[range]), "**bold**") + } + + // MARK: - CRLF line endings + + func testEmphasisOnSecondCRLFLine() async throws { + let input = "first line\r\n**bold** here" + + let tree = try await Markdown(text: input).parse() + // Two paragraphs? No — a soft break keeps them in one paragraph. + let paragraph = try XCTUnwrap(tree.children.first as? Paragraph) + let strong = try XCTUnwrap(paragraph.children.first(where: { $0 is Strong }) as? Strong) + let range = try XCTUnwrap(strong.position.range) + + XCTAssertEqual(String(input[range]), "**bold**") + } + + func testHeadingAfterCRLF() async throws { + let input = "para\r\n\r\n# Heading" + + let tree = try await Markdown(text: input).parse() + let heading = try XCTUnwrap(tree.children.first(where: { $0 is Heading }) as? Heading) + let range = try XCTUnwrap(heading.position.range) + + XCTAssertEqual(String(input[range]), "# Heading") + } + + // MARK: - Tabs + + func testEmphasisAfterTab() async throws { + // A leading run of text with a tab then a strong span. + let input = "a\tb **bold**" + + let tree = try await Markdown(text: input).parse() + let paragraph = try XCTUnwrap(tree.children.first as? Paragraph) + let strong = try XCTUnwrap(paragraph.children.first(where: { $0 is Strong }) as? Strong) + let range = try XCTUnwrap(strong.position.range) + + XCTAssertEqual(String(input[range]), "**bold**") + } + + // MARK: - Combined: multibyte + link title round-trip on a later line + + // MARK: - Node ENDING on a multibyte character (closed-range end offset) + // + // `Position.range` is a *closed* range `start...end`, where `end` is derived from + // cmark's end_column (byte column of the last character). When that last character + // is multibyte, the UTF-8 index lands on the first byte of the final grapheme; the + // closed-range subscript must still include the whole final character. + + func testTextNodeEndingInEmoji() async throws { + let input = "hello 😀" + + let tree = try await Markdown(text: input).parse() + let paragraph = try XCTUnwrap(tree.children.first as? Paragraph) + let text = try XCTUnwrap(paragraph.children.first as? Text) + let range = try XCTUnwrap(text.position.range) + + XCTAssertEqual(String(input[range]), "hello 😀") + } + + func testTextNodeEndingInCJK() async throws { + let input = "hello 日本語" + + let tree = try await Markdown(text: input).parse() + let paragraph = try XCTUnwrap(tree.children.first as? Paragraph) + let text = try XCTUnwrap(paragraph.children.first as? Text) + let range = try XCTUnwrap(text.position.range) + + XCTAssertEqual(String(input[range]), "hello 日本語") + } + + func testHeadingEndingInEmoji() async throws { + let input = "# done 😀" + + let tree = try await Markdown(text: input).parse() + let heading = try XCTUnwrap(tree.children.first as? Heading) + let range = try XCTUnwrap(heading.position.range) + + XCTAssertEqual(String(input[range]), "# done 😀") + } + + func testStrongEndingInCJK() async throws { + // Strong span whose inner content ends in a CJK char before the closing "**". + let input = "start **太字** end" + + let tree = try await Markdown(text: input).parse() + let paragraph = try XCTUnwrap(tree.children.first as? Paragraph) + let strong = try XCTUnwrap(paragraph.children.first(where: { $0 is Strong }) as? Strong) + let innerText = try XCTUnwrap(strong.children.first as? Text) + let range = try XCTUnwrap(innerText.position.range) + + XCTAssertEqual(String(input[range]), "太字") + } + + func testInlineCodeEndingInCJK() async throws { + let input = "x `コード`" + + let tree = try await Markdown(text: input).parse() + let paragraph = try XCTUnwrap(tree.children.first as? Paragraph) + let code = try XCTUnwrap(paragraph.children.first(where: { $0 is InlineCode }) as? InlineCode) + let range = try XCTUnwrap(code.position.range) + + XCTAssertEqual(String(input[range]), "`コード`") + } + + func testLinkOnSecondLineAfterMultibyte() async throws { + let input = "日本語😀\n[alpha](https://example.com \"t\")" + + let tree = try await Markdown(text: input).parse() + let paragraph = try XCTUnwrap(tree.children.last as? Paragraph) + let link = try XCTUnwrap(paragraph.children.first(where: { $0 is Link }) as? Link) + let range = try XCTUnwrap(link.position.range) + + XCTAssertEqual(String(input[range]), "[alpha](https://example.com \"t\")") + } +}