From b985d6bb4f9feb99b7caaec5b9aeb13d52527dc5 Mon Sep 17 00:00:00 2001 From: DemchaAV Date: Wed, 1 Jul 2026 23:23:56 +0100 Subject: [PATCH] feat(parser): opt-in typographic smart punctuation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MarkdownComposer.builder().smartPunctuation(true) enables flexmark-ext-typographic: straight quotes become curly quotes (nested formatting inside the quotes survives via TypographicQuotes flattening in the mapper), -- an en-dash, --- an em-dash, ... an ellipsis. Off by default — GitHub does not smart-quote, so default output is byte-identical — and code spans/blocks stay verbatim. Replacement characters ride the existing HTML-entity decoder; curly-quote and guillemet entities added to NAMED_ENTITIES. Tests: replacements on, literal off-by-default, code immunity, bold-inside-quotes. Suite green (173). --- CHANGELOG.md | 19 +++ README.md | 4 + pom.xml | 5 + .../markdown/composer/MarkdownComposer.java | 18 ++- .../markdown/mapper/FlexmarkAstMapper.java | 24 +++- .../parser/FlexmarkMarkdownParser.java | 25 +++- .../markdown/SmartPunctuationTest.java | 117 ++++++++++++++++++ 7 files changed, 207 insertions(+), 5 deletions(-) create mode 100644 src/test/java/io/github/demchaav/markdown/SmartPunctuationTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 79e6ae3..c6e062a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,11 @@ and the project follows [Semantic Versioning](https://semver.org/). link — a heading's single anchor slot already carries its navigation slug. ### Changed +- **The HTML-entity decoder now handles curly-quote and guillemet entities (all modes).** A + literal `’` / `“` / `«` … in the Markdown source now decodes to its character + (`’` `“` `«`) instead of rendering as the raw entity text. This applies regardless of the + `smartPunctuation` flag — it is the GitHub-parity behaviour for HTML entities — and is the one + default-mode output change in this release's punctuation work. - **Renderer classes reorganised (source-compatible for most users).** The four largest built-in renderers moved from nested classes to top-level classes in the same package: `BuiltinRenderers.ListRenderer` → `ListRenderer`, and likewise `TableRenderer`, @@ -28,6 +33,20 @@ and the project follows [Semantic Versioning](https://semver.org/). needs the import updated. `BuiltinRenderers` drops from ~750 to ~440 lines. ### Added +- **Opt-in smart punctuation — `MarkdownComposer.builder().smartPunctuation(true)`.** Typographic + replacements at parse time: straight quotes become curly quotes (nested formatting inside the + quotes survives), apostrophes curl (`don't` → `don’t`), `--` becomes an en-dash, `---` an + em-dash, `...` (and spaced `. . .`) an ellipsis, and `<>` becomes `«text»` guillemets. + Off by default — GitHub does not smart-quote — and code spans / code blocks always stay + verbatim. Backed by `flexmark-ext-typographic`. +- **Book-style page-numbered table of contents — `BookTocRenderer`.** An opt-in alternative to + the default `[TOC]` link list: swap the `TocNode` renderer + (`MarkdownTheme.builder(base).renderer(TocNode.class, new BookTocRenderer("Contents"))`) and + the marker renders as dot-leader contents rows — "Introduction ….. 3" — with **page numbers + resolved automatically from the laid-out document** (the engine's `addTableOfContents`; no + manual two-pass) and every label a clickable jump. Heading nesting indents the label; an + optional title row; empty-heading and no-heading documents degrade exactly like the default + renderer. Runnable `BookTocExample` (paired with the page-number footer). - **Book-style page-numbered table of contents — `BookTocRenderer`.** An opt-in alternative to the default `[TOC]` link list: swap the `TocNode` renderer (`MarkdownTheme.builder(base).renderer(TocNode.class, new BookTocRenderer("Contents"))`) and diff --git a/README.md b/README.md index e8000ee..df612f0 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,10 @@ MarkdownTheme numbered = MarkdownTheme.builder(t) .build(); ``` +Prefer typographic output? `MarkdownComposer.builder().smartPunctuation(true)` turns +`--` / `---` / `...` and straight quotes into – — … and “curly” quotes (code stays +verbatim; off by default, matching GitHub). + ## Command-line (CLI) A standalone `cli/` module renders Markdown to PDF from the shell — no Java code diff --git a/pom.xml b/pom.xml index da2eb7c..c4208b3 100644 --- a/pom.xml +++ b/pom.xml @@ -187,6 +187,11 @@ flexmark-ext-yaml-front-matter ${flexmark.version} + + com.vladsch.flexmark + flexmark-ext-typographic + ${flexmark.version} + diff --git a/src/main/java/io/github/demchaav/markdown/composer/MarkdownComposer.java b/src/main/java/io/github/demchaav/markdown/composer/MarkdownComposer.java index 6fa40ed..0db115a 100644 --- a/src/main/java/io/github/demchaav/markdown/composer/MarkdownComposer.java +++ b/src/main/java/io/github/demchaav/markdown/composer/MarkdownComposer.java @@ -57,7 +57,7 @@ public final class MarkdownComposer { private final MarkdownTheme theme; private MarkdownComposer(Builder builder) { - this.parser = new FlexmarkMarkdownParser(); + this.parser = new FlexmarkMarkdownParser(builder.smartPunctuation); this.mapper = new FlexmarkAstMapper(); this.customBlockParser = new CustomBlockParser(parser, mapper); this.customBlocksEnabled = builder.customBlocks; @@ -209,6 +209,7 @@ public static final class Builder { private boolean customBlocks = true; private boolean strict = false; private boolean openOutline = true; + private boolean smartPunctuation = false; private Builder() { } @@ -250,6 +251,21 @@ public Builder strictMode(boolean enabled) { return this; } + /** + * Enables typographic smart punctuation (default off, matching GitHub's rendering): + * straight quotes become curly quotes, apostrophes curl ({@code don't} → {@code don’t}), + * {@code --} becomes an en-dash, {@code ---} an em-dash, {@code ...} (and spaced + * {@code . . .}) an ellipsis, and {@code <>} becomes {@code «text»} guillemets. + * Code spans and code blocks always stay verbatim. + * + * @param enabled whether to apply typographic replacements + * @return this builder + */ + public Builder smartPunctuation(boolean enabled) { + this.smartPunctuation = enabled; + return this; + } + /** * Controls whether a rendered PDF asks the viewer to open its bookmark/outline panel * (default on). The preference is only written when the document actually has at least diff --git a/src/main/java/io/github/demchaav/markdown/mapper/FlexmarkAstMapper.java b/src/main/java/io/github/demchaav/markdown/mapper/FlexmarkAstMapper.java index 0a304f0..b95bba3 100644 --- a/src/main/java/io/github/demchaav/markdown/mapper/FlexmarkAstMapper.java +++ b/src/main/java/io/github/demchaav/markdown/mapper/FlexmarkAstMapper.java @@ -6,6 +6,8 @@ import com.vladsch.flexmark.ext.footnotes.FootnoteBlock; import com.vladsch.flexmark.ext.gfm.strikethrough.Strikethrough; import com.vladsch.flexmark.ext.gfm.tasklist.TaskListItem; +import com.vladsch.flexmark.ext.typographic.TypographicQuotes; +import com.vladsch.flexmark.ext.typographic.TypographicSmarts; import com.vladsch.flexmark.ext.tables.*; import com.vladsch.flexmark.ext.yaml.front.matter.AbstractYamlFrontMatterVisitor; import com.vladsch.flexmark.ext.yaml.front.matter.YamlFrontMatterBlock; @@ -38,7 +40,10 @@ public final class FlexmarkAstMapper { Map.entry("amp", "&"), Map.entry("lt", "<"), Map.entry("gt", ">"), Map.entry("quot", "\""), Map.entry("apos", "'"), Map.entry("nbsp", " "), Map.entry("copy", "©"), Map.entry("reg", "®"), Map.entry("trade", "™"), - Map.entry("mdash", "—"), Map.entry("ndash", "–"), Map.entry("hellip", "…")); + Map.entry("mdash", "—"), Map.entry("ndash", "–"), Map.entry("hellip", "…"), + Map.entry("lsquo", "‘"), Map.entry("rsquo", "’"), + Map.entry("ldquo", "“"), Map.entry("rdquo", "”"), + Map.entry("laquo", "«"), Map.entry("raquo", "»")); // Footnote label -> number, assigned by first-reference order, scoped to the current // mapping call. Flexmark does not populate footnote ordinals at parse time, and the @@ -141,6 +146,11 @@ private static String normalizeCodeText(String code) { return normalized; } + /** Decodes a typographic mark carried as an HTML entity; {@code null} (hand-built node) → empty. */ + private static String decodeTypographic(String entity) { + return entity == null ? "" : decodeEntity(entity); + } + private static String decodeEntity(String entity) { if (!entity.startsWith("&") || !entity.endsWith(";") || entity.length() < 3) { return entity; @@ -387,6 +397,13 @@ private List mapInlines(Node parent) { // A transparent wrapper Flexmark inserts (e.g. for autolinking) — flatten it, // so the Link/Text nodes inside are mapped rather than swallowed whole. result.addAll(mapInlines(child)); + } else if (child instanceof TypographicQuotes quotes) { + // Smart punctuation (opt-in): curly quote marks around the mapped inner + // inlines. Flexmark carries the marks as HTML entity names; reuse the decoder. + // Null marks (possible on hand-built nodes fed via render(Document)) emit nothing. + result.add(new TextRun(decodeTypographic(quotes.getTypographicOpening()))); + result.addAll(mapInlines(quotes)); + result.add(new TextRun(decodeTypographic(quotes.getTypographicClosing()))); } else { InlineNode mapped = mapInline(child); if (mapped != null) { @@ -465,6 +482,11 @@ private InlineNode mapInline(Node node) { if (node instanceof HtmlEntity entity) { return new TextRun(decodeEntity(entity.getChars().toString())); } + if (node instanceof TypographicSmarts smarts) { + // Smart punctuation (opt-in): -- / --- / ... as en/em-dash and ellipsis. Flexmark + // carries the replacement as an HTML entity name; reuse the decoder. + return new TextRun(decodeTypographic(smarts.getTypographicText())); + } if (node instanceof HtmlInline inline) { String raw = inline.getChars().toString(); String html = raw.trim().toLowerCase(); diff --git a/src/main/java/io/github/demchaav/markdown/parser/FlexmarkMarkdownParser.java b/src/main/java/io/github/demchaav/markdown/parser/FlexmarkMarkdownParser.java index 493743a..4529732 100644 --- a/src/main/java/io/github/demchaav/markdown/parser/FlexmarkMarkdownParser.java +++ b/src/main/java/io/github/demchaav/markdown/parser/FlexmarkMarkdownParser.java @@ -6,11 +6,14 @@ import com.vladsch.flexmark.ext.gfm.strikethrough.StrikethroughExtension; import com.vladsch.flexmark.ext.gfm.tasklist.TaskListExtension; import com.vladsch.flexmark.ext.tables.TablesExtension; +import com.vladsch.flexmark.ext.typographic.TypographicExtension; import com.vladsch.flexmark.ext.yaml.front.matter.YamlFrontMatterExtension; import com.vladsch.flexmark.parser.Parser; import com.vladsch.flexmark.util.ast.Document; import com.vladsch.flexmark.util.data.MutableDataSet; +import com.vladsch.flexmark.util.misc.Extension; +import java.util.ArrayList; import java.util.List; /** @@ -19,8 +22,9 @@ *

This is the only place in the library that touches Flexmark. The configured * {@link Parser} enables CommonMark plus the GFM extensions for tables, task lists, * strikethrough and footnotes, and the extensions for emoji shortcodes, autolinking - * and YAML front matter. A single parser instance is reused across calls; Flexmark's - * {@code parse} is safe to call concurrently once the parser is built.

+ * and YAML front matter — plus, optionally, typographic smart punctuation. A single + * parser instance is reused across calls; Flexmark's {@code parse} is safe to call + * concurrently once the parser is built.

*/ public final class FlexmarkMarkdownParser { @@ -28,12 +32,27 @@ public final class FlexmarkMarkdownParser { /** Creates a parser for CommonMark plus the GFM, emoji, autolink and front-matter extensions. */ public FlexmarkMarkdownParser() { + this(false); + } + + /** + * Creates a parser, optionally with typographic smart punctuation. + * + * @param smartPunctuation when {@code true}, straight quotes become curly quotes, + * {@code --} an en-dash, {@code ---} an em-dash and {@code ...} + * an ellipsis (code spans and code blocks stay verbatim) + */ + public FlexmarkMarkdownParser(boolean smartPunctuation) { MutableDataSet options = new MutableDataSet(); - options.set(Parser.EXTENSIONS, List.of( + List extensions = new ArrayList<>(List.of( StrikethroughExtension.create(), TablesExtension.create(), TaskListExtension.create(), FootnoteExtension.create(), EmojiExtension.create(), AutolinkExtension.create(), YamlFrontMatterExtension.create())); + if (smartPunctuation) { + extensions.add(TypographicExtension.create()); + } + options.set(Parser.EXTENSIONS, extensions); this.parser = Parser.builder(options).build(); } diff --git a/src/test/java/io/github/demchaav/markdown/SmartPunctuationTest.java b/src/test/java/io/github/demchaav/markdown/SmartPunctuationTest.java new file mode 100644 index 0000000..ae2b39e --- /dev/null +++ b/src/test/java/io/github/demchaav/markdown/SmartPunctuationTest.java @@ -0,0 +1,117 @@ +package io.github.demchaav.markdown; + +import io.github.demchaav.markdown.composer.MarkdownComposer; +import io.github.demchaav.markdown.model.ParagraphNode; +import io.github.demchaav.markdown.model.inline.StrongRun; +import io.github.demchaav.markdown.model.inline.TextRun; +import io.github.demchaav.markdown.theme.DefaultMarkdownTheme; +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.text.PDFTextStripper; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Opt-in typographic smart punctuation ({@code builder().smartPunctuation(true)}): dashes, + * ellipsis and curly quotes. Off by default (matching GitHub), and code stays verbatim. + */ +class SmartPunctuationTest { + + private static String text(MarkdownComposer composer, String md) throws Exception { + try (PDDocument doc = Loader.loadPDF(composer.render(md).toPdfBytes())) { + return new PDFTextStripper().getText(doc); + } + } + + @Test + void smartPunctuationReplacesDashesEllipsisAndQuotes() throws Exception { + MarkdownComposer composer = MarkdownComposer.builder().smartPunctuation(true).build(); + + String out = text(composer, "Pages 3--7 --- \"quoted\" and 'single', wait..."); + + assertThat(out).contains("–"); // en-dash from -- + assertThat(out).contains("—"); // em-dash from --- + assertThat(out).contains("…"); // ellipsis from ... + assertThat(out).contains("“").contains("”"); // curly double quotes + assertThat(out).contains("‘").contains("’"); // curly single quotes + assertThat(out).doesNotContain("--").doesNotContain("..."); + } + + @Test + void offByDefaultKeepsLiteralPunctuation() throws Exception { + MarkdownComposer composer = MarkdownComposer.create(DefaultMarkdownTheme.light()); + + String out = text(composer, "Pages 3--7 \"quoted\" wait..."); + + assertThat(out).contains("3--7").contains("\"quoted\"").contains("wait..."); + assertThat(out).doesNotContain("–").doesNotContain("…"); + } + + @Test + void codeStaysVerbatimEvenWithSmartPunctuationOn() throws Exception { + MarkdownComposer composer = MarkdownComposer.builder().smartPunctuation(true).build(); + + String out = text(composer, "Run `a --flag \"x\"` and:\n\n```\nb --opt 'y'...\n```\n"); + + assertThat(out).contains("--flag").contains("--opt"); + assertThat(out).contains("'y'..."); + assertThat(out).doesNotContain("–").doesNotContain("—"); + } + + @Test + void formattingInsideSmartQuotesSurvives() throws Exception { + MarkdownComposer composer = MarkdownComposer.builder().smartPunctuation(true).build(); + + String out = text(composer, "\"a **bold** word\""); + + // The quotes wrap mapped children (bold still renders); marks are curly. + assertThat(out).contains("“").contains("bold").contains("”"); + } + + @Test + void apostrophesCurlAndGuillemetsConvert() throws Exception { + // The unmatched-single-quote (apostrophe) path is distinct from paired quotes, + // and <> converts to guillemets. + MarkdownComposer composer = MarkdownComposer.builder().smartPunctuation(true).build(); + + String out = text(composer, "It don't matter <> here"); + + assertThat(out).contains("don’t").contains("«much»"); + assertThat(out).doesNotContain("&"); // no entity text ever leaks into the PDF + } + + @Test + void replacementsApplyInsideCustomBlocksToo() throws Exception { + // The ::: segmented parse must use the same flag-configured parser instance. + MarkdownComposer composer = MarkdownComposer.builder().smartPunctuation(true).build(); + + String out = text(composer, """ + :::note + range 3--7 wait... + ::: + + outside 1--2 + """); + + assertThat(out).contains("3–7").contains("…").contains("1–2"); + assertThat(out).doesNotContain("--"); + } + + @Test + void boldInsideSmartQuotesSurvivesAtTheModelLevel() { + // Model-level proof (extraction can't distinguish bold): TextRun(curly-open) + + // StrongRun + ... + TextRun(curly-close). + MarkdownComposer composer = MarkdownComposer.builder().smartPunctuation(true).build(); + + var blocks = composer.render("\"a **bold** word\"").document().blocks(); + var paragraph = (ParagraphNode) blocks.get(0); + var runs = paragraph.content(); + + assertThat(runs.get(0)).isInstanceOfSatisfying(TextRun.class, + r -> assertThat(r.text()).isEqualTo("“")); + assertThat(runs).anyMatch(r -> r instanceof StrongRun); + assertThat(runs.get(runs.size() - 1)).isInstanceOfSatisfying(TextRun.class, + r -> assertThat(r.text()).isEqualTo("”")); + } +}