diff --git a/Cargo.toml b/Cargo.toml index 704c090..770e9a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ codegen-units = 1 [dependencies] html-escape = ">=0.2" pyo3 = ">=0.28" +unicode-properties = "0.1" unicode-width = "0.2" [features] diff --git a/src/inline.rs b/src/inline.rs index 82b99e9..aac69be 100644 --- a/src/inline.rs +++ b/src/inline.rs @@ -6,6 +6,7 @@ use crate::entity::decode_entities as decode_html_entities; use crate::tagfilter::tagfilter_html; use crate::{MathMode, Options}; use std::collections::{HashMap, HashSet}; +use unicode_properties::{GeneralCategoryGroup, UnicodeGeneralCategory}; const ESCAPED_AMP: char = '\u{E000}'; @@ -601,11 +602,16 @@ struct Delimiter { active: bool, } +// Spec "punctuation character": ASCII punctuation or Unicode general category P. +fn is_flanking_punct(ch: char) -> bool { + ch.is_ascii_punctuation() || ch.general_category_group() == GeneralCategoryGroup::Punctuation +} + fn delimiter_run_flags(ch: char, before: char, after: char) -> (bool, bool) { let before_ws = before == '\0' || before.is_whitespace(); let after_ws = after == '\0' || after.is_whitespace(); - let before_punct = before.is_ascii_punctuation(); - let after_punct = after.is_ascii_punctuation(); + let before_punct = is_flanking_punct(before); + let after_punct = is_flanking_punct(after); let left = !after_ws && (!after_punct || before_ws || before_punct); let right = !before_ws && (!before_punct || after_ws || after_punct); match ch { diff --git a/tests/test_focused.py b/tests/test_focused.py index 50a59c1..5cd33a0 100644 --- a/tests/test_focused.py +++ b/tests/test_focused.py @@ -12,6 +12,12 @@ def test_dialect_fixture(): expected = (FIX / "dialect.xhtml").read_text() assert _norm(to_xhtml(md)) == _norm(expected) +def test_flanking_treats_unicode_punctuation_as_punctuation(): + html = to_xhtml("(“***{{company_common_name}}***”)") + assert "“{{company_common_name}}”" in html + html = to_xhtml("“__{{x}}__”") + assert "“{{x}}”" in html + def test_default_math_mode_is_brackets(): html = to_xhtml("\\(y\\)\n\n\\[\nx^2\n\\]\n\n$x$") assert 'y' in html