diff --git a/.github/workflows/lintrunner.yml b/.github/workflows/lintrunner.yml index fbfc48e9944..11144209f24 100644 --- a/.github/workflows/lintrunner.yml +++ b/.github/workflows/lintrunner.yml @@ -23,6 +23,9 @@ jobs: with: python-version: '3.12' + - name: Test local linter adapters + run: python3 -m unittest discover -s tools/linter/tests -p 'test_*.py' + - name: Install Lintrunner run: | pip install lintrunner==0.12.5 diff --git a/.lintrunner.toml b/.lintrunner.toml index 94dcc437f73..87bb917a86e 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -2,6 +2,21 @@ merge_base_with = "origin/main" # 4805a6ead6f1e7f32351056e2602be4e908f69b7 is from pytorch/pytorch main branch 2025-07-16 +[[linter]] +code = 'TUTORIAL_MARKUP' +include_patterns = [ + 'advanced_source/**/*.py', + 'beginner_source/**/*.py', + 'intermediate_source/**/*.py', + 'recipes_source/**/*.py', + 'unstable_source/**/*.py', +] +command = [ + 'python3', + 'tools/linter/adapters/tutorial_markup_linter.py', + '@{{PATHSFILE}}', +] + [[linter]] code = 'SPACES' include_patterns = ['**'] diff --git a/tools/linter/adapters/tutorial_markup_linter.py b/tools/linter/adapters/tutorial_markup_linter.py new file mode 100644 index 00000000000..8bc879e1a51 --- /dev/null +++ b/tools/linter/adapters/tutorial_markup_linter.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +"""Lint tutorial prose for lists that break during notebook conversion. + +Sphinx-Gallery reads prose from module docstrings and from comment blocks that +follow a gallery separator. That prose is parsed as reStructuredText for the +HTML documentation, but is converted to Markdown for generated notebooks. +Some list layouts accepted by reStructuredText are interpreted differently by +Pandoc and produce malformed Markdown cells. This linter detects those +layouts in the source, before a tutorial is built. +""" + +from __future__ import annotations + +import argparse +import ast +import json +import re +from enum import Enum +from pathlib import Path +from typing import Iterable, NamedTuple, Sequence + + +LINTER_CODE = "TUTORIAL_MARKUP" +GALLERY_SEPARATOR = re.compile(r"^#{20,}\s*$") +LIST_MARKER = re.compile( + r"^(?P *)(?P[-+*]|\d+[.)]|[A-Za-z][.)])\s+(?P.*)$" +) +LIST_TABLE_ROW = re.compile(r"^\s*\*\s+-(?:\s|$)") +SECTION_ADORNMENT = re.compile(r"^\s*([!\"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~])\1{2,}\s*$") + + +class LintSeverity(str, Enum): + ERROR = "error" + WARNING = "warning" + ADVICE = "advice" + DISABLED = "disabled" + + +class LintMessage(NamedTuple): + path: str | None + line: int | None + char: int | None + code: str + severity: LintSeverity + name: str + original: str | None + replacement: str | None + description: str | None + + +class ProseLine(NamedTuple): + text: str + source_line: int + + +def _module_docstring(source: str) -> list[ProseLine]: + """Return a module docstring without normalizing meaningful indentation.""" + try: + module = ast.parse(source) + except SyntaxError: + return [] + + if not module.body: + return [] + expression = module.body[0] + if not ( + isinstance(expression, ast.Expr) + and isinstance(expression.value, ast.Constant) + and isinstance(expression.value.value, str) + ): + return [] + + value = expression.value.value + return [ + ProseLine(text, expression.lineno + offset) + for offset, text in enumerate(value.splitlines()) + ] + + +def _comment_text(line: str) -> str: + if line.startswith("# "): + return line[2:] + return line[1:] + + +def extract_prose_blocks(source: str) -> list[list[ProseLine]]: + """Extract the source regions Sphinx-Gallery treats as narrative prose.""" + source_lines = source.splitlines() + blocks: list[list[ProseLine]] = [] + + docstring = _module_docstring(source) + if docstring: + blocks.append(docstring) + + index = 0 + while index < len(source_lines): + if not GALLERY_SEPARATOR.fullmatch(source_lines[index]): + index += 1 + continue + + index += 1 + block: list[ProseLine] = [] + while index < len(source_lines) and source_lines[index].startswith("#"): + block.append(ProseLine(_comment_text(source_lines[index]), index + 1)) + index += 1 + if block: + blocks.append(block) + + return blocks + + +def _leading_spaces(line: str) -> int: + return len(line) - len(line.lstrip(" ")) + + +def _marker(line: str) -> re.Match[str] | None: + return LIST_MARKER.match(line) + + +def _has_same_indent_marker_since_blank( + lines: Sequence[ProseLine], index: int, indent: int +) -> bool: + for previous in reversed(lines[:index]): + if not previous.text.strip(): + return False + match = _marker(previous.text) + if match and len(match.group("indent")) == indent: + return True + return False + + +def _has_valid_parent_list( + lines: Sequence[ProseLine], index: int, indent: int +) -> bool: + """Recognize a conventionally indented child of an active list item.""" + for previous in reversed(lines[:index]): + if not previous.text.strip(): + return False + match = _marker(previous.text) + if not match: + continue + parent_indent = len(match.group("indent")) + if parent_indent >= indent: + continue + content_column = parent_indent + len(match.group("marker")) + 1 + return indent == content_column + return False + + +def _missing_blank_before_indented_list( + lines: Sequence[ProseLine], index: int +) -> bool: + match = _marker(lines[index].text) + if not match: + return False + + indent = len(match.group("indent")) + marker = match.group("marker") + if index == 0 or indent == 0 or marker.endswith(")"): + return False + + previous = lines[index - 1].text + if not previous.strip() or SECTION_ADORNMENT.fullmatch(previous): + return False + + # Later items in the same list do not need another separating blank line. + if _has_same_indent_marker_since_blank(lines, index, indent): + return False + + # A child list aligned to its parent's content is valid reStructuredText + # and converts cleanly. The broken examples use an arbitrary indent. + if _has_valid_parent_list(lines, index, indent): + return False + + # Do not confuse list-table cells with ordinary list items. + if LIST_TABLE_ROW.match(previous): + return False + + # An item aligned with the preceding directive content (for example an + # ``.. note::`` body) is already separated by the directive structure. + if indent > 0 and _leading_spaces(previous) == indent: + return False + + return True + + +def _bad_list_continuations(lines: Sequence[ProseLine]) -> Iterable[ProseLine]: + """Yield unindented continuation lines from blank-separated top-level lists.""" + index = 0 + while index < len(lines): + first_match = _marker(lines[index].text) + if not first_match or first_match.group("indent"): + index += 1 + continue + if index > 0 and lines[index - 1].text.strip(): + index += 1 + continue + + marker_indent = 0 + cursor = index + 1 + first_bad: ProseLine | None = None + has_next_item = False + while cursor < len(lines) and lines[cursor].text.strip(): + match = _marker(lines[cursor].text) + if match and len(match.group("indent")) == marker_indent: + has_next_item = True + elif not match and _leading_spaces(lines[cursor].text) <= marker_indent: + first_bad = first_bad or lines[cursor] + cursor += 1 + + if has_next_item and first_bad is not None: + yield first_bad + index = max(cursor, index + 1) + + +def lint_source(filename: str, source: str) -> list[LintMessage]: + messages: list[LintMessage] = [] + + for block in extract_prose_blocks(source): + for index, prose_line in enumerate(block): + if _missing_blank_before_indented_list(block, index): + messages.append( + LintMessage( + path=filename, + line=prose_line.source_line, + char=1, + code=LINTER_CODE, + severity=LintSeverity.ERROR, + name="list missing a preceding blank line", + original=None, + replacement=None, + description=( + "This indented list starts immediately after prose. " + "It renders as a list in the HTML tutorial, but Pandoc " + "can merge or mis-indent it in the generated notebook. " + "Add a blank narrative line before the list." + ), + ) + ) + + for prose_line in _bad_list_continuations(block): + messages.append( + LintMessage( + path=filename, + line=prose_line.source_line, + char=1, + code=LINTER_CODE, + severity=LintSeverity.ERROR, + name="unindented list continuation", + original=None, + replacement=None, + description=( + "This list item continues at the same indentation as the " + "list marker. Pandoc treats the continuation as ordinary " + "text in generated notebooks. Indent continuation lines " + "to the list item's content column." + ), + ) + ) + + return messages + + +def lint_file(filename: str) -> list[LintMessage]: + return lint_source(filename, Path(filename).read_text(encoding="utf-8")) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(fromfile_prefix_chars="@") + parser.add_argument("filenames", nargs="+") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + for filename in args.filenames: + for message in lint_file(filename): + print(json.dumps(message._asdict())) + + +if __name__ == "__main__": + main() diff --git a/tools/linter/tests/test_tutorial_markup_linter.py b/tools/linter/tests/test_tutorial_markup_linter.py new file mode 100644 index 00000000000..c04cda34e37 --- /dev/null +++ b/tools/linter/tests/test_tutorial_markup_linter.py @@ -0,0 +1,109 @@ +import textwrap +import unittest + +from tools.linter.adapters.tutorial_markup_linter import lint_source + + +def lint(prose: str): + source = ( + '"""\n' + "Example tutorial\n" + "================\n\n" + f"{textwrap.dedent(prose)}\n" + '"""\n' + ) + return lint_source("example_tutorial.py", source) + + +class TutorialMarkupLinterTest(unittest.TestCase): + def test_reports_indented_list_without_blank_line(self): + messages = lint( + """\ + Parameters: + - first value + - second value""" + ) + + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].name, "list missing a preceding blank line") + + def test_allows_indented_list_after_blank_line(self): + messages = lint( + """\ + Parameters: + + - first value + - second value""" + ) + + self.assertEqual(messages, []) + + def test_allows_conventionally_indented_nested_list(self): + messages = lint( + """\ + - parent item + - nested item + - another nested item""" + ) + + self.assertEqual(messages, []) + + def test_allows_list_table_cells(self): + messages = lint( + """\ + .. list-table:: + + * - Heading + - Value""" + ) + + self.assertEqual(messages, []) + + def test_allows_list_aligned_inside_directive(self): + messages = lint( + """\ + .. note:: + This is directive content. + * first item + * second item""" + ) + + self.assertEqual(messages, []) + + def test_reports_unindented_list_continuation(self): + messages = lint( + """\ + - first item starts here + but its continuation is not indented + - second item""" + ) + + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].name, "unindented list continuation") + + def test_allows_indented_list_continuation(self): + messages = lint( + """\ + - first item starts here + and its continuation is indented + - second item""" + ) + + self.assertEqual(messages, []) + + def test_ignores_comments_outside_gallery_prose(self): + source = textwrap.dedent( + '''\ + """Example tutorial""" + + # Implementation details: + # - this is a code comment, not narrative prose + value = 1 + ''' + ) + + self.assertEqual(lint_source("example_tutorial.py", source), []) + + +if __name__ == "__main__": + unittest.main()