Match the Pages Router table of contents

- Only collect top-level headings (run the extractor before the MaxWidth
  wrapper). Headings nested in <Note> etc. were never in the TOC before.
- Compile TOC entries from the already-processed mdast heading children
  instead of re-parsing the heading source. Re-parsing dropped smartypants
  (straight quotes on 43 pages) and turned headings like "1. Install" into
  an <ol> inside the TOC link.

Co-Authored-By: Claude <noreply@anthropic.com>

Author: gaearon

src/utils/compileMDX.tsx

@@ -7,11 +7,12 @@
 
 import {Fragment} from 'react';
 import type {ReactNode} from 'react';
-import {compile, run} from '@mdx-js/mdx';
+import {compile, createProcessor, run} from '@mdx-js/mdx';
 import * as runtime from 'react/jsx-runtime';
 import matter from 'gray-matter';
 import remarkGfm from 'remark-gfm';
 import remarkFrontmatter from 'remark-frontmatter';
+import type {Root} from 'mdast';
 import {remarkPlugins} from '../../plugins/markdownToHtml';
 import {createMDXComponents} from 'components/MDX/MDXComponents';
 import type {LanguageItem} from 'components/MDX/LanguagesContext';
@@ -43,24 +44,33 @@ function compileOptions() {
       ...remarkPlugins,
       remarkGfm,
       remarkFrontmatter,
-      MaxWidthWrapperPlugin,
+      // Order matters: the TOC only includes top-level headings, so it has
+      // to be collected before they get wrapped into <MaxWidth> elements.
       TOCExtractorPlugin,
+      MaxWidthWrapperPlugin,
     ],
     rehypePlugins: [MetaAttributesPlugin],
     outputFormat: 'function-body' as const,
   };
 }
 
+// Compiles a heading's already-processed mdast children (so typography and
+// inline components match the heading itself) without re-parsing source
+// text, which would turn e.g. "1. Install" into an ordered list.
+const tocProcessor = createProcessor({outputFormat: 'function-body'});
+
 async function compileToc(toc: ExtractedTocItem[]) {
   return Promise.all(
-    toc.map(async ({source, ...item}): Promise<CompiledTocItem> => {
-      if (!source) {
+    toc.map(async ({children, ...item}): Promise<CompiledTocItem> => {
+      if (!children) {
         return item;
       }
-      const code = await compile(source, {
-        remarkPlugins: [remarkGfm],
-        outputFormat: 'function-body',
-      });
+      const tree: Root = {
+        type: 'root',
+        children: [{type: 'paragraph', children}],
+      };
+      const transformed = await tocProcessor.run(tree);
+      const code = tocProcessor.stringify(transformed as any);
       return {...item, code: String(code)};
     })
   );

src/utils/mdx/TOCExtractorPlugin.ts

@@ -1,45 +1,61 @@
-import type {Root} from 'mdast';
+import type {PhrasingContent, Root} from 'mdast';
 import type {VFile} from 'vfile';
-import visit from 'unist-util-visit';
 
 export interface ExtractedTocItem {
   url: string;
   depth: number;
-  source?: string;
+  /**
+   * Heading content as already-processed mdast (custom IDs, smartypants,
+   * etc. applied). Compiled separately by compileMDX so the TOC can render
+   * inline code and badges. Mutually exclusive with `text`.
+   */
+  children?: PhrasingContent[];
   text?: string;
 }
 
-function sourceForChildren(
-  source: string,
-  children: Array<{
-    position?: {start: {offset?: number}; end: {offset?: number}};
-  }>
-) {
-  const start = children[0]?.position?.start.offset;
-  const end = children.at(-1)?.position?.end.offset;
-  return start == null || end == null ? '' : source.slice(start, end).trim();
+// Heading children without the trailing custom ID comment expression
+// (`{/*custom-id*/}` in the source) and any whitespace that preceded it.
+function headingContent(children: PhrasingContent[]): PhrasingContent[] {
+  const content = [...children];
+  const last = content[content.length - 1] as {type: string} | undefined;
+  if (last?.type === 'mdxTextExpression') {
+    content.pop();
+  }
+  const lastText = content[content.length - 1];
+  if (lastText?.type === 'text') {
+    content[content.length - 1] = {
+      ...lastText,
+      value: lastText.value.replace(/\s+$/, ''),
+    };
+  }
+  return content;
 }
 
+/**
+ * Collects the table of contents from the top-level nodes of the document,
+ * matching the Pages Router build: headings nested inside other components
+ * (<Note>, <DeepDive>, ...) are not part of the TOC. Must run before
+ * MaxWidthWrapperPlugin, which moves those nodes into <MaxWidth> wrappers.
+ */
 export function TOCExtractorPlugin({maxDepth = 3} = {}) {
   return (tree: Root, file: VFile) => {
     const toc: ExtractedTocItem[] = [];
-    const source = String(file.value);
 
-    visit(tree, (node: any) => {
-      if (node.type === 'heading' && node.depth <= maxDepth) {
+    for (const node of tree.children as any[]) {
+      if (node.type === 'heading') {
         const id = node.data?.hProperties?.id;
-        if (id) {
+        if (node.depth <= maxDepth && id) {
           toc.push({
             url: `#${id}`,
             depth: node.depth,
-            source: sourceForChildren(source, node.children),
+            children: headingContent(node.children),
           });
         }
-        return;
+        continue;
       }
 
       if (node.type !== 'mdxJsxFlowElement') {
-        return;
+        continue;
       }
 
       if (node.name === 'Challenges' || node.name === 'Recap') {
@@ -59,7 +75,7 @@ export function TOCExtractorPlugin({maxDepth = 3} = {}) {
         const permalink = String(attributes.get('permalink') ?? 'team-member');
         toc.push({url: `#${permalink}`, depth: 3, text: name});
       }
-    });
+    }
 
     if (toc.length > 0) {
       toc.unshift({url: '#', depth: 2, text: 'Overview'});