Load Markdown content through the bundler

Reading src/content with `fs` inside 'use cache' meant that in development
an edited .md file kept serving stale content on a normal reload (only a
hard refresh bypassed the cache), and nothing triggered a refresh on save
now that next-remote-watch is gone.

Import the files via import.meta.glob with Turbopack's built-in raw-loader
instead. The content is part of the module graph, so saving a .md file
Fast Refreshes the open page in place, and production bundles the files
rather than depending on outputFileTracingIncludes.

Co-Authored-By: Claude <noreply@anthropic.com>

Author: gaearon

next.config.js

@@ -17,9 +17,6 @@ const nextConfig = {
   reactStrictMode: true,
   reactCompiler: true,
   cacheComponents: true,
-  outputFileTracingIncludes: {
-    '/*': ['./src/content/**/*.md'],
-  },
   serverExternalPackages: [
     '@babel/core',
     '@babel/plugin-transform-modules-commonjs',
@@ -31,6 +28,15 @@ const nextConfig = {
     'remark-frontmatter',
   ],
   turbopack: {
+    // Lets src/contentFiles.ts import the Markdown sources as strings via
+    // import.meta.glob, so content is bundled and hot-reloads in development.
+    // raw-loader is built into Turbopack; it doesn't need to be installed.
+    rules: {
+      '*.md': {
+        loaders: ['raw-loader'],
+        as: '*.js',
+      },
+    },
     resolveAlias: {
       'use-sync-external-store/shim': 'react',
       esquery: 'esquery/dist/esquery.min.js',

src/app/api/md/[...path]/route.ts

@@ -5,11 +5,10 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import fs from 'fs';
-import path from 'path';
 import {NextResponse} from 'next/server';
 import {cacheLife} from 'next/cache';
 import {collectAllContentPaths, isContentPageAvailable} from 'lib/collectPaths';
+import {readContentPage} from 'contentFiles';
 
 const FOOTER = `
 ---
@@ -29,8 +28,7 @@ export async function generateStaticParams() {
 }
 
 /**
- * Read a markdown file for the given URL segments. Cached so prerendered
- * `.md` endpoints don't re-read from disk per request. Returns null when no
+ * Read a markdown file for the given URL segments. Returns null when no
  * matching file exists.
  */
 async function readContentMarkdown(
@@ -45,18 +43,7 @@ async function readContentMarkdown(
   // Block /index.md URLs - use /foo.md instead of /foo/index.md
   if (filePath.endsWith('/index') || filePath === 'index') return null;
 
-  const candidates = [
-    path.join(process.cwd(), 'src/content', filePath + '.md'),
-    path.join(process.cwd(), 'src/content', filePath, 'index.md'),
-  ];
-  for (const fullPath of candidates) {
-    try {
-      return fs.readFileSync(/* turbopackIgnore: true */ fullPath, 'utf8');
-    } catch {
-      // Try next candidate
-    }
-  }
-  return null;
+  return readContentPage(filePath);
 }
 
 export async function GET(

src/contentFiles.ts

@@ -0,0 +1,54 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import 'server-only';
+
+// Every Markdown file under src/content, keyed by its path relative to that
+// folder (e.g. "learn/index.md"). Loading the content through the bundler
+// instead of `fs` makes it part of the module graph: edits trigger Fast
+// Refresh in development, and the files are bundled for production instead
+// of relying on output file tracing. The `*.md` -> raw-loader rule lives in
+// next.config.js.
+//
+// This file has to sit next to `content/`: Turbopack's import.meta.glob does
+// not match patterns that start with `../`.
+const contentModules = import.meta.glob('./content/**/*.md', {
+  import: 'default',
+}) as Record<string, () => Promise<string>>;
+
+const PREFIX = './content/';
+
+const contentFiles = new Map<string, () => Promise<string>>();
+for (const [key, load] of Object.entries(contentModules)) {
+  contentFiles.set(key.slice(PREFIX.length), load);
+}
+
+/** Relative paths of all Markdown files, e.g. ["learn/index.md", ...]. */
+export function listContentFiles(): string[] {
+  return Array.from(contentFiles.keys());
+}
+
+/** Read "learn/index.md" etc. Returns null when the file doesn't exist. */
+export async function readContentFile(
+  relativePath: string
+): Promise<string | null> {
+  const load = contentFiles.get(relativePath);
+  return load ? load() : null;
+}
+
+/**
+ * Resolve a route path ("learn/state") to `<path>.md` or `<path>/index.md`,
+ * mirroring the old Pages Router lookup.
+ */
+export async function readContentPage(
+  routePath: string
+): Promise<string | null> {
+  return (
+    (await readContentFile(routePath + '.md')) ??
+    (await readContentFile(routePath + '/index.md'))
+  );
+}

src/lib/collectPaths.ts

@@ -6,15 +6,8 @@
  */
 
 import 'server-only';
-import fs from 'fs';
-import path from 'path';
-import {promisify} from 'util';
-import {cacheLife} from 'next/cache';
+import {listContentFiles} from 'contentFiles';
 
-const readdir = promisify(fs.readdir);
-const stat = promisify(fs.stat);
-
-const ROOT = path.join(process.cwd(), 'src/content');
 const DEV_ONLY_PAGES = new Set(['learn/rsc-sandbox-test']);
 
 export function isContentPageAvailable(segments: string[]): boolean {
@@ -24,21 +17,10 @@ export function isContentPageAvailable(segments: string[]): boolean {
   );
 }
 
-async function getFiles(dir: string, base: string): Promise<string[]> {
-  const subdirs = await readdir(dir);
-  const files = await Promise.all(
-    subdirs.map(async (subdir) => {
-      const res = path.resolve(dir, subdir);
-      return (await stat(res)).isDirectory()
-        ? getFiles(res, base)
-        : res.slice(base.length + 1);
-    })
-  );
-  return files.flat().filter((file) => file.endsWith('.md'));
-}
-
+// 'foo/bar/baz.md' -> ['foo', 'bar', 'baz']
+// 'foo/bar/qux/index.md' -> ['foo', 'bar', 'qux']
 function getSegments(file: string): string[] {
-  const segments = file.slice(0, -3).replace(/\\/g, '/').split('/');
+  const segments = file.slice(0, -3).split('/');
   if (segments[segments.length - 1] === 'index') {
     segments.pop();
   }
@@ -54,13 +36,9 @@ function getSegments(file: string): string[] {
 export async function collectSectionPaths(
   section: string
 ): Promise<string[][]> {
-  'use cache';
-  cacheLife('max');
-  const dir = path.join(ROOT, section);
-  if (!fs.existsSync(dir)) return [];
-  const files = await getFiles(dir, dir);
-  return files
-    .map((file) => getSegments(file))
+  return listContentFiles()
+    .filter((file) => file.startsWith(section + '/'))
+    .map((file) => getSegments(file).slice(1))
     .filter((segments) => isContentPageAvailable([section, ...segments]));
 }
 
@@ -71,12 +49,9 @@ export async function collectSectionPaths(
  * statically prerender the markdown route handler.
  */
 export async function collectAllContentPaths(): Promise<string[][]> {
-  'use cache';
-  cacheLife('max');
-  const files = await getFiles(ROOT, ROOT);
   return (
-    files
-      .map((file) => getSegments(file))
+    listContentFiles()
+      .map(getSegments)
       // Drop the root `index.md` (-> []); `/index.md` isn't a served URL and an
       // empty catch-all param can't be prerendered.
       .filter((segments) => segments.length > 0)
@@ -91,12 +66,7 @@ export async function collectAllContentPaths(): Promise<string[][]> {
 export async function collectFlatSectionSlugs(
   section: string
 ): Promise<string[]> {
-  'use cache';
-  cacheLife('max');
-  const dir = path.join(ROOT, section);
-  if (!fs.existsSync(dir)) return [];
-  const entries = await readdir(dir);
-  return entries
-    .filter((name) => name.endsWith('.md') && name !== 'index.md')
-    .map((name) => name.slice(0, -3));
+  return (await collectSectionPaths(section))
+    .filter((segments) => segments.length === 1)
+    .map(([slug]) => slug);
 }

src/lib/loadErrorDecoderData.ts

@@ -6,12 +6,11 @@
  */
 
 import 'server-only';
-import fs from 'fs';
-import path from 'path';
 import {notFound} from 'next/navigation';
 import {cacheLife} from 'next/cache';
 import compileMDX from 'utils/compileMDX';
 import type {CompiledMDX} from 'utils/compileMDX';
+import {readContentFile} from 'contentFiles';
 
 export interface ErrorDecoderData extends CompiledMDX {
   errorCode: string | null;
@@ -38,13 +37,12 @@ async function compileErrorDecoderData(
 ): Promise<ErrorDecoderData> {
   'use cache';
   cacheLife('max');
-  const rootDir = path.join(process.cwd(), 'src/content/errors');
-  const targetPath = code || 'index';
-  let mdx: string;
-  try {
-    mdx = fs.readFileSync(path.join(rootDir, targetPath + '.md'), 'utf8');
-  } catch {
-    mdx = fs.readFileSync(path.join(rootDir, 'generic.md'), 'utf8');
+  // Use errors/<code>.md when it exists, otherwise fall back to generic.md.
+  const mdx =
+    (await readContentFile(`errors/${code || 'index'}.md`)) ??
+    (await readContentFile('errors/generic.md'));
+  if (mdx == null) {
+    throw new Error('Missing src/content/errors/generic.md');
   }
 
   const compiled = await compileMDX(mdx);

src/lib/readMarkdownPage.ts

@@ -6,17 +6,14 @@
  */
 
 import 'server-only';
-import fs from 'fs/promises';
-import path from 'path';
 import {cacheLife} from 'next/cache';
 import {isContentPageAvailable} from './collectPaths';
+import {readContentPage} from 'contentFiles';
 import compileMDX from 'utils/compileMDX';
 import type {CompiledMDX} from 'utils/compileMDX';
 
 export type PageData = CompiledMDX;
 
-const ROOT = path.join(process.cwd(), 'src/content');
-
 /**
  * Read and compile an MDX page from src/content. Resolves either
  * `<segments>.md` or `<segments>/index.md`. Returns null when neither exists.
@@ -36,18 +33,7 @@ export async function readMarkdownPage(
   cacheLife('max');
   if (!isContentPageAvailable(segments)) return null;
   const routePath = segments.join('/') || 'index';
-  let mdx: string | null = null;
-  for (const candidate of [
-    path.join(ROOT, routePath + '.md'),
-    path.join(ROOT, routePath, 'index.md'),
-  ]) {
-    try {
-      mdx = await fs.readFile(/* turbopackIgnore: true */ candidate, 'utf8');
-      break;
-    } catch {
-      // Try next candidate.
-    }
-  }
+  const mdx = await readContentPage(routePath);
   if (mdx == null) return null;
   const compiled = await compileMDX(mdx);
   if (routePath === 'index') {