feat(seo): add robots.txt and sitemap.xml

apps/diffshub/app/robots.ts

@@ -0,0 +1,14 @@
+import type { MetadataRoute } from 'next';
+
+import { SITE_ORIGIN } from '@/lib/site';
+
+// Crawling stays allowed even though the viewer routes are noindex: a crawler
+// has to fetch a page to see its noindex directive, so disallowing them here
+// would leave any already-indexed viewer URL stuck in the index. The home page
+// is the only thing in the sitemap.
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: [{ userAgent: '*', allow: '/' }],
+    sitemap: new URL('/sitemap.xml', SITE_ORIGIN).href,
+  };
+}

apps/diffshub/app/sitemap.ts

@@ -0,0 +1,17 @@
+import type { MetadataRoute } from 'next';
+
+import { SITE_ORIGIN } from '@/lib/site';
+
+// Only the home page. The catch-all viewer mirrors arbitrary upstream paths, so
+// there is no finite URL set to enumerate, and those pages render third-party
+// diffs client-side rather than content of our own worth indexing.
+export default function sitemap(): MetadataRoute.Sitemap {
+  return [
+    {
+      url: `${SITE_ORIGIN}/`,
+      lastModified: new Date(),
+      changeFrequency: 'weekly',
+      priority: 1,
+    },
+  ];
+}

apps/docs/app/robots.ts

@@ -0,0 +1,14 @@
+import type { MetadataRoute } from 'next';
+
+import { SITE_ORIGIN } from '@/lib/site-origin';
+
+// Both sites are fully public marketing/docs, so the only job here is to point
+// crawlers at the sitemap — without a robots.txt they get neither crawl
+// guidance nor a URL inventory, which matters because `/docs` is a single route
+// whose sections are hash anchors rather than followable links.
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: [{ userAgent: '*', allow: '/' }],
+    sitemap: new URL('/sitemap.xml', SITE_ORIGIN).href,
+  };
+}

apps/docs/app/sitemap.ts

@@ -0,0 +1,71 @@
+import type { MetadataRoute } from 'next';
+
+import type { ProductId } from '@/lib/product-config';
+import { SITE, SITE_ORIGIN } from '@/lib/site-origin';
+
+type SitemapEntry = MetadataRoute.Sitemap[number];
+
+interface SitemapRoute {
+  path: string;
+  priority: number;
+  changeFrequency: NonNullable<SitemapEntry['changeFrequency']>;
+}
+
+// Listed per site because `app/(diffs)/layout.tsx` and `app/(trees)/layout.tsx`
+// call notFound() for the other product's routes: advertising both sets would
+// point crawlers at guaranteed 404s. Priorities rank the marketing home page and
+// docs above the interactive demos, which are useful but not what we want
+// ranking for product queries.
+const ROUTES_BY_SITE: Record<ProductId, readonly SitemapRoute[]> = {
+  diffs: [
+    { path: '/', priority: 1, changeFrequency: 'weekly' },
+    { path: '/docs', priority: 0.9, changeFrequency: 'weekly' },
+    { path: '/edit', priority: 0.8, changeFrequency: 'monthly' },
+    { path: '/theme', priority: 0.7, changeFrequency: 'monthly' },
+    { path: '/theme/gallery', priority: 0.5, changeFrequency: 'monthly' },
+    { path: '/playground', priority: 0.5, changeFrequency: 'monthly' },
+    { path: '/edit/live', priority: 0.4, changeFrequency: 'monthly' },
+    { path: '/ssr', priority: 0.3, changeFrequency: 'monthly' },
+  ],
+  trees: [
+    { path: '/', priority: 1, changeFrequency: 'weekly' },
+    { path: '/docs', priority: 0.9, changeFrequency: 'weekly' },
+    { path: '/trees-dev', priority: 0.5, changeFrequency: 'monthly' },
+    { path: '/trees-dev/react', priority: 0.4, changeFrequency: 'monthly' },
+    { path: '/trees-dev/search', priority: 0.4, changeFrequency: 'monthly' },
+    {
+      path: '/trees-dev/git-status',
+      priority: 0.4,
+      changeFrequency: 'monthly',
+    },
+    { path: '/trees-dev/density', priority: 0.4, changeFrequency: 'monthly' },
+    {
+      path: '/trees-dev/item-customization',
+      priority: 0.4,
+      changeFrequency: 'monthly',
+    },
+    {
+      path: '/trees-dev/drag-and-drop',
+      priority: 0.4,
+      changeFrequency: 'monthly',
+    },
+    {
+      path: '/trees-dev/responsiveness',
+      priority: 0.4,
+      changeFrequency: 'monthly',
+    },
+  ],
+};
+
+export default function sitemap(): MetadataRoute.Sitemap {
+  // These routes are defined in code, not authored content, so "last modified"
+  // is really "last deployed" — one build timestamp for every entry.
+  const lastModified = new Date();
+
+  return ROUTES_BY_SITE[SITE].map(({ path, priority, changeFrequency }) => ({
+    url: new URL(path, SITE_ORIGIN).href,
+    lastModified,
+    changeFrequency,
+    priority,
+  }));
+}

apps/docs/lib/site-origin.ts

@@ -0,0 +1,33 @@
+import type { ProductId } from './product-config';
+
+const PROD_ORIGIN_BY_SITE: Record<ProductId, string> = {
+  diffs: 'https://diffs.com',
+  trees: 'https://trees.software',
+};
+
+const DEV_PORT_BY_SITE: Record<ProductId, string> = {
+  diffs: '3690',
+  trees: '3691',
+};
+
+/**
+ * Which product this build serves. One codebase ships two sites, selected at
+ * build time, so anything that needs the active product (layout branding,
+ * robots.txt, sitemap.xml, canonical URLs) reads this rather than re-deriving
+ * it from the env var.
+ */
+export const SITE = (process.env.NEXT_PUBLIC_SITE ?? 'diffs') as ProductId;
+
+/** The public origin this site is deployed to. */
+export const PROD_ORIGIN = PROD_ORIGIN_BY_SITE[SITE];
+
+const isDev = process.env.NODE_ENV !== 'production';
+const DEV_PORT = process.env.PORT ?? DEV_PORT_BY_SITE[SITE];
+
+/**
+ * Origin that every absolute metadata URL resolves against: `metadataBase`,
+ * canonicals, `og:url`, and the sitemap/robots entries. In dev this points at
+ * localhost so OG previewers fetch in-progress assets and so a locally
+ * generated sitemap lists URLs you can actually click.
+ */
+export const SITE_ORIGIN = isDev ? `http://localhost:${DEV_PORT}` : PROD_ORIGIN;