feat(docs-site): SEO improvements — OG image, RSS feed, sitemap lastmod, structured data

.claude/skills/release/task6-docs.md

@@ -2,7 +2,9 @@
 
 *Skip if `$ARGUMENTS` is a beta release. Requires Task 2. Runs concurrently with Task 3.*
 
-The tag push triggers a docs deployment workflow that builds and publishes the versioned docs to `/icp-cli/X.Y/`. The `versions.json` PR must not be merged until that deployment succeeds, otherwise the root redirect will point to a path that does not exist yet.
+The tag push triggers a docs deployment workflow that builds and publishes the versioned docs to `/X.Y/` on the `docs-deployment` branch (served at `https://cli.internetcomputer.org/X.Y/`). The `versions.json` PR must not be merged until that deployment succeeds, otherwise the root redirect will point to a path that does not exist yet.
+
+Once the `versions.json` PR merges to `main`, the `publish-root-files` CI job runs automatically and copies `og-image.png`, `llms.txt`, `llms-full.txt`, and `feed.xml` from the new version's folder to the deployment root — no manual step needed.
 
 **1. Wait for the docs deployment triggered by the tag**
 ```bash

.github/workflows/docs.yml

@@ -39,6 +39,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
 
       - name: Setup Node
         uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
@@ -115,19 +117,75 @@ jobs:
 
           echo "LATEST_VERSION=${LATEST_VERSION}" >> $GITHUB_ENV
 
-      - name: Copy llms.txt from latest version on docs-deployment branch
+          # Generate robots.txt — allow only the latest version, block old versions.
+          # /main/ is disallowed unless it IS the latest version (no releases yet fallback).
+          {
+            echo "User-agent: *"
+            echo "Allow: /${LATEST_VERSION}/"
+            for version in $(jq -r '.versions[].version' docs-site/versions.json); do
+              if [[ "$version" != "$LATEST_VERSION" ]]; then
+                echo "Disallow: /${version}/"
+              fi
+            done
+            if [[ "$LATEST_VERSION" != "main" ]]; then echo "Disallow: /main/"; fi
+            echo ""
+            echo "Sitemap: ${PUBLIC_SITE}/sitemap.xml"
+            echo ""
+            echo "# LLM and AI agent discovery (llmstxt.org)"
+            echo "# ${PUBLIC_SITE}/llms.txt"
+            echo "# ${PUBLIC_SITE}/llms-full.txt"
+          } > root/robots.txt
+          echo "✅ Generated robots.txt (latest: ${LATEST_VERSION})"
+
+          # Generate root sitemap.xml pointing directly to the latest version's sitemap.
+          # Points to sitemap-0.xml (not sitemap-index.xml) to stay spec-compliant:
+          # a sitemapindex must reference sitemaps, not other sitemapindex files.
+          {
+            echo '<?xml version="1.0" encoding="UTF-8"?>'
+            echo '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+            echo '  <sitemap>'
+            echo "    <loc>${PUBLIC_SITE}/${LATEST_VERSION}/sitemap-0.xml</loc>"
+            echo '  </sitemap>'
+            echo '</sitemapindex>'
+          } > root/sitemap.xml
+          echo "✅ Generated sitemap.xml → /${LATEST_VERSION}/sitemap-0.xml"
+
+      - name: Copy files from latest version on docs-deployment branch
         run: |
-          # Fetch the llms.txt from the latest version's folder on docs-deployment.
-          # This file was deployed by publish-versioned-docs and is always in sync
-          # with the versioned .md endpoints it links to.
           git fetch origin docs-deployment --depth=1
+
+          # llms.txt — agent discovery index (versioned copy is always in sync with .md endpoints)
           if git show "origin/docs-deployment:${LATEST_VERSION}/llms.txt" > root/llms.txt 2>/dev/null; then
             echo "✅ Copied llms.txt from /${LATEST_VERSION}/llms.txt to root"
           else
             echo "⚠️  No llms.txt found at /${LATEST_VERSION}/llms.txt on docs-deployment — skipping"
             rm -f root/llms.txt
           fi
 
+          # llms-full.txt — full content dump for bulk ingestion / RAG pipelines
+          if git show "origin/docs-deployment:${LATEST_VERSION}/llms-full.txt" > root/llms-full.txt 2>/dev/null; then
+            echo "✅ Copied llms-full.txt from /${LATEST_VERSION}/llms-full.txt to root"
+          else
+            echo "⚠️  No llms-full.txt found at /${LATEST_VERSION}/llms-full.txt on docs-deployment — skipping"
+            rm -f root/llms-full.txt
+          fi
+
+          # og-image.png — social sharing preview image
+          if git show "origin/docs-deployment:${LATEST_VERSION}/og-image.png" > root/og-image.png 2>/dev/null; then
+            echo "✅ Copied og-image.png from /${LATEST_VERSION}/og-image.png to root"
+          else
+            echo "⚠️  No og-image.png found at /${LATEST_VERSION}/og-image.png on docs-deployment — skipping"
+            rm -f root/og-image.png
+          fi
+
+          # feed.xml — RSS feed
+          if git show "origin/docs-deployment:${LATEST_VERSION}/feed.xml" > root/feed.xml 2>/dev/null; then
+            echo "✅ Copied feed.xml from /${LATEST_VERSION}/feed.xml to root"
+          else
+            echo "⚠️  No feed.xml found at /${LATEST_VERSION}/feed.xml on docs-deployment — skipping"
+            rm -f root/feed.xml
+          fi
+
       - name: Prepend version navigation to root llms.txt
         if: hashFiles('root/llms.txt') != ''
         run: |
@@ -164,6 +222,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
 
       - name: Setup Node
         uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
@@ -203,6 +263,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
 
       - name: Setup Node
         uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0

docs-site/README.md

@@ -10,15 +10,19 @@ The documentation site is built with [Astro](https://astro.build/) and [Starligh
 
 ```
 docs-site/
-├── astro.config.mjs       # Starlight configuration (sidebar, theme)
+├── astro.config.mjs       # Starlight configuration (sidebar, theme, SEO meta tags, JSON-LD)
+├── versions.json          # Version registry (controls root redirect and robots.txt)
 ├── plugins/
-│   └── rehype-rewrite-links.mjs  # Rewrites .md links for Starlight's clean URLs
+│   ├── rehype-rewrite-links.mjs  # Rewrites .md links for Starlight's clean URLs
+│   └── astro-agent-docs.mjs     # Generates llms.txt, feed.xml, og-image.png, sitemap lastmod
 ├── src/
 │   ├── content.config.ts  # Content loader configuration
-│   ├── components/        # Custom Starlight component overrides
+│   ├── components/        # Custom Starlight component overrides (Footer, Banner, SiteTitle)
 │   ├── assets/            # Logo and static assets
 │   └── styles/            # DFINITY theme CSS
-├── public/                # Static files (favicon, etc.)
+├── public/
+│   ├── og-image.svg       # Source SVG for the social sharing image (converted to PNG at build time)
+│   └── ...                # Other static files (favicon, well-known, etc.)
 └── package.json           # Dependencies and scripts
 ```
 
@@ -35,6 +39,12 @@ docs-site/
 2. **Rehype plugin** (`plugins/rehype-rewrite-links.mjs`) strips `.md` extensions from relative links and adjusts paths for Astro's directory-based output
 3. **DFINITY theme CSS** is applied for consistent branding
 4. **Static HTML** is produced in `dist/`
+5. **`astro-agent-docs` plugin** runs in the `astro:build:done` hook and generates additional files:
+   - `llms.txt` — LLM/agent-friendly page index (agentdocsspec.com)
+   - `llms-full.txt` — full content dump for RAG pipelines
+   - `feed.xml` — RSS 2.0 feed with git-accurate publish dates per page
+   - `og-image.png` — social sharing preview image, rendered from `public/og-image.svg` via `@resvg/resvg-js`
+   - Sitemap `<lastmod>` injection — adds git commit dates to the Starlight-generated sitemap
 
 Source docs use `.md` extensions in links (GitHub-friendly), and the rehype plugin transforms them to clean URLs at build time.
 
@@ -93,6 +103,7 @@ Removes `dist/` and `.astro/` directories
 - `build` - Builds for production
 - `preview` - Previews production build locally
 - `clean` - Removes build artifacts (`dist/`, `.astro/`)
+- `test:versions` - Simulates the full multi-version production layout locally for testing the version switcher UI
 
 ## Deployment
 
@@ -107,10 +118,20 @@ The site is hosted on an IC asset canister and served at `https://cli.internetco
 
 ### Triggers
 
-- **Push to `main`**: Rebuilds `/main/` docs and root files (`index.html`, `versions.json`, IC config)
+- **Push to `main`**: Rebuilds `/main/` docs and root files (`index.html`, `versions.json`, `robots.txt`, `sitemap.xml`, IC config). Also copies `og-image.png`, `llms.txt`, `llms-full.txt`, and `feed.xml` from the latest versioned deployment to the root.
 - **Tags (`v*`)**: Builds versioned docs (e.g., `v0.2.0` → `/0.2/`)
 - **Branches (`docs/v*`)**: Updates versioned docs (e.g., `docs/v0.1` → `/0.1/`)
 
+### Root-level files
+
+Several files must live at the deployment root (not inside a versioned subfolder) to be discovered correctly:
+
+- **`robots.txt`** — generated dynamically by the CI `publish-root-files` job from `versions.json`; allows only the latest version's path, disallows old versions, and disallows `/main/` (except when no releases exist yet and `/main/` is the fallback). Never placed in versioned build output.
+- **`sitemap.xml`** — root sitemap index pointing directly to the latest version's `sitemap-0.xml` (spec-compliant: a sitemapindex must reference sitemaps, not other sitemapindex files); generated by `publish-root-files`.
+- **`og-image.png`** — the `og:image` meta tag always references `https://cli.internetcomputer.org/og-image.png`. The CI `publish-root-files` job copies it from the latest versioned build folder to root after each versioned deployment.
+- **`llms.txt` / `llms-full.txt`** — same pattern as `og-image.png`; `publish-root-files` prepends a version navigation header to the root `llms.txt`.
+- **`feed.xml`** — same pattern; copied from the latest versioned folder to root.
+
 ### Legacy redirect
 
 The old GitHub Pages site at `https://dfinity.github.io/icp-cli/` redirects all paths to `https://cli.internetcomputer.org/`.

docs-site/astro.config.mjs

@@ -4,9 +4,11 @@ import rehypeExternalLinks from 'rehype-external-links';
 import rehypeRewriteLinks from './plugins/rehype-rewrite-links.mjs';
 import agentDocs from './plugins/astro-agent-docs.mjs';
 
+const SITE = (process.env.PUBLIC_SITE || 'https://cli.internetcomputer.org').replace(/\/$/, '');
+
 // https://astro.build/config
 export default defineConfig({
-  site: process.env.PUBLIC_SITE,
+  site: SITE,
   // For versioned deployments: /0.1/, /0.2/, etc.
   // PUBLIC_BASE_PATH is set per-version in CI (e.g., /0.2/, /main/)
   base: process.env.PUBLIC_BASE_PATH || '/',
@@ -26,6 +28,7 @@ export default defineConfig({
       components: {
         SiteTitle: './src/components/SiteTitle.astro',
         Banner: './src/components/Banner.astro',
+        Footer: './src/components/Footer.astro',
       },
       head: [
         {
@@ -34,11 +37,63 @@ export default defineConfig({
           tag: 'link',
           attrs: {
             rel: 'help',
-            href: `${process.env.PUBLIC_BASE_PATH || '/'}llms.txt`,
+            href: '/llms.txt',
             type: 'text/plain',
             title: 'LLM-friendly documentation index',
           },
         },
+        {
+          tag: 'link',
+          attrs: {
+            rel: 'alternate',
+            type: 'application/rss+xml',
+            title: 'ICP CLI Documentation',
+            href: '/feed.xml',
+          },
+        },
+        {
+          tag: 'meta',
+          attrs: { name: 'robots', content: 'index, follow, max-image-preview:large' },
+        },
+        {
+          tag: 'meta',
+          attrs: { name: 'author', content: 'DFINITY Foundation' },
+        },
+        {
+          tag: 'meta',
+          attrs: { property: 'og:image', content: `${SITE}/og-image.png` },
+        },
+        {
+          tag: 'meta',
+          attrs: { property: 'og:image:alt', content: 'ICP CLI Documentation — Build on Internet Computer' },
+        },
+        {
+          tag: 'meta',
+          attrs: { name: 'twitter:image', content: `${SITE}/og-image.png` },
+        },
+        {
+          tag: 'script',
+          attrs: { type: 'application/ld+json' },
+          content: JSON.stringify({
+            '@context': 'https://schema.org',
+            '@graph': [
+              {
+                '@type': 'WebSite',
+                '@id': `${SITE}/#website`,
+                'name': 'ICP CLI',
+                'description': 'Command-line tool for developing and deploying applications on the Internet Computer Protocol (ICP)',
+                'url': SITE,
+                'publisher': { '@id': `${SITE}/#organization` },
+              },
+              {
+                '@type': 'Organization',
+                '@id': `${SITE}/#organization`,
+                'name': 'DFINITY Foundation',
+                'url': 'https://dfinity.org',
+              },
+            ],
+          }),
+        },
         {
           tag: 'script',
           attrs: {},