DOC-6951 Publish redirects so a moved page is not read as a deletion

Makefile

@@ -30,8 +30,16 @@ json_transform: hugo
 	@echo "Transforming JSON files for RAG..."
 	@npx tsx build/transform_json_sections.ts
 
+# Tombstones need public/redirects.json, which hugo renders, and must run before
+# ndjson so that generate_ndjson.py can filter them back out of the feed. They are
+# written after json_transform only for tidiness -- the transform skips a record
+# that has a page_type and no content, so either order is safe.
+redirect_tombstones: json_transform
+	@echo "Writing redirect tombstones..."
+	@python3 build/write_redirect_tombstones.py public
+
 # ndjson requires json_transform to have processed the JSON files
-ndjson: json_transform
+ndjson: redirect_tombstones
 	@echo "Generating NDJSON feed..."
 	@python3 build/generate_ndjson.py
 	@echo "Compressing NDJSON feed..."

build/generate_ndjson.py

@@ -32,6 +32,15 @@ def load_and_validate_json_files(public_dir: Path) -> list[tuple[Path, dict]]:
         try:
             with open(json_file, 'r', encoding='utf-8') as f:
                 data = json.load(f)
+                # Redirect tombstones live at a moved page's old URL and share the
+                # index.json name, so rglob picks them up. They are deliberately
+                # kept out of the feed: they would add roughly 1,100 near-empty
+                # records to 2,600 real ones and make every "how many documents"
+                # figure ambiguous. Consumers resolve a moved URL either per-URL at
+                # its own /index.json, or in bulk from /redirects.json. See
+                # DOC-6951.
+                if data.get('page_type') == 'moved':
+                    continue
                 # Check for our expected fields
                 if all(key in data for key in ['id', 'title', 'url']):
                     valid_files.append((json_file, data))

build/transform_json_sections.ts

@@ -49,21 +49,27 @@ interface CodeExample {
 interface PageJsonInput {
   // Emitted by the Hugo templates and carried through untouched. Declared so the
   // spread below preserves them explicitly rather than by accident: schema_version is
-  // what consumers gate re-parsing on, and since is the only version information the
-  // feed carries.
+  // what consumers gate re-parsing on, since is the only version information the
+  // feed carries, and aliases is how a consumer holding a record resolves a stale
+  // URL without fetching the redirect map.
   schema_version?: number;
   id: string;
   title: string;
   url: string;
   summary: string;
   since?: string;
+  aliases?: string[];
   content?: string;
   tags: string[];
   last_updated: string;
   children?: unknown[];
 }
 
-type PageType = 'content' | 'index';
+// 'moved' is written by the tombstone pass at a moved page's old URL, and carries
+// only url and moved_to -- no sections, examples or content_hash. A consumer must
+// switch on page_type before assuming the documented content shape. Added in
+// schema_version 2 (DOC-6951).
+type PageType = 'content' | 'index' | 'moved';
 
 interface PageJsonOutput {
   schema_version?: number;
@@ -72,6 +78,7 @@ interface PageJsonOutput {
   url: string;
   summary: string;
   since?: string;
+  aliases?: string[];
   page_type: PageType;
   content_hash?: string;
   tags: string[];

build/write_redirect_tombstones.py

@@ -0,0 +1,170 @@
+"""Write a JSON tombstone beside every alias stub Hugo emitted.
+
+Hugo generates alias stubs for the **HTML output format only**. So a moved page's
+old URL serves a 200 meta-refresh page, while its ``/index.json`` and
+``/index.html.md`` both 404 -- verified on three independent correctly-aliased
+moves and reproduced against Hugo 0.143.1.
+
+That matters because ``content/ai-agent-resources.md`` tells consumers to find a
+page's JSON by appending ``/index.json`` to its URL. Following that instruction on
+a page that moved returns 404, so the move is indistinguishable from a deletion --
+and this happens for *every* move, including the ones we alias correctly. Fixing
+alias coverage does not fix it; this does.
+
+At each of those URLs we now publish a minimal record:
+
+    {"schema_version": 2, "page_type": "moved",
+     "id": "develop/ai/agent-memory",
+     "url": "https://redis.io/docs/latest/develop/ai/agent-memory/",
+     "moved_to": "https://redis.io/docs/latest/develop/ai/context-engine/agent-memory/"}
+
+``page_type: "moved"`` is a new value in that vocabulary and ``moved_to`` a new
+field, so this is a record-shape change and bumps ``aiSchemaVersion`` to 2. A
+consumer must switch on ``page_type`` before assuming the documented content shape:
+a tombstone deliberately has no ``sections``, ``examples`` or ``content_hash``.
+
+Reads ``public/redirects.json`` -- the map Hugo renders from the same ``.Aliases``
+data it uses for the stubs -- rather than parsing stub HTML, so the tombstones
+cannot disagree with the map or with the site.
+
+Two things it will not do, both load-bearing:
+
+- **Never overwrite a real page's ``index.json``.** An alias whose path a real page
+  occupies gets no tombstone; Hugo does not emit a stub there either, and clobbering
+  a real record would be far worse than a 404. A file that is recognisably one of
+  our own tombstones *is* rewritten, so an incremental build cannot keep serving a
+  ``moved_to`` that has since changed, and one the map no longer names is removed.
+- **Only write where a stub exists.** An alias declared on a draft, or one Hugo
+  dropped because the URL was taken, produces no stub and so gets no tombstone.
+
+See DOC-6951.
+"""
+
+import argparse
+import json
+import logging
+import os
+import sys
+
+logger = logging.getLogger("write_redirect_tombstones")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("public_dir", nargs="?", default="public",
+                        help="path to the rendered site (default: public)")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="report what would be written without writing it")
+    return parser.parse_args()
+
+
+def main() -> int:
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+    args = parse_args()
+
+    map_path = os.path.join(args.public_dir, "redirects.json")
+    if not os.path.isfile(map_path):
+        logger.error("write_redirect_tombstones: %s not found. Run hugo first; the "
+                     "map is rendered by layouts/index.redirects.json.", map_path)
+        return 1
+
+    with open(map_path, encoding="utf-8") as handle:
+        redirect_map = json.load(handle)
+
+    schema_version = redirect_map.get("schema_version")
+    base_url = (redirect_map.get("base_url") or "").rstrip("/")
+    entries = redirect_map.get("redirects") or []
+
+    def is_tombstone(path: str) -> bool:
+        """True if this index.json is one of ours rather than a real page record."""
+        try:
+            with open(path, encoding="utf-8") as existing:
+                return json.load(existing).get("page_type") == "moved"
+        except (OSError, json.JSONDecodeError):
+            return False
+
+    written = refreshed = no_stub = occupied = 0
+    expected: set[str] = set()
+    for entry in entries:
+        source = (entry.get("from") or "").strip()
+        target = (entry.get("to") or "").strip()
+        if not source or not target:
+            continue
+
+        rel = source.lstrip("/")
+        directory = (os.path.join(args.public_dir, *rel.split("/")) if rel
+                     else args.public_dir)
+
+        # Only where Hugo actually emitted a stub. A missing directory or missing
+        # index.html means the alias was dropped -- the URL was already taken, or
+        # it was declared on a draft -- and inventing a record there would publish
+        # a JSON document at a URL that serves no page.
+        if not os.path.isfile(os.path.join(directory, "index.html")):
+            no_stub += 1
+            continue
+
+        tombstone_path = os.path.join(directory, "index.json")
+        already = os.path.exists(tombstone_path)
+        if already and not is_tombstone(tombstone_path):
+            # A real page lives here. Never clobber a real record.
+            occupied += 1
+            continue
+        expected.add(os.path.realpath(tombstone_path))
+
+        record = {
+            "schema_version": schema_version,
+            "id": rel.rstrip("/"),
+            "title": "Moved",
+            "url": f"{base_url}/{rel}/" if rel else f"{base_url}/",
+            "page_type": "moved",
+            "moved_to": target,
+        }
+        if not args.dry_run:
+            with open(tombstone_path, "w", encoding="utf-8") as handle:
+                json.dump(record, handle, indent=2)
+                handle.write("\n")
+        if already:
+            # One of ours from a previous run. Rewritten rather than left alone, or
+            # an incremental build would keep serving a moved_to that has since
+            # changed. CI always builds into a fresh tree, so this only shows up
+            # locally -- but "correct because CI starts clean" is not correct.
+            refreshed += 1
+        else:
+            written += 1
+
+    # Sweep tombstones that no longer belong: the alias was removed, or it became
+    # ambiguous and is now published as a candidate list instead. Only files that
+    # are recognisably ours are ever removed, and only when the map no longer names
+    # them, so a real page record can never be caught by this.
+    removed = 0
+    for root, _dirs, files in os.walk(args.public_dir):
+        if "index.json" not in files:
+            continue
+        path = os.path.join(root, "index.json")
+        if os.path.realpath(path) in expected:
+            continue
+        if not is_tombstone(path):
+            continue
+        if not args.dry_run:
+            os.remove(path)
+        removed += 1
+
+    verb = "would be written" if args.dry_run else "written"
+    logger.info("write_redirect_tombstones: %d tombstone(s) %s from %d map entries.",
+                written, verb, len(entries))
+    if refreshed:
+        logger.info("  %d existing tombstone(s) %s.", refreshed,
+                    "would be refreshed" if args.dry_run else "refreshed")
+    if removed:
+        logger.info("  %d obsolete tombstone(s) %s -- no longer in the map.", removed,
+                    "would be removed" if args.dry_run else "removed")
+    if no_stub:
+        logger.info("  %d alias(es) had no stub, so were skipped -- the URL is taken "
+                    "by a real page, or the alias is on a draft.", no_stub)
+    if occupied:
+        logger.info("  %d had an index.json already and were left alone.", occupied)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

config.toml

@@ -71,7 +71,14 @@ gitHubRepo = "https://github.com/redis/docs"
 # content is a second content hash, which they will learn to ignore.
 #
 # Changing what a field *contains* is not a shape change and does not bump this.
-aiSchemaVersion = 1
+#
+# History:
+#   1 -- initial: sections, examples, content_hash, page_type, id, since (DOC-6939)
+#   2 -- adds `aliases` to page records, and a new `page_type` value, "moved", for
+#        the redirect tombstones published at a moved page's old URL. A parser
+#        switching on page_type now meets a value it has not seen, and those
+#        records carry a `moved_to` field, so this is a shape change (DOC-6951).
+aiSchemaVersion = 2
 
 # Display and sort order for client examples
 clientsExamples = ["Python", "Node.js", "ioredis", "Java-Sync", "Lettuce-Sync", "Java-Async", "Java-Reactive", "Go", "C", "C#-Sync (NRedisStack)", "C#-Async (NRedisStack)", "C#-Sync (SE.Redis)", "C#-Async (SE.Redis)", "RedisVL", "PHP", "Ruby", "Rust-Sync", "Rust-Async"]
@@ -149,7 +156,22 @@ rdi_current_version = "1.19.0"
     mediaType = "application/json"
     isPlainText = true
 
+  # A site-wide map of every alias to the page it resolves to, published once on
+  # the home page as /redirects.json. Rendered from `.Aliases` -- the same data
+  # Hugo uses to emit its own redirect stubs -- so the map and the site's actual
+  # behavior cannot drift, and there is no generated file to keep in step.
+  [outputFormats.redirects]
+    name = "redirects"
+    baseName = "redirects"
+    mediaType = "application/json"
+    isPlainText = true
+
 # Comment out if you don't want the "print entire section" link enabled.
 [outputs]
 section = ["HTML", "RSS", "Markdown", "JSON"]
-page = ["HTML", "Markdown", "JSON"]
+page = ["HTML", "Markdown", "JSON"]
+# The home page had no entry, so it was on Hugo's default of HTML and RSS. That
+# default is reproduced here deliberately: adding Markdown or JSON would publish a
+# new record for the site root, which is one of the pages the feed excludes today.
+# Only the redirects map is new.
+home = ["HTML", "RSS", "redirects"]

content/ai-agent-resources.md

@@ -68,14 +68,86 @@ Two consequences worth knowing if you diff the feed against the sitemap:
 ### Schema version
 
 Every record carries a `schema_version` integer, and the same value appears in the
-`json metadata` block of the Markdown output. It is currently **1**.
+`json metadata` block of the Markdown output. It is currently **2**.
 
 It increments only when the **shape** of a record changes: a field added, removed or
 renamed, or a new value entering the [role vocabulary](#section-roles). It does **not**
 change when page content changes, and it does not change when the value inside a field
 changes without the field itself changing. Use `content_hash` to detect content changes;
 use `schema_version` to detect when your parser might need attention.
 
+Version 2 added the `aliases` field to page records, and a new `page_type` value,
+`moved`, for the [redirect records](#pages-that-have-moved) served at a moved page's
+old URL. Version 1 was the initial shape: `sections`, `examples`, `content_hash`,
+`page_type`, `id` and `since`.
+
+## Pages that have moved
+
+When a page moves, its old URL keeps working — but until now it served only an HTML
+redirect, so appending `/index.json` to it returned 404 and a move was
+indistinguishable from a deletion. Two things now fix that.
+
+### A redirect record at the old URL
+
+Appending `/index.json` to a moved page's URL returns a record with
+`page_type` set to `moved`:
+
+```json
+{
+  "schema_version": 2,
+  "id": "develop/ai/agent-memory",
+  "title": "Moved",
+  "url": "https://redis.io/docs/latest/develop/ai/agent-memory/",
+  "page_type": "moved",
+  "moved_to": "https://redis.io/docs/latest/develop/ai/context-engine/agent-memory/"
+}
+```
+
+**Check `page_type` before assuming the shape of a record.** A `moved` record
+deliberately carries no `sections`, `examples` or `content_hash` — there is no content
+at that URL, only a pointer. Fetch `moved_to` to get the page itself.
+
+These records are **not** included in `docs.ndjson`. The feed is one record per
+documentation page, and adding roughly a thousand pointer records would make any count
+of the corpus ambiguous. Use the map below if you need them in bulk.
+
+### A map of every redirect
+
+`https://redis.io/docs/latest/redirects.json` lists every alias the site publishes and
+the page it resolves to:
+
+```json
+{
+  "schema_version": 2,
+  "base_url": "https://redis.io/docs/latest/",
+  "generated": "2026-08-07T15:00:00Z",
+  "count": 771,
+  "ambiguous_count": 12,
+  "redirects": [
+    {"from": "/develop/ai/langcache", "to": "https://redis.io/docs/latest/develop/ai/context-engine/langcache/"}
+  ],
+  "ambiguous": [
+    {"from": "/develop/use/pipelining", "candidates": ["https://redis.io/docs/latest/develop/using-commands/", "https://redis.io/docs/latest/develop/using-commands/pipelining/"]}
+  ]
+}
+```
+
+Three things worth knowing before you rely on it:
+
+- `from` is normalized to a leading slash and no trailing slash. `to` is absolute,
+  matching the `url` field on page records.
+- **It is an alias map, not a move log.** Many entries are vanity or legacy paths that
+  were never a page's location, and there is no date, because the source data does not
+  record when a page moved.
+- `ambiguous` holds the keys that more than one page claims, with every candidate
+  listed. We publish them separately rather than picking one, because the site itself
+  resolves those arbitrarily — so any single answer we gave you would sometimes
+  disagree with what you would actually be served. Treat an `ambiguous` key as
+  unresolved.
+
+The map covers the current version of the documentation. Version-specific
+documentation is outside both machine-readable feeds.
+
 ### JSON schema
 
 Each document contains:

layouts/_default/section.json

@@ -27,7 +27,8 @@
   "title": {{ .Title | jsonify }},
   "url": {{ .Permalink | jsonify }},
   "summary": {{ $summary | jsonify }},{{ with .Params.since }}
-  "since": {{ . | jsonify }},{{ end }}
+  "since": {{ . | jsonify }},{{ end }}{{ with .Aliases }}
+  "aliases": {{ . | jsonify }},{{ end }}
   "content": {{ $content | jsonify }},
   "tags": {{ $tags | jsonify }},
   "last_updated": {{ $lastUpdated | jsonify }},

layouts/_default/single.json

@@ -16,7 +16,8 @@
   "title": {{ .Title | jsonify }},
   "url": {{ .Permalink | jsonify }},
   "summary": {{ $summary | jsonify }},{{ with .Params.since }}
-  "since": {{ . | jsonify }},{{ end }}
+  "since": {{ . | jsonify }},{{ end }}{{ with .Aliases }}
+  "aliases": {{ . | jsonify }},{{ end }}
   "content": {{ $content | jsonify }},
   "tags": {{ $tags | jsonify }},
   "last_updated": {{ $lastUpdated | jsonify }}