diff --git a/Makefile b/Makefile index 272eac0f24..5073aac866 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,16 @@ json_transform: hugo @echo "Transforming JSON files for RAG..." @npx tsx build/transform_json_sections.ts +# Tombstones need public/redirects.json, which hugo renders, and must run before +# ndjson so that generate_ndjson.py can filter them back out of the feed. They are +# written after json_transform only for tidiness -- the transform skips a record +# that has a page_type and no content, so either order is safe. +redirect_tombstones: json_transform + @echo "Writing redirect tombstones..." + @python3 build/write_redirect_tombstones.py public + # ndjson requires json_transform to have processed the JSON files -ndjson: json_transform +ndjson: redirect_tombstones @echo "Generating NDJSON feed..." @python3 build/generate_ndjson.py @echo "Compressing NDJSON feed..." diff --git a/build/generate_ndjson.py b/build/generate_ndjson.py index ab0c125698..0767493264 100644 --- a/build/generate_ndjson.py +++ b/build/generate_ndjson.py @@ -32,6 +32,15 @@ def load_and_validate_json_files(public_dir: Path) -> list[tuple[Path, dict]]: try: with open(json_file, 'r', encoding='utf-8') as f: data = json.load(f) + # Redirect tombstones live at a moved page's old URL and share the + # index.json name, so rglob picks them up. They are deliberately + # kept out of the feed: they would add roughly 1,100 near-empty + # records to 2,600 real ones and make every "how many documents" + # figure ambiguous. Consumers resolve a moved URL either per-URL at + # its own /index.json, or in bulk from /redirects.json. See + # DOC-6951. + if data.get('page_type') == 'moved': + continue # Check for our expected fields if all(key in data for key in ['id', 'title', 'url']): valid_files.append((json_file, data)) diff --git a/build/transform_json_sections.ts b/build/transform_json_sections.ts index 65ea758bad..1fb7776baa 100644 --- a/build/transform_json_sections.ts +++ b/build/transform_json_sections.ts @@ -49,21 +49,27 @@ interface CodeExample { interface PageJsonInput { // Emitted by the Hugo templates and carried through untouched. Declared so the // spread below preserves them explicitly rather than by accident: schema_version is - // what consumers gate re-parsing on, and since is the only version information the - // feed carries. + // what consumers gate re-parsing on, since is the only version information the + // feed carries, and aliases is how a consumer holding a record resolves a stale + // URL without fetching the redirect map. schema_version?: number; id: string; title: string; url: string; summary: string; since?: string; + aliases?: string[]; content?: string; tags: string[]; last_updated: string; children?: unknown[]; } -type PageType = 'content' | 'index'; +// 'moved' is written by the tombstone pass at a moved page's old URL, and carries +// only url and moved_to -- no sections, examples or content_hash. A consumer must +// switch on page_type before assuming the documented content shape. Added in +// schema_version 2 (DOC-6951). +type PageType = 'content' | 'index' | 'moved'; interface PageJsonOutput { schema_version?: number; @@ -72,6 +78,7 @@ interface PageJsonOutput { url: string; summary: string; since?: string; + aliases?: string[]; page_type: PageType; content_hash?: string; tags: string[]; diff --git a/build/write_redirect_tombstones.py b/build/write_redirect_tombstones.py new file mode 100644 index 0000000000..e380a38de5 --- /dev/null +++ b/build/write_redirect_tombstones.py @@ -0,0 +1,170 @@ +"""Write a JSON tombstone beside every alias stub Hugo emitted. + +Hugo generates alias stubs for the **HTML output format only**. So a moved page's +old URL serves a 200 meta-refresh page, while its ``/index.json`` and +``/index.html.md`` both 404 -- verified on three independent correctly-aliased +moves and reproduced against Hugo 0.143.1. + +That matters because ``content/ai-agent-resources.md`` tells consumers to find a +page's JSON by appending ``/index.json`` to its URL. Following that instruction on +a page that moved returns 404, so the move is indistinguishable from a deletion -- +and this happens for *every* move, including the ones we alias correctly. Fixing +alias coverage does not fix it; this does. + +At each of those URLs we now publish a minimal record: + + {"schema_version": 2, "page_type": "moved", + "id": "develop/ai/agent-memory", + "url": "https://redis.io/docs/latest/develop/ai/agent-memory/", + "moved_to": "https://redis.io/docs/latest/develop/ai/context-engine/agent-memory/"} + +``page_type: "moved"`` is a new value in that vocabulary and ``moved_to`` a new +field, so this is a record-shape change and bumps ``aiSchemaVersion`` to 2. A +consumer must switch on ``page_type`` before assuming the documented content shape: +a tombstone deliberately has no ``sections``, ``examples`` or ``content_hash``. + +Reads ``public/redirects.json`` -- the map Hugo renders from the same ``.Aliases`` +data it uses for the stubs -- rather than parsing stub HTML, so the tombstones +cannot disagree with the map or with the site. + +Two things it will not do, both load-bearing: + +- **Never overwrite a real page's ``index.json``.** An alias whose path a real page + occupies gets no tombstone; Hugo does not emit a stub there either, and clobbering + a real record would be far worse than a 404. A file that is recognisably one of + our own tombstones *is* rewritten, so an incremental build cannot keep serving a + ``moved_to`` that has since changed, and one the map no longer names is removed. +- **Only write where a stub exists.** An alias declared on a draft, or one Hugo + dropped because the URL was taken, produces no stub and so gets no tombstone. + +See DOC-6951. +""" + +import argparse +import json +import logging +import os +import sys + +logger = logging.getLogger("write_redirect_tombstones") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("public_dir", nargs="?", default="public", + help="path to the rendered site (default: public)") + parser.add_argument("--dry-run", action="store_true", + help="report what would be written without writing it") + return parser.parse_args() + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(message)s") + args = parse_args() + + map_path = os.path.join(args.public_dir, "redirects.json") + if not os.path.isfile(map_path): + logger.error("write_redirect_tombstones: %s not found. Run hugo first; the " + "map is rendered by layouts/index.redirects.json.", map_path) + return 1 + + with open(map_path, encoding="utf-8") as handle: + redirect_map = json.load(handle) + + schema_version = redirect_map.get("schema_version") + base_url = (redirect_map.get("base_url") or "").rstrip("/") + entries = redirect_map.get("redirects") or [] + + def is_tombstone(path: str) -> bool: + """True if this index.json is one of ours rather than a real page record.""" + try: + with open(path, encoding="utf-8") as existing: + return json.load(existing).get("page_type") == "moved" + except (OSError, json.JSONDecodeError): + return False + + written = refreshed = no_stub = occupied = 0 + expected: set[str] = set() + for entry in entries: + source = (entry.get("from") or "").strip() + target = (entry.get("to") or "").strip() + if not source or not target: + continue + + rel = source.lstrip("/") + directory = (os.path.join(args.public_dir, *rel.split("/")) if rel + else args.public_dir) + + # Only where Hugo actually emitted a stub. A missing directory or missing + # index.html means the alias was dropped -- the URL was already taken, or + # it was declared on a draft -- and inventing a record there would publish + # a JSON document at a URL that serves no page. + if not os.path.isfile(os.path.join(directory, "index.html")): + no_stub += 1 + continue + + tombstone_path = os.path.join(directory, "index.json") + already = os.path.exists(tombstone_path) + if already and not is_tombstone(tombstone_path): + # A real page lives here. Never clobber a real record. + occupied += 1 + continue + expected.add(os.path.realpath(tombstone_path)) + + record = { + "schema_version": schema_version, + "id": rel.rstrip("/"), + "title": "Moved", + "url": f"{base_url}/{rel}/" if rel else f"{base_url}/", + "page_type": "moved", + "moved_to": target, + } + if not args.dry_run: + with open(tombstone_path, "w", encoding="utf-8") as handle: + json.dump(record, handle, indent=2) + handle.write("\n") + if already: + # One of ours from a previous run. Rewritten rather than left alone, or + # an incremental build would keep serving a moved_to that has since + # changed. CI always builds into a fresh tree, so this only shows up + # locally -- but "correct because CI starts clean" is not correct. + refreshed += 1 + else: + written += 1 + + # Sweep tombstones that no longer belong: the alias was removed, or it became + # ambiguous and is now published as a candidate list instead. Only files that + # are recognisably ours are ever removed, and only when the map no longer names + # them, so a real page record can never be caught by this. + removed = 0 + for root, _dirs, files in os.walk(args.public_dir): + if "index.json" not in files: + continue + path = os.path.join(root, "index.json") + if os.path.realpath(path) in expected: + continue + if not is_tombstone(path): + continue + if not args.dry_run: + os.remove(path) + removed += 1 + + verb = "would be written" if args.dry_run else "written" + logger.info("write_redirect_tombstones: %d tombstone(s) %s from %d map entries.", + written, verb, len(entries)) + if refreshed: + logger.info(" %d existing tombstone(s) %s.", refreshed, + "would be refreshed" if args.dry_run else "refreshed") + if removed: + logger.info(" %d obsolete tombstone(s) %s -- no longer in the map.", removed, + "would be removed" if args.dry_run else "removed") + if no_stub: + logger.info(" %d alias(es) had no stub, so were skipped -- the URL is taken " + "by a real page, or the alias is on a draft.", no_stub) + if occupied: + logger.info(" %d had an index.json already and were left alone.", occupied) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/config.toml b/config.toml index 480907858b..fb347a5e7e 100644 --- a/config.toml +++ b/config.toml @@ -71,7 +71,14 @@ gitHubRepo = "https://github.com/redis/docs" # content is a second content hash, which they will learn to ignore. # # Changing what a field *contains* is not a shape change and does not bump this. -aiSchemaVersion = 1 +# +# History: +# 1 -- initial: sections, examples, content_hash, page_type, id, since (DOC-6939) +# 2 -- adds `aliases` to page records, and a new `page_type` value, "moved", for +# the redirect tombstones published at a moved page's old URL. A parser +# switching on page_type now meets a value it has not seen, and those +# records carry a `moved_to` field, so this is a shape change (DOC-6951). +aiSchemaVersion = 2 # Display and sort order for client examples clientsExamples = ["Python", "Node.js", "ioredis", "Java-Sync", "Lettuce-Sync", "Java-Async", "Java-Reactive", "Go", "C", "C#-Sync (NRedisStack)", "C#-Async (NRedisStack)", "C#-Sync (SE.Redis)", "C#-Async (SE.Redis)", "RedisVL", "PHP", "Ruby", "Rust-Sync", "Rust-Async"] @@ -149,7 +156,22 @@ rdi_current_version = "1.19.0" mediaType = "application/json" isPlainText = true + # A site-wide map of every alias to the page it resolves to, published once on + # the home page as /redirects.json. Rendered from `.Aliases` -- the same data + # Hugo uses to emit its own redirect stubs -- so the map and the site's actual + # behavior cannot drift, and there is no generated file to keep in step. + [outputFormats.redirects] + name = "redirects" + baseName = "redirects" + mediaType = "application/json" + isPlainText = true + # Comment out if you don't want the "print entire section" link enabled. [outputs] section = ["HTML", "RSS", "Markdown", "JSON"] -page = ["HTML", "Markdown", "JSON"] \ No newline at end of file +page = ["HTML", "Markdown", "JSON"] +# The home page had no entry, so it was on Hugo's default of HTML and RSS. That +# default is reproduced here deliberately: adding Markdown or JSON would publish a +# new record for the site root, which is one of the pages the feed excludes today. +# Only the redirects map is new. +home = ["HTML", "RSS", "redirects"] \ No newline at end of file diff --git a/content/ai-agent-resources.md b/content/ai-agent-resources.md index b5c579f26f..fb372dbc7a 100644 --- a/content/ai-agent-resources.md +++ b/content/ai-agent-resources.md @@ -68,7 +68,7 @@ Two consequences worth knowing if you diff the feed against the sitemap: ### Schema version Every record carries a `schema_version` integer, and the same value appears in the -`json metadata` block of the Markdown output. It is currently **1**. +`json metadata` block of the Markdown output. It is currently **2**. It increments only when the **shape** of a record changes: a field added, removed or renamed, or a new value entering the [role vocabulary](#section-roles). It does **not** @@ -76,6 +76,78 @@ change when page content changes, and it does not change when the value inside a changes without the field itself changing. Use `content_hash` to detect content changes; use `schema_version` to detect when your parser might need attention. +Version 2 added the `aliases` field to page records, and a new `page_type` value, +`moved`, for the [redirect records](#pages-that-have-moved) served at a moved page's +old URL. Version 1 was the initial shape: `sections`, `examples`, `content_hash`, +`page_type`, `id` and `since`. + +## Pages that have moved + +When a page moves, its old URL keeps working — but until now it served only an HTML +redirect, so appending `/index.json` to it returned 404 and a move was +indistinguishable from a deletion. Two things now fix that. + +### A redirect record at the old URL + +Appending `/index.json` to a moved page's URL returns a record with +`page_type` set to `moved`: + +```json +{ + "schema_version": 2, + "id": "develop/ai/agent-memory", + "title": "Moved", + "url": "https://redis.io/docs/latest/develop/ai/agent-memory/", + "page_type": "moved", + "moved_to": "https://redis.io/docs/latest/develop/ai/context-engine/agent-memory/" +} +``` + +**Check `page_type` before assuming the shape of a record.** A `moved` record +deliberately carries no `sections`, `examples` or `content_hash` — there is no content +at that URL, only a pointer. Fetch `moved_to` to get the page itself. + +These records are **not** included in `docs.ndjson`. The feed is one record per +documentation page, and adding roughly a thousand pointer records would make any count +of the corpus ambiguous. Use the map below if you need them in bulk. + +### A map of every redirect + +`https://redis.io/docs/latest/redirects.json` lists every alias the site publishes and +the page it resolves to: + +```json +{ + "schema_version": 2, + "base_url": "https://redis.io/docs/latest/", + "generated": "2026-08-07T15:00:00Z", + "count": 771, + "ambiguous_count": 12, + "redirects": [ + {"from": "/develop/ai/langcache", "to": "https://redis.io/docs/latest/develop/ai/context-engine/langcache/"} + ], + "ambiguous": [ + {"from": "/develop/use/pipelining", "candidates": ["https://redis.io/docs/latest/develop/using-commands/", "https://redis.io/docs/latest/develop/using-commands/pipelining/"]} + ] +} +``` + +Three things worth knowing before you rely on it: + +- `from` is normalized to a leading slash and no trailing slash. `to` is absolute, + matching the `url` field on page records. +- **It is an alias map, not a move log.** Many entries are vanity or legacy paths that + were never a page's location, and there is no date, because the source data does not + record when a page moved. +- `ambiguous` holds the keys that more than one page claims, with every candidate + listed. We publish them separately rather than picking one, because the site itself + resolves those arbitrarily — so any single answer we gave you would sometimes + disagree with what you would actually be served. Treat an `ambiguous` key as + unresolved. + +The map covers the current version of the documentation. Version-specific +documentation is outside both machine-readable feeds. + ### JSON schema Each document contains: diff --git a/layouts/_default/section.json b/layouts/_default/section.json index 95915770b7..13c7ec7995 100644 --- a/layouts/_default/section.json +++ b/layouts/_default/section.json @@ -27,7 +27,8 @@ "title": {{ .Title | jsonify }}, "url": {{ .Permalink | jsonify }}, "summary": {{ $summary | jsonify }},{{ with .Params.since }} - "since": {{ . | jsonify }},{{ end }} + "since": {{ . | jsonify }},{{ end }}{{ with .Aliases }} + "aliases": {{ . | jsonify }},{{ end }} "content": {{ $content | jsonify }}, "tags": {{ $tags | jsonify }}, "last_updated": {{ $lastUpdated | jsonify }}, diff --git a/layouts/_default/single.json b/layouts/_default/single.json index 30727f7fed..a25a4c7ecd 100644 --- a/layouts/_default/single.json +++ b/layouts/_default/single.json @@ -16,7 +16,8 @@ "title": {{ .Title | jsonify }}, "url": {{ .Permalink | jsonify }}, "summary": {{ $summary | jsonify }},{{ with .Params.since }} - "since": {{ . | jsonify }},{{ end }} + "since": {{ . | jsonify }},{{ end }}{{ with .Aliases }} + "aliases": {{ . | jsonify }},{{ end }} "content": {{ $content | jsonify }}, "tags": {{ $tags | jsonify }}, "last_updated": {{ $lastUpdated | jsonify }} diff --git a/layouts/index.redirects.json b/layouts/index.redirects.json new file mode 100644 index 0000000000..97b2af1721 --- /dev/null +++ b/layouts/index.redirects.json @@ -0,0 +1,60 @@ +{{- /* Site-wide redirect map, published as /redirects.json. + + Every alias any published page declares, mapped to the page it resolves to. + Rendered from `.Aliases`, which is the same data Hugo uses to emit its own + redirect stubs, so the map and the site's real behavior cannot drift and there + is no generated file to keep in step. + + Ambiguity is separated out rather than resolved. Two pages can declare the same + alias -- 40 do in a production build, 12 of them pointing somewhere different -- + and Hugo settles that by emitting one stub and picking a winner arbitrarily. + Publishing one of those targets as though it were the answer would hand + consumers data the site does not honour, so those keys go in `ambiguous` with + every candidate listed, and `redirects` contains only keys with exactly one + target. An alias declared twice with the same target is simply deduplicated. + + Two caveats a consumer needs, both documented on ai-agent-resources: + + - `from` is normalized to a leading slash and no trailing slash. Authors write + both forms -- 918 of 929 entries carry a leading slash, and the trailing slash + is an even split -- so normalizing here saves every consumer doing it. `to` is + absolute, matching the `url` field every per-page record already publishes. + - This is an *alias* map, not a move log. Many entries are vanity or legacy paths + that were never a page's location, and it carries no date, because frontmatter + does not record when a page moved. + + Only pages Hugo publishes appear, so drafts contribute nothing -- which matches + the site, since Hugo emits no stub for a draft's aliases either. + + See DOC-6951. +*/ -}} +{{- $seen := newScratch -}} +{{- range .Site.Pages -}} + {{- $to := .Permalink -}} + {{- range .Aliases -}} + {{- $from := . | strings.TrimSuffix "/" -}} + {{- if not (hasPrefix $from "/") -}} + {{- $from = printf "/%s" $from -}} + {{- end -}} + {{- $targets := index ($seen.Get "map" | default dict) $from | default slice -}} + {{- $seen.SetInMap "map" $from ($targets | append $to | uniq) -}} + {{- end -}} +{{- end -}} +{{- $redirects := slice -}} +{{- $ambiguous := slice -}} +{{- range $from, $targets := ($seen.Get "map" | default dict) -}} + {{- if eq (len $targets) 1 -}} + {{- $redirects = $redirects | append (dict "from" $from "to" (index $targets 0)) -}} + {{- else -}} + {{- $ambiguous = $ambiguous | append (dict "from" $from "candidates" (sort $targets)) -}} + {{- end -}} +{{- end -}} +{{ dict + "schema_version" site.Params.aiSchemaVersion + "base_url" site.BaseURL + "generated" (now.Format "2006-01-02T15:04:05Z07:00") + "count" (len $redirects) + "ambiguous_count" (len $ambiguous) + "redirects" (sort $redirects "from") + "ambiguous" (sort $ambiguous "from") + | jsonify (dict "indent" " ") }}