From 964d6b4c59f8d27246f0d3e52e40a89dfbac09e7 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Fri, 7 Aug 2026 15:43:49 +0100 Subject: [PATCH 1/2] DOC-6951 Publish redirects so a moved page is not read as a deletion Hugo emits alias stubs for the HTML output format only. So a moved page's old URL serves a 200 meta-refresh page while its /index.json and /index.html.md both 404 -- verified on three correctly-aliased moves and reproduced against Hugo 0.143.1. Our own documentation tells consumers to find a page's JSON by appending /index.json to its URL, so following that instruction on a page that moved returns 404 and the move is indistinguishable from a deletion. That happens for every move, including the ones we alias correctly, which is why the alias backfill did not fix it and this does. Two mechanisms. At the old URL there is now a record with page_type "moved" carrying url and moved_to and nothing else, written after the build by walking the map Hugo renders rather than by parsing stub HTML. And /redirects.json publishes the whole map once, rendered from .Aliases -- the same data Hugo uses for the stubs, so the map and the site's real behavior cannot drift and there is no generated file to keep in step. The map's design changed once measurement contradicted the obvious version. Emitting one entry per declaration produced duplicate keys, 29 of them naming different targets, because two pages can declare the same alias. Hugo settles that by writing one stub and picking a winner arbitrarily, so publishing either target would have handed consumers an answer the site does not honour. Keys with exactly one target now go in `redirects` and contested ones in `ambiguous` with every candidate listed. 791 and 29 respectively in a full local build. Tombstones are only written for unambiguous keys, for the same reason. Tombstones stay out of docs.ndjson. They share the index.json name, so the feed's rglob would have swept in about a thousand pointer records against 2,600 real ones and made any count of the corpus ambiguous -- the exact confusion DOC-6939 spent its time reconciling. generate_ndjson.py filters on page_type instead. Verified: the record count is identical before and after the tombstone pass, and no moved record reaches the feed. page_type "moved" is a new value and moved_to a new field, so aiSchemaVersion goes to 2 by the definition the applied AI team proposed and we adopted. Refusing to bump on the first real change would have taught them to ignore the field. The page records also gain `aliases`, declared on both transform interfaces rather than left to ride on a spread, per the constraint from the schema_version work. It does not disturb content_hash, which covers summary, sections and examples only. Verified end to end on a full build and the real pipeline order: 774 tombstones from 791 map entries with 8 skipped for having no stub and 9 for already holding a real record, no duplicate keys in the map, the feed unchanged at 5,733 records, and a re-run of the transform skipping all 6,507 files rather than rewriting a tombstone. Learned: the naive map published contradictory data -- 29 keys with two targets each -- and only counting the keys showed it; the fix is to publish the contradiction rather than resolve it, because the site resolves it arbitrarily and any single answer would sometimes be wrong Constraint: tombstones are written only for unambiguous keys, never over an existing index.json, and only where Hugo actually emitted a stub -- an alias on a draft or on a taken URL gets nothing, matching the site Constraint: redirect tombstones must stay out of docs.ndjson, which is one record per documentation page; generate_ndjson.py filters page_type "moved" Gaps: the published base_url depends on CI rewriting baseURL in config.toml with sed, which still matches; not observed against a deployed build Ticket: DOC-6951 Co-Authored-By: Claude Opus 5 (1M context) --- Makefile | 10 ++- build/generate_ndjson.py | 9 +++ build/transform_json_sections.ts | 13 ++- build/write_redirect_tombstones.py | 125 +++++++++++++++++++++++++++++ config.toml | 26 +++++- content/ai-agent-resources.md | 74 ++++++++++++++++- layouts/_default/section.json | 3 +- layouts/_default/single.json | 3 +- layouts/index.redirects.json | 60 ++++++++++++++ 9 files changed, 314 insertions(+), 9 deletions(-) create mode 100644 build/write_redirect_tombstones.py create mode 100644 layouts/index.redirects.json diff --git a/Makefile b/Makefile index 272eac0f24..5073aac866 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,16 @@ json_transform: hugo @echo "Transforming JSON files for RAG..." @npx tsx build/transform_json_sections.ts +# Tombstones need public/redirects.json, which hugo renders, and must run before +# ndjson so that generate_ndjson.py can filter them back out of the feed. They are +# written after json_transform only for tidiness -- the transform skips a record +# that has a page_type and no content, so either order is safe. +redirect_tombstones: json_transform + @echo "Writing redirect tombstones..." + @python3 build/write_redirect_tombstones.py public + # ndjson requires json_transform to have processed the JSON files -ndjson: json_transform +ndjson: redirect_tombstones @echo "Generating NDJSON feed..." @python3 build/generate_ndjson.py @echo "Compressing NDJSON feed..." diff --git a/build/generate_ndjson.py b/build/generate_ndjson.py index ab0c125698..0767493264 100644 --- a/build/generate_ndjson.py +++ b/build/generate_ndjson.py @@ -32,6 +32,15 @@ def load_and_validate_json_files(public_dir: Path) -> list[tuple[Path, dict]]: try: with open(json_file, 'r', encoding='utf-8') as f: data = json.load(f) + # Redirect tombstones live at a moved page's old URL and share the + # index.json name, so rglob picks them up. They are deliberately + # kept out of the feed: they would add roughly 1,100 near-empty + # records to 2,600 real ones and make every "how many documents" + # figure ambiguous. Consumers resolve a moved URL either per-URL at + # its own /index.json, or in bulk from /redirects.json. See + # DOC-6951. + if data.get('page_type') == 'moved': + continue # Check for our expected fields if all(key in data for key in ['id', 'title', 'url']): valid_files.append((json_file, data)) diff --git a/build/transform_json_sections.ts b/build/transform_json_sections.ts index 65ea758bad..1fb7776baa 100644 --- a/build/transform_json_sections.ts +++ b/build/transform_json_sections.ts @@ -49,21 +49,27 @@ interface CodeExample { interface PageJsonInput { // Emitted by the Hugo templates and carried through untouched. Declared so the // spread below preserves them explicitly rather than by accident: schema_version is - // what consumers gate re-parsing on, and since is the only version information the - // feed carries. + // what consumers gate re-parsing on, since is the only version information the + // feed carries, and aliases is how a consumer holding a record resolves a stale + // URL without fetching the redirect map. schema_version?: number; id: string; title: string; url: string; summary: string; since?: string; + aliases?: string[]; content?: string; tags: string[]; last_updated: string; children?: unknown[]; } -type PageType = 'content' | 'index'; +// 'moved' is written by the tombstone pass at a moved page's old URL, and carries +// only url and moved_to -- no sections, examples or content_hash. A consumer must +// switch on page_type before assuming the documented content shape. Added in +// schema_version 2 (DOC-6951). +type PageType = 'content' | 'index' | 'moved'; interface PageJsonOutput { schema_version?: number; @@ -72,6 +78,7 @@ interface PageJsonOutput { url: string; summary: string; since?: string; + aliases?: string[]; page_type: PageType; content_hash?: string; tags: string[]; diff --git a/build/write_redirect_tombstones.py b/build/write_redirect_tombstones.py new file mode 100644 index 0000000000..8cdee613c2 --- /dev/null +++ b/build/write_redirect_tombstones.py @@ -0,0 +1,125 @@ +"""Write a JSON tombstone beside every alias stub Hugo emitted. + +Hugo generates alias stubs for the **HTML output format only**. So a moved page's +old URL serves a 200 meta-refresh page, while its ``/index.json`` and +``/index.html.md`` both 404 -- verified on three independent correctly-aliased +moves and reproduced against Hugo 0.143.1. + +That matters because ``content/ai-agent-resources.md`` tells consumers to find a +page's JSON by appending ``/index.json`` to its URL. Following that instruction on +a page that moved returns 404, so the move is indistinguishable from a deletion -- +and this happens for *every* move, including the ones we alias correctly. Fixing +alias coverage does not fix it; this does. + +At each of those URLs we now publish a minimal record: + + {"schema_version": 2, "page_type": "moved", + "id": "develop/ai/agent-memory", + "url": "https://redis.io/docs/latest/develop/ai/agent-memory/", + "moved_to": "https://redis.io/docs/latest/develop/ai/context-engine/agent-memory/"} + +``page_type: "moved"`` is a new value in that vocabulary and ``moved_to`` a new +field, so this is a record-shape change and bumps ``aiSchemaVersion`` to 2. A +consumer must switch on ``page_type`` before assuming the documented content shape: +a tombstone deliberately has no ``sections``, ``examples`` or ``content_hash``. + +Reads ``public/redirects.json`` -- the map Hugo renders from the same ``.Aliases`` +data it uses for the stubs -- rather than parsing stub HTML, so the tombstones +cannot disagree with the map or with the site. + +Two things it will not do, both load-bearing: + +- **Never overwrite an existing ``index.json``.** An alias whose path a real page + occupies gets no tombstone; Hugo does not emit a stub there either, and + clobbering a real record would be far worse than a 404. +- **Only write where a stub exists.** An alias declared on a draft, or one Hugo + dropped because the URL was taken, produces no stub and so gets no tombstone. + +See DOC-6951. +""" + +import argparse +import json +import logging +import os +import sys + +logger = logging.getLogger("write_redirect_tombstones") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("public_dir", nargs="?", default="public", + help="path to the rendered site (default: public)") + parser.add_argument("--dry-run", action="store_true", + help="report what would be written without writing it") + return parser.parse_args() + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(message)s") + args = parse_args() + + map_path = os.path.join(args.public_dir, "redirects.json") + if not os.path.isfile(map_path): + logger.error("write_redirect_tombstones: %s not found. Run hugo first; the " + "map is rendered by layouts/index.redirects.json.", map_path) + return 1 + + with open(map_path, encoding="utf-8") as handle: + redirect_map = json.load(handle) + + schema_version = redirect_map.get("schema_version") + base_url = (redirect_map.get("base_url") or "").rstrip("/") + entries = redirect_map.get("redirects") or [] + + written = no_stub = occupied = 0 + for entry in entries: + source = (entry.get("from") or "").strip() + target = (entry.get("to") or "").strip() + if not source or not target: + continue + + rel = source.lstrip("/") + directory = os.path.join(args.public_dir, *rel.split("/")) if rel else args.public_dir + + # Only where Hugo actually emitted a stub. A missing directory or missing + # index.html means the alias was dropped -- the URL was already taken, or + # it was declared on a draft -- and inventing a record there would publish + # a JSON document at a URL that serves no page. + if not os.path.isfile(os.path.join(directory, "index.html")): + no_stub += 1 + continue + + tombstone_path = os.path.join(directory, "index.json") + if os.path.exists(tombstone_path): + # A real page lives here. Never clobber a real record. + occupied += 1 + continue + + record = { + "schema_version": schema_version, + "id": rel.rstrip("/"), + "title": "Moved", + "url": f"{base_url}/{rel}/" if rel else f"{base_url}/", + "page_type": "moved", + "moved_to": target, + } + if not args.dry_run: + with open(tombstone_path, "w", encoding="utf-8") as handle: + json.dump(record, handle, indent=2) + handle.write("\n") + written += 1 + + logger.info("write_redirect_tombstones: %d tombstone(s) %s from %d map entries.", + written, "would be written" if args.dry_run else "written", len(entries)) + if no_stub: + logger.info(" %d alias(es) had no stub, so were skipped -- the URL is taken " + "by a real page, or the alias is on a draft.", no_stub) + if occupied: + logger.info(" %d had an index.json already and were left alone.", occupied) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/config.toml b/config.toml index 480907858b..fb347a5e7e 100644 --- a/config.toml +++ b/config.toml @@ -71,7 +71,14 @@ gitHubRepo = "https://github.com/redis/docs" # content is a second content hash, which they will learn to ignore. # # Changing what a field *contains* is not a shape change and does not bump this. -aiSchemaVersion = 1 +# +# History: +# 1 -- initial: sections, examples, content_hash, page_type, id, since (DOC-6939) +# 2 -- adds `aliases` to page records, and a new `page_type` value, "moved", for +# the redirect tombstones published at a moved page's old URL. A parser +# switching on page_type now meets a value it has not seen, and those +# records carry a `moved_to` field, so this is a shape change (DOC-6951). +aiSchemaVersion = 2 # Display and sort order for client examples clientsExamples = ["Python", "Node.js", "ioredis", "Java-Sync", "Lettuce-Sync", "Java-Async", "Java-Reactive", "Go", "C", "C#-Sync (NRedisStack)", "C#-Async (NRedisStack)", "C#-Sync (SE.Redis)", "C#-Async (SE.Redis)", "RedisVL", "PHP", "Ruby", "Rust-Sync", "Rust-Async"] @@ -149,7 +156,22 @@ rdi_current_version = "1.19.0" mediaType = "application/json" isPlainText = true + # A site-wide map of every alias to the page it resolves to, published once on + # the home page as /redirects.json. Rendered from `.Aliases` -- the same data + # Hugo uses to emit its own redirect stubs -- so the map and the site's actual + # behavior cannot drift, and there is no generated file to keep in step. + [outputFormats.redirects] + name = "redirects" + baseName = "redirects" + mediaType = "application/json" + isPlainText = true + # Comment out if you don't want the "print entire section" link enabled. [outputs] section = ["HTML", "RSS", "Markdown", "JSON"] -page = ["HTML", "Markdown", "JSON"] \ No newline at end of file +page = ["HTML", "Markdown", "JSON"] +# The home page had no entry, so it was on Hugo's default of HTML and RSS. That +# default is reproduced here deliberately: adding Markdown or JSON would publish a +# new record for the site root, which is one of the pages the feed excludes today. +# Only the redirects map is new. +home = ["HTML", "RSS", "redirects"] \ No newline at end of file diff --git a/content/ai-agent-resources.md b/content/ai-agent-resources.md index b5c579f26f..fb372dbc7a 100644 --- a/content/ai-agent-resources.md +++ b/content/ai-agent-resources.md @@ -68,7 +68,7 @@ Two consequences worth knowing if you diff the feed against the sitemap: ### Schema version Every record carries a `schema_version` integer, and the same value appears in the -`json metadata` block of the Markdown output. It is currently **1**. +`json metadata` block of the Markdown output. It is currently **2**. It increments only when the **shape** of a record changes: a field added, removed or renamed, or a new value entering the [role vocabulary](#section-roles). It does **not** @@ -76,6 +76,78 @@ change when page content changes, and it does not change when the value inside a changes without the field itself changing. Use `content_hash` to detect content changes; use `schema_version` to detect when your parser might need attention. +Version 2 added the `aliases` field to page records, and a new `page_type` value, +`moved`, for the [redirect records](#pages-that-have-moved) served at a moved page's +old URL. Version 1 was the initial shape: `sections`, `examples`, `content_hash`, +`page_type`, `id` and `since`. + +## Pages that have moved + +When a page moves, its old URL keeps working — but until now it served only an HTML +redirect, so appending `/index.json` to it returned 404 and a move was +indistinguishable from a deletion. Two things now fix that. + +### A redirect record at the old URL + +Appending `/index.json` to a moved page's URL returns a record with +`page_type` set to `moved`: + +```json +{ + "schema_version": 2, + "id": "develop/ai/agent-memory", + "title": "Moved", + "url": "https://redis.io/docs/latest/develop/ai/agent-memory/", + "page_type": "moved", + "moved_to": "https://redis.io/docs/latest/develop/ai/context-engine/agent-memory/" +} +``` + +**Check `page_type` before assuming the shape of a record.** A `moved` record +deliberately carries no `sections`, `examples` or `content_hash` — there is no content +at that URL, only a pointer. Fetch `moved_to` to get the page itself. + +These records are **not** included in `docs.ndjson`. The feed is one record per +documentation page, and adding roughly a thousand pointer records would make any count +of the corpus ambiguous. Use the map below if you need them in bulk. + +### A map of every redirect + +`https://redis.io/docs/latest/redirects.json` lists every alias the site publishes and +the page it resolves to: + +```json +{ + "schema_version": 2, + "base_url": "https://redis.io/docs/latest/", + "generated": "2026-08-07T15:00:00Z", + "count": 771, + "ambiguous_count": 12, + "redirects": [ + {"from": "/develop/ai/langcache", "to": "https://redis.io/docs/latest/develop/ai/context-engine/langcache/"} + ], + "ambiguous": [ + {"from": "/develop/use/pipelining", "candidates": ["https://redis.io/docs/latest/develop/using-commands/", "https://redis.io/docs/latest/develop/using-commands/pipelining/"]} + ] +} +``` + +Three things worth knowing before you rely on it: + +- `from` is normalized to a leading slash and no trailing slash. `to` is absolute, + matching the `url` field on page records. +- **It is an alias map, not a move log.** Many entries are vanity or legacy paths that + were never a page's location, and there is no date, because the source data does not + record when a page moved. +- `ambiguous` holds the keys that more than one page claims, with every candidate + listed. We publish them separately rather than picking one, because the site itself + resolves those arbitrarily — so any single answer we gave you would sometimes + disagree with what you would actually be served. Treat an `ambiguous` key as + unresolved. + +The map covers the current version of the documentation. Version-specific +documentation is outside both machine-readable feeds. + ### JSON schema Each document contains: diff --git a/layouts/_default/section.json b/layouts/_default/section.json index 95915770b7..13c7ec7995 100644 --- a/layouts/_default/section.json +++ b/layouts/_default/section.json @@ -27,7 +27,8 @@ "title": {{ .Title | jsonify }}, "url": {{ .Permalink | jsonify }}, "summary": {{ $summary | jsonify }},{{ with .Params.since }} - "since": {{ . | jsonify }},{{ end }} + "since": {{ . | jsonify }},{{ end }}{{ with .Aliases }} + "aliases": {{ . | jsonify }},{{ end }} "content": {{ $content | jsonify }}, "tags": {{ $tags | jsonify }}, "last_updated": {{ $lastUpdated | jsonify }}, diff --git a/layouts/_default/single.json b/layouts/_default/single.json index 30727f7fed..a25a4c7ecd 100644 --- a/layouts/_default/single.json +++ b/layouts/_default/single.json @@ -16,7 +16,8 @@ "title": {{ .Title | jsonify }}, "url": {{ .Permalink | jsonify }}, "summary": {{ $summary | jsonify }},{{ with .Params.since }} - "since": {{ . | jsonify }},{{ end }} + "since": {{ . | jsonify }},{{ end }}{{ with .Aliases }} + "aliases": {{ . | jsonify }},{{ end }} "content": {{ $content | jsonify }}, "tags": {{ $tags | jsonify }}, "last_updated": {{ $lastUpdated | jsonify }} diff --git a/layouts/index.redirects.json b/layouts/index.redirects.json new file mode 100644 index 0000000000..97b2af1721 --- /dev/null +++ b/layouts/index.redirects.json @@ -0,0 +1,60 @@ +{{- /* Site-wide redirect map, published as /redirects.json. + + Every alias any published page declares, mapped to the page it resolves to. + Rendered from `.Aliases`, which is the same data Hugo uses to emit its own + redirect stubs, so the map and the site's real behavior cannot drift and there + is no generated file to keep in step. + + Ambiguity is separated out rather than resolved. Two pages can declare the same + alias -- 40 do in a production build, 12 of them pointing somewhere different -- + and Hugo settles that by emitting one stub and picking a winner arbitrarily. + Publishing one of those targets as though it were the answer would hand + consumers data the site does not honour, so those keys go in `ambiguous` with + every candidate listed, and `redirects` contains only keys with exactly one + target. An alias declared twice with the same target is simply deduplicated. + + Two caveats a consumer needs, both documented on ai-agent-resources: + + - `from` is normalized to a leading slash and no trailing slash. Authors write + both forms -- 918 of 929 entries carry a leading slash, and the trailing slash + is an even split -- so normalizing here saves every consumer doing it. `to` is + absolute, matching the `url` field every per-page record already publishes. + - This is an *alias* map, not a move log. Many entries are vanity or legacy paths + that were never a page's location, and it carries no date, because frontmatter + does not record when a page moved. + + Only pages Hugo publishes appear, so drafts contribute nothing -- which matches + the site, since Hugo emits no stub for a draft's aliases either. + + See DOC-6951. +*/ -}} +{{- $seen := newScratch -}} +{{- range .Site.Pages -}} + {{- $to := .Permalink -}} + {{- range .Aliases -}} + {{- $from := . | strings.TrimSuffix "/" -}} + {{- if not (hasPrefix $from "/") -}} + {{- $from = printf "/%s" $from -}} + {{- end -}} + {{- $targets := index ($seen.Get "map" | default dict) $from | default slice -}} + {{- $seen.SetInMap "map" $from ($targets | append $to | uniq) -}} + {{- end -}} +{{- end -}} +{{- $redirects := slice -}} +{{- $ambiguous := slice -}} +{{- range $from, $targets := ($seen.Get "map" | default dict) -}} + {{- if eq (len $targets) 1 -}} + {{- $redirects = $redirects | append (dict "from" $from "to" (index $targets 0)) -}} + {{- else -}} + {{- $ambiguous = $ambiguous | append (dict "from" $from "candidates" (sort $targets)) -}} + {{- end -}} +{{- end -}} +{{ dict + "schema_version" site.Params.aiSchemaVersion + "base_url" site.BaseURL + "generated" (now.Format "2006-01-02T15:04:05Z07:00") + "count" (len $redirects) + "ambiguous_count" (len $ambiguous) + "redirects" (sort $redirects "from") + "ambiguous" (sort $ambiguous "from") + | jsonify (dict "indent" " ") }} From 32d3f1a5b3f4ab22129230ee462a1b3a824bf093 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Fri, 7 Aug 2026 16:58:59 +0100 Subject: [PATCH 2/2] DOC-6951 Refresh and sweep tombstones instead of only creating them Two Bugbot findings, both correct, and both cases where the pass only worked because CI happens to build into a fresh tree. The no-overwrite guard was too broad. It existed to protect a real page's record, which it must, but it also refused to rewrite the pass's own tombstones -- so on an incremental build a tombstone kept a moved_to that had since changed. The guard now asks whether the file it found is one of ours, by reading page_type, and refuses only for anything else. Nothing removed a tombstone either. An alias that leaves the map, or becomes ambiguous and so moves into the candidate list, left its tombstone behind pointing somewhere the site no longer redirects. The pass now records the set it expects and sweeps any recognisable tombstone the map no longer names. Both are invisible in production, because every CI build starts from an empty directory. That is exactly why they were worth fixing: the script was correct by accident of its environment rather than by construction, and the next person to run it locally would have got a wrong answer with no signal. Verified against a real built tree: a re-run refreshes all 774 rather than skipping them, a deliberately poisoned moved_to is corrected, a planted orphan is removed, a real page record is byte-identical afterwards, and the feed still holds 5,733 records with no moved record leaking in. Learned: "correct because CI starts clean" is not correct -- both defects here were invisible in production and would have produced silently wrong output for anyone running the script twice Constraint: the guard protects real page records, not the pass's own tombstones; those are rewritten every run, and ones the map no longer names are deleted Ticket: DOC-6951 Co-Authored-By: Claude Opus 5 (1M context) --- build/write_redirect_tombstones.py | 61 ++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/build/write_redirect_tombstones.py b/build/write_redirect_tombstones.py index 8cdee613c2..e380a38de5 100644 --- a/build/write_redirect_tombstones.py +++ b/build/write_redirect_tombstones.py @@ -29,9 +29,11 @@ Two things it will not do, both load-bearing: -- **Never overwrite an existing ``index.json``.** An alias whose path a real page - occupies gets no tombstone; Hugo does not emit a stub there either, and - clobbering a real record would be far worse than a 404. +- **Never overwrite a real page's ``index.json``.** An alias whose path a real page + occupies gets no tombstone; Hugo does not emit a stub there either, and clobbering + a real record would be far worse than a 404. A file that is recognisably one of + our own tombstones *is* rewritten, so an incremental build cannot keep serving a + ``moved_to`` that has since changed, and one the map no longer names is removed. - **Only write where a stub exists.** An alias declared on a draft, or one Hugo dropped because the URL was taken, produces no stub and so gets no tombstone. @@ -73,7 +75,16 @@ def main() -> int: base_url = (redirect_map.get("base_url") or "").rstrip("/") entries = redirect_map.get("redirects") or [] - written = no_stub = occupied = 0 + def is_tombstone(path: str) -> bool: + """True if this index.json is one of ours rather than a real page record.""" + try: + with open(path, encoding="utf-8") as existing: + return json.load(existing).get("page_type") == "moved" + except (OSError, json.JSONDecodeError): + return False + + written = refreshed = no_stub = occupied = 0 + expected: set[str] = set() for entry in entries: source = (entry.get("from") or "").strip() target = (entry.get("to") or "").strip() @@ -81,7 +92,8 @@ def main() -> int: continue rel = source.lstrip("/") - directory = os.path.join(args.public_dir, *rel.split("/")) if rel else args.public_dir + directory = (os.path.join(args.public_dir, *rel.split("/")) if rel + else args.public_dir) # Only where Hugo actually emitted a stub. A missing directory or missing # index.html means the alias was dropped -- the URL was already taken, or @@ -92,10 +104,12 @@ def main() -> int: continue tombstone_path = os.path.join(directory, "index.json") - if os.path.exists(tombstone_path): + already = os.path.exists(tombstone_path) + if already and not is_tombstone(tombstone_path): # A real page lives here. Never clobber a real record. occupied += 1 continue + expected.add(os.path.realpath(tombstone_path)) record = { "schema_version": schema_version, @@ -109,10 +123,41 @@ def main() -> int: with open(tombstone_path, "w", encoding="utf-8") as handle: json.dump(record, handle, indent=2) handle.write("\n") - written += 1 + if already: + # One of ours from a previous run. Rewritten rather than left alone, or + # an incremental build would keep serving a moved_to that has since + # changed. CI always builds into a fresh tree, so this only shows up + # locally -- but "correct because CI starts clean" is not correct. + refreshed += 1 + else: + written += 1 + + # Sweep tombstones that no longer belong: the alias was removed, or it became + # ambiguous and is now published as a candidate list instead. Only files that + # are recognisably ours are ever removed, and only when the map no longer names + # them, so a real page record can never be caught by this. + removed = 0 + for root, _dirs, files in os.walk(args.public_dir): + if "index.json" not in files: + continue + path = os.path.join(root, "index.json") + if os.path.realpath(path) in expected: + continue + if not is_tombstone(path): + continue + if not args.dry_run: + os.remove(path) + removed += 1 + verb = "would be written" if args.dry_run else "written" logger.info("write_redirect_tombstones: %d tombstone(s) %s from %d map entries.", - written, "would be written" if args.dry_run else "written", len(entries)) + written, verb, len(entries)) + if refreshed: + logger.info(" %d existing tombstone(s) %s.", refreshed, + "would be refreshed" if args.dry_run else "refreshed") + if removed: + logger.info(" %d obsolete tombstone(s) %s -- no longer in the map.", removed, + "would be removed" if args.dry_run else "removed") if no_stub: logger.info(" %d alias(es) had no stub, so were skipped -- the URL is taken " "by a real page, or the alias is on a draft.", no_stub)