fix(news): derive titles from URL slug for MSN/aggregator articles

disaster_report/pipeline.py

@@ -11,6 +11,7 @@
 from disaster_report._search_keys import derive_repoll_keys
 from disaster_report.fetchers import fetch_article
 from disaster_report.models import IncidentLog, NewsItem, SourceReport
+from disaster_report.sources.ddg_news import derive_title_from_slug, is_generic_title
 from disaster_report.sources.errors import SourceFetchError
 from disaster_report.store.content import ContentStore
 
@@ -34,10 +35,21 @@ def _mint_id() -> str:
 def _enrich_one(news: NewsItem) -> NewsItem:
     fetched = fetch_article(news.url)
     if fetched is None:
+        if is_generic_title(news.title):
+            derived = derive_title_from_slug(news.url)
+            if derived:
+                return dataclasses.replace(news, title=derived)
         return news
+    fetched_title = fetched.title or ""
+    if not is_generic_title(fetched_title):
+        final_title = fetched_title
+    elif not is_generic_title(news.title):
+        final_title = news.title
+    else:
+        final_title = derive_title_from_slug(news.url) or news.title
     return dataclasses.replace(
         news,
-        title=fetched.title or news.title,
+        title=final_title,
         body=fetched.description or news.body,
         published_date=fetched.published_date or news.published_date,
         author=fetched.author,

disaster_report/sources/ddg_news.py

@@ -28,6 +28,16 @@
 _URL_DATE_RE_1 = re.compile(r"/(\d{4})/(\d{2})/(\d{2})/")
 _URL_DATE_RE_2 = re.compile(r"/(\d{8})/")
 
+_MSN_SLUG_RE = re.compile(r"/([a-z0-9][a-z0-9-]*?)/(?:ar|vi)-[A-Za-z0-9]+", re.IGNORECASE)
+
+_GENERIC_TITLES = frozenset({
+    "msn",
+    "client challenge",
+    "just a moment...",
+    "just a moment",
+    "attention required! | cloudflare",
+})
+
 _RELATIVE_RE = re.compile(
     r"(\d+)\s*(minute|hour|day|week|month|year)s?\s*ago", re.IGNORECASE
 )
@@ -209,3 +219,19 @@ def _to_news_item(result: Any, timelimit: str | None = None) -> NewsItem | None:
 def _domain_of(url: str) -> str:
     hostname = urlparse(url).hostname
     return hostname or ""
+
+
+def is_generic_title(title: str) -> bool:
+    cleaned = title.strip().strip("'\"").lower()
+    if not cleaned:
+        return True
+    return cleaned in _GENERIC_TITLES
+
+
+def derive_title_from_slug(url: str) -> str:
+    m = _MSN_SLUG_RE.search(url)
+    if not m:
+        return ""
+    slug = m.group(1)
+    words = [w for w in slug.split("-") if w]
+    return " ".join(w.capitalize() for w in words)

disaster_report/sources/ddg_news.pyi

@@ -5,3 +5,5 @@ class DuckDuckGoNewsAdapter:
     def search(self, query: str, timelimit: str | None = None) -> list[NewsItem]: ...
 
 def _resolve_date(raw_date: str, url: str, timelimit: str | None = None) -> str: ...
+def is_generic_title(title: str) -> bool: ...
+def derive_title_from_slug(url: str) -> str: ...

scripts/backfill_news_titles.py

@@ -0,0 +1,84 @@
+"""Backfill generic news titles (MSN, Client Challenge) from URL slugs.
+
+Walks every news YAML under data/incidents/.../logs/.../news/ and rewrites
+the `title` field when it is a known generic value, deriving a human-readable
+title from the article URL slug. Only the `title:` line is rewritten; the rest
+of the file is preserved byte-for-byte.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from pathlib import Path
+
+_REPO_ROOT = Path(__file__).resolve().parent.parent
+if str(_REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(_REPO_ROOT))
+
+from disaster_report.sources.ddg_news import derive_title_from_slug, is_generic_title
+
+_TITLE_LINE_RE = re.compile(r"^(?P<key>title): (?P<value>.*)$", re.MULTILINE)
+_URL_LINE_RE = re.compile(r"^url: (?P<value>.*)$", re.MULTILINE)
+
+
+def _replace_title_line(text: str, new_title: str) -> str:
+    return _TITLE_LINE_RE.sub(
+        lambda m: f"{m.group('key')}: {new_title}", text, count=1
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Backfill generic news titles from URL slugs"
+    )
+    parser.add_argument(
+        "--tree-root",
+        default="data",
+        help="Data tree root (default: data)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Show changes without writing files",
+    )
+    args = parser.parse_args()
+
+    news_files = sorted(
+        Path(args.tree_root).glob("incidents/*/logs/*/news/*.yaml")
+    )
+    changed = 0
+    skipped = 0
+
+    for f in news_files:
+        text = f.read_text(encoding="utf-8")
+        title_match = _TITLE_LINE_RE.search(text)
+        if not title_match:
+            continue
+        title = title_match.group("value")
+        if not is_generic_title(title):
+            skipped += 1
+            continue
+        url_match = _URL_LINE_RE.search(text)
+        if not url_match:
+            print(f"SKIP (no url): {f.name}")
+            skipped += 1
+            continue
+        url = url_match.group("value").strip()
+        new_title = derive_title_from_slug(url)
+        if not new_title:
+            print(f"SKIP (no slug derivable): {f.name} — {url[:80]}")
+            skipped += 1
+            continue
+        if args.dry_run:
+            print(f"  {title!r} -> {new_title!r}")
+        else:
+            f.write_text(_replace_title_line(text, new_title), encoding="utf-8")
+        changed += 1
+
+    print(f"\n{changed} titles updated, {skipped} skipped")
+
+
+if __name__ == "__main__":
+    main()

tests/integration/ddg_news_source_test.py

@@ -203,6 +203,57 @@ def first_news_item(items: list[NewsItem]) -> NewsItem:
     return items[0]
 
 
+class TestDeriveTitleFromSlug:
+    def test_msn_article_url_extracts_slug(self) -> None:
+        from disaster_report.sources.ddg_news import derive_title_from_slug
+
+        url = (
+            "https://www.msn.com/en-us/news/world/"
+            "venezuela-quakes-caused-196-billion-in-damage-world-bank-report-shows"
+            "/ar-AA28x1CA?ocid=BingNewsVerp"
+        )
+        result = derive_title_from_slug(url)
+        assert result == "Venezuela Quakes Caused 196 Billion In Damage World Bank Report Shows"
+
+    def test_msn_vi_suffix_url_extracts_slug(self) -> None:
+        from disaster_report.sources.ddg_news import derive_title_from_slug
+
+        url = (
+            "https://www.msn.com/en-us/news/world/"
+            "series-of-earthquakes-strike-coast-of-mexico-guatemala"
+            "/vi-AA289xHF"
+        )
+        result = derive_title_from_slug(url)
+        assert result == "Series Of Earthquakes Strike Coast Of Mexico Guatemala"
+
+    def test_non_msn_url_returns_empty(self) -> None:
+        from disaster_report.sources.ddg_news import derive_title_from_slug
+
+        assert derive_title_from_slug("https://www.reuters.com/world/americas/some-article-2026/") == ""
+
+
+class TestIsGenericTitle:
+    def test_msn_is_generic(self) -> None:
+        from disaster_report.sources.ddg_news import is_generic_title
+
+        assert is_generic_title("MSN") is True
+
+    def test_client_challenge_is_generic(self) -> None:
+        from disaster_report.sources.ddg_news import is_generic_title
+
+        assert is_generic_title("Client Challenge") is True
+
+    def test_empty_is_generic(self) -> None:
+        from disaster_report.sources.ddg_news import is_generic_title
+
+        assert is_generic_title("") is True
+
+    def test_real_title_is_not_generic(self) -> None:
+        from disaster_report.sources.ddg_news import is_generic_title
+
+        assert is_generic_title("Venezuela quakes caused $19.6 billion in damage") is False
+
+
 def _load_fixture() -> list[dict[str, object]]:
     import json
 

tests/integration/ddg_news_source_test.pyi

@@ -15,3 +15,14 @@ class TestDuckDuckGoNewsAdapter:
     def test_ddgs_exception_returns_empty_not_raises(self, monkeypatch) -> None: ...
 
 def first_news_item(items: list[NewsItem]) -> NewsItem: ...
+
+class TestDeriveTitleFromSlug:
+    def test_msn_article_url_extracts_slug(self) -> None: ...
+    def test_msn_vi_suffix_url_extracts_slug(self) -> None: ...
+    def test_non_msn_url_returns_empty(self) -> None: ...
+
+class TestIsGenericTitle:
+    def test_msn_is_generic(self) -> None: ...
+    def test_client_challenge_is_generic(self) -> None: ...
+    def test_empty_is_generic(self) -> None: ...
+    def test_real_title_is_not_generic(self) -> None: ...