diff --git a/disaster_report/pipeline.py b/disaster_report/pipeline.py index ba6db3dc..9fb7745f 100644 --- a/disaster_report/pipeline.py +++ b/disaster_report/pipeline.py @@ -11,6 +11,7 @@ from disaster_report._search_keys import derive_repoll_keys from disaster_report.fetchers import fetch_article from disaster_report.models import IncidentLog, NewsItem, SourceReport +from disaster_report.sources.ddg_news import derive_title_from_slug, is_generic_title from disaster_report.sources.errors import SourceFetchError from disaster_report.store.content import ContentStore @@ -34,10 +35,21 @@ def _mint_id() -> str: def _enrich_one(news: NewsItem) -> NewsItem: fetched = fetch_article(news.url) if fetched is None: + if is_generic_title(news.title): + derived = derive_title_from_slug(news.url) + if derived: + return dataclasses.replace(news, title=derived) return news + fetched_title = fetched.title or "" + if not is_generic_title(fetched_title): + final_title = fetched_title + elif not is_generic_title(news.title): + final_title = news.title + else: + final_title = derive_title_from_slug(news.url) or news.title return dataclasses.replace( news, - title=fetched.title or news.title, + title=final_title, body=fetched.description or news.body, published_date=fetched.published_date or news.published_date, author=fetched.author, diff --git a/disaster_report/sources/ddg_news.py b/disaster_report/sources/ddg_news.py index 8e9ed20d..0f23461a 100644 --- a/disaster_report/sources/ddg_news.py +++ b/disaster_report/sources/ddg_news.py @@ -28,6 +28,16 @@ _URL_DATE_RE_1 = re.compile(r"/(\d{4})/(\d{2})/(\d{2})/") _URL_DATE_RE_2 = re.compile(r"/(\d{8})/") +_MSN_SLUG_RE = re.compile(r"/([a-z0-9][a-z0-9-]*?)/(?:ar|vi)-[A-Za-z0-9]+", re.IGNORECASE) + +_GENERIC_TITLES = frozenset({ + "msn", + "client challenge", + "just a moment...", + "just a moment", + "attention required! | cloudflare", +}) + _RELATIVE_RE = re.compile( r"(\d+)\s*(minute|hour|day|week|month|year)s?\s*ago", re.IGNORECASE ) @@ -209,3 +219,19 @@ def _to_news_item(result: Any, timelimit: str | None = None) -> NewsItem | None: def _domain_of(url: str) -> str: hostname = urlparse(url).hostname return hostname or "" + + +def is_generic_title(title: str) -> bool: + cleaned = title.strip().strip("'\"").lower() + if not cleaned: + return True + return cleaned in _GENERIC_TITLES + + +def derive_title_from_slug(url: str) -> str: + m = _MSN_SLUG_RE.search(url) + if not m: + return "" + slug = m.group(1) + words = [w for w in slug.split("-") if w] + return " ".join(w.capitalize() for w in words) diff --git a/disaster_report/sources/ddg_news.pyi b/disaster_report/sources/ddg_news.pyi index 8452ea4d..5a12fde3 100644 --- a/disaster_report/sources/ddg_news.pyi +++ b/disaster_report/sources/ddg_news.pyi @@ -5,3 +5,5 @@ class DuckDuckGoNewsAdapter: def search(self, query: str, timelimit: str | None = None) -> list[NewsItem]: ... def _resolve_date(raw_date: str, url: str, timelimit: str | None = None) -> str: ... +def is_generic_title(title: str) -> bool: ... +def derive_title_from_slug(url: str) -> str: ... diff --git a/scripts/backfill_news_titles.py b/scripts/backfill_news_titles.py new file mode 100644 index 00000000..e01de469 --- /dev/null +++ b/scripts/backfill_news_titles.py @@ -0,0 +1,84 @@ +"""Backfill generic news titles (MSN, Client Challenge) from URL slugs. + +Walks every news YAML under data/incidents/.../logs/.../news/ and rewrites +the `title` field when it is a known generic value, deriving a human-readable +title from the article URL slug. Only the `title:` line is rewritten; the rest +of the file is preserved byte-for-byte. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from disaster_report.sources.ddg_news import derive_title_from_slug, is_generic_title + +_TITLE_LINE_RE = re.compile(r"^(?Ptitle): (?P.*)$", re.MULTILINE) +_URL_LINE_RE = re.compile(r"^url: (?P.*)$", re.MULTILINE) + + +def _replace_title_line(text: str, new_title: str) -> str: + return _TITLE_LINE_RE.sub( + lambda m: f"{m.group('key')}: {new_title}", text, count=1 + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Backfill generic news titles from URL slugs" + ) + parser.add_argument( + "--tree-root", + default="data", + help="Data tree root (default: data)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show changes without writing files", + ) + args = parser.parse_args() + + news_files = sorted( + Path(args.tree_root).glob("incidents/*/logs/*/news/*.yaml") + ) + changed = 0 + skipped = 0 + + for f in news_files: + text = f.read_text(encoding="utf-8") + title_match = _TITLE_LINE_RE.search(text) + if not title_match: + continue + title = title_match.group("value") + if not is_generic_title(title): + skipped += 1 + continue + url_match = _URL_LINE_RE.search(text) + if not url_match: + print(f"SKIP (no url): {f.name}") + skipped += 1 + continue + url = url_match.group("value").strip() + new_title = derive_title_from_slug(url) + if not new_title: + print(f"SKIP (no slug derivable): {f.name} — {url[:80]}") + skipped += 1 + continue + if args.dry_run: + print(f" {title!r} -> {new_title!r}") + else: + f.write_text(_replace_title_line(text, new_title), encoding="utf-8") + changed += 1 + + print(f"\n{changed} titles updated, {skipped} skipped") + + +if __name__ == "__main__": + main() diff --git a/tests/integration/ddg_news_source_test.py b/tests/integration/ddg_news_source_test.py index f1dccb8e..12a4abe8 100644 --- a/tests/integration/ddg_news_source_test.py +++ b/tests/integration/ddg_news_source_test.py @@ -203,6 +203,57 @@ def first_news_item(items: list[NewsItem]) -> NewsItem: return items[0] +class TestDeriveTitleFromSlug: + def test_msn_article_url_extracts_slug(self) -> None: + from disaster_report.sources.ddg_news import derive_title_from_slug + + url = ( + "https://www.msn.com/en-us/news/world/" + "venezuela-quakes-caused-196-billion-in-damage-world-bank-report-shows" + "/ar-AA28x1CA?ocid=BingNewsVerp" + ) + result = derive_title_from_slug(url) + assert result == "Venezuela Quakes Caused 196 Billion In Damage World Bank Report Shows" + + def test_msn_vi_suffix_url_extracts_slug(self) -> None: + from disaster_report.sources.ddg_news import derive_title_from_slug + + url = ( + "https://www.msn.com/en-us/news/world/" + "series-of-earthquakes-strike-coast-of-mexico-guatemala" + "/vi-AA289xHF" + ) + result = derive_title_from_slug(url) + assert result == "Series Of Earthquakes Strike Coast Of Mexico Guatemala" + + def test_non_msn_url_returns_empty(self) -> None: + from disaster_report.sources.ddg_news import derive_title_from_slug + + assert derive_title_from_slug("https://www.reuters.com/world/americas/some-article-2026/") == "" + + +class TestIsGenericTitle: + def test_msn_is_generic(self) -> None: + from disaster_report.sources.ddg_news import is_generic_title + + assert is_generic_title("MSN") is True + + def test_client_challenge_is_generic(self) -> None: + from disaster_report.sources.ddg_news import is_generic_title + + assert is_generic_title("Client Challenge") is True + + def test_empty_is_generic(self) -> None: + from disaster_report.sources.ddg_news import is_generic_title + + assert is_generic_title("") is True + + def test_real_title_is_not_generic(self) -> None: + from disaster_report.sources.ddg_news import is_generic_title + + assert is_generic_title("Venezuela quakes caused $19.6 billion in damage") is False + + def _load_fixture() -> list[dict[str, object]]: import json diff --git a/tests/integration/ddg_news_source_test.pyi b/tests/integration/ddg_news_source_test.pyi index cda53f75..a34f8d9e 100644 --- a/tests/integration/ddg_news_source_test.pyi +++ b/tests/integration/ddg_news_source_test.pyi @@ -15,3 +15,14 @@ class TestDuckDuckGoNewsAdapter: def test_ddgs_exception_returns_empty_not_raises(self, monkeypatch) -> None: ... def first_news_item(items: list[NewsItem]) -> NewsItem: ... + +class TestDeriveTitleFromSlug: + def test_msn_article_url_extracts_slug(self) -> None: ... + def test_msn_vi_suffix_url_extracts_slug(self) -> None: ... + def test_non_msn_url_returns_empty(self) -> None: ... + +class TestIsGenericTitle: + def test_msn_is_generic(self) -> None: ... + def test_client_challenge_is_generic(self) -> None: ... + def test_empty_is_generic(self) -> None: ... + def test_real_title_is_not_generic(self) -> None: ...