diff --git a/functions-python/tasks_executor/src/main.py b/functions-python/tasks_executor/src/main.py index f1ff61b85..0c1e534a8 100644 --- a/functions-python/tasks_executor/src/main.py +++ b/functions-python/tasks_executor/src/main.py @@ -76,6 +76,18 @@ notifications_dispatch_monitor_handler, ) from tasks.changelog.backfill_changelog import backfill_changelog_handler +from tasks.seal_of_reliability.backfill.backfill_seal_of_reliability import ( + backfill_seal_of_reliability_handler, +) + +from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, +) + +from tasks.seal_of_reliability.backfill.seal_backfill_worker import ( + seal_backfill_worker_handler, +) + from tasks.seal_of_reliability.update_seal_of_reliability import ( update_seal_of_reliability_handler, ) @@ -302,6 +314,53 @@ ), "handler": update_seal_of_reliability_handler, }, + "backfill_seal_of_reliability": { + "description": ( + "Establishes a starting Seal of Reliability state for feeds that have none " + "(issue #1763), by cold-starting each feed 12 months back and replaying the " + "nightly evaluation forward one day at a time to end_date, writing only the " + "final day. The intermediate days are held in memory and discarded unless " + "snapshot_mode says otherwise. " + "Parameters: stable_feed_ids (required, non-empty), start_date (ISO date, " + "default end_date minus days_back; clamped up to each feed's created_at), " + "end_date (ISO date, default yesterday UTC), days_back (default 365), " + "dry_run (default true), limit (default null), criteria (default null " + "meaning every implemented criterion), batch_size (default 200), " + "only_missing (default true; skips feeds that already have seal state), " + "snapshot_mode (final|all|none, default final), resume_from_snapshot " + "(default false; the #1803 hook), max_reported_feeds (default 50)." + ), + "handler": backfill_seal_of_reliability_handler, + }, + "seal_backfill_orchestrator": { + "description": ( + "Cloud Tasks producer for the Seal of Reliability backfill across the whole " + "catalog (issue #1763). Resolves every seal-eligible GTFS feed that has no " + "seal state yet, chunks it, registers a run in TaskExecutionTracker (feeds " + "DB), and enqueues one 'seal_backfill_worker' task per batch plus a single " + "'seal_orchestrator_monitor' barrier task. The window is resolved here once " + "and passed to every worker, so all batches of a run end on the same day. " + "Parameters: dry_run (default true), batch_size (default 100), start_date " + "(ISO date, default end_date minus days_back), end_date (ISO date, default " + "yesterday UTC), days_back (default 365), criteria (default null), limit " + "(default null), stable_feed_ids (restrict eligibility to these ids, default " + "null), only_missing (default true), snapshot_mode (final|all|none, default " + "final), resume_from_snapshot (default false), deadline_seconds (default " + "7200), monitor_delay_seconds (default 300)." + ), + "handler": seal_backfill_orchestrator_handler, + }, + "seal_backfill_worker": { + "description": ( + "Cloud Tasks worker: march one batch's worth of feeds for the Seal of " + "Reliability backfill and report completion/failure to TaskExecutionTracker. " + "Parameters: run_id (required), batch_id (required), stable_feed_ids " + "(required, non-empty), start_date (required, ISO date), end_date (required, " + "ISO date), criteria (default null), only_missing (default true), " + "snapshot_mode (default final), resume_from_snapshot (default false)." + ), + "handler": seal_backfill_worker_handler, + }, "seal_orchestrator": { "description": ( "Cloud Tasks producer for the nightly Seal of Reliability run across the " @@ -333,8 +392,10 @@ "batch of a seal orchestrator run has reported, or the run's " "deadline_seconds passes, then aggregates each batch's report and marks " "the run completed (every batch succeeded) or failed (any batch failed, " - "or the deadline was reached with batches still unaccounted for). " - "Parameters: run_id (required)." + "or the deadline was reached with batches still unaccounted for). Settles " + "both the nightly fan-out and the backfill fan-out; task_name selects which. " + "Parameters: run_id (required), task_name (default 'seal_orchestrator_run'; " + "pass 'seal_backfill_run' for a backfill run)." ), "handler": seal_orchestrator_monitor_handler, }, diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py new file mode 100644 index 000000000..2fa7c8d82 --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py @@ -0,0 +1,132 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Task entry point for the Seal of Reliability backfill (issue #1763).""" + +from datetime import date, datetime +from typing import Optional + +from tasks.seal_of_reliability.backfill.seal_backfill import ( + DEFAULT_DAYS_BACK, + DEFAULT_SNAPSHOT_MODE, + backfill_seals, +) +from tasks.seal_of_reliability.seal_updater import ( + DEFAULT_BATCH_SIZE, + DEFAULT_MAX_REPORTED_FEEDS, +) + + +def _parse_day(value: Optional[str], field: str) -> Optional[date]: + """Parse a payload date string to a `date`, or None when absent. + + Accepts a full timestamp too — an operator pasting the nightly task's `now` should not + hit a parse error. The march is day-granular, so the time is dropped either way. + """ + if value is None: + return None + try: + return date.fromisoformat(value) + except ValueError: + pass + try: + return datetime.fromisoformat(value).date() + except ValueError: + raise ValueError( + f"{field} must be an ISO date such as 2026-01-31, got {value!r}" + ) + + +def get_parameters(payload: dict): + """Read the task parameters from the payload, applying defaults.""" + payload = payload or {} + return ( + payload.get("stable_feed_ids"), + _parse_day(payload.get("start_date"), "start_date"), + _parse_day(payload.get("end_date"), "end_date"), + payload.get("days_back", DEFAULT_DAYS_BACK), + payload.get("dry_run", True), + payload.get("limit", None), + payload.get("criteria", None), + payload.get("batch_size", DEFAULT_BATCH_SIZE), + payload.get("only_missing", True), + payload.get("snapshot_mode", DEFAULT_SNAPSHOT_MODE), + payload.get("resume_from_snapshot", False), + payload.get("max_reported_feeds", DEFAULT_MAX_REPORTED_FEEDS), + payload.get("simulate", None), + payload.get("trace", False), + ) + + +def backfill_seal_of_reliability_handler(payload: dict) -> dict: + """Handler for the Seal of Reliability backfill. A dry run returns the plan only. + + Payload, all optional but `stable_feed_ids`: + stable_feed_ids required, non-empty; there is no run-the-whole-catalogue mode + start_date ISO date, clamped up to each feed's created_at. Default: end_date + minus days_back + end_date ISO date, last day simulated. Default: yesterday UTC + days_back window length when start_date is absent. Default: 365 + dry_run Default: True + limit cap the number of feeds. Default: no limit + criteria restrict to these criteria. Default: every implemented one + batch_size Default: 200 + only_missing skip feeds that already have seal state. Default: True + snapshot_mode final | all | none. Default: final + resume_from_snapshot seed from the snapshot before march_start (#1803). + Default: False + max_reported_feeds cap on the `feeds` list in the response. Default: 50 + simulate force statuses per criterion, on days counted from each feed's + march start: {"fresh_coverage": {"default": "pass", "fail": [3]}}. + see `parse_simulation` for the full shape. Combining it with + dry_run=false writes fabricated verdicts, which is refused in + production and on production's tunnel port + trace return the march day by day: every seal_criterion field, plus where + it came from. Marches without writing when dry_run. Consecutive days + in which nothing changed are always collapsed into one entry — its + first day, its last, and the count between + """ + ( + stable_feed_ids, + start_date, + end_date, + days_back, + dry_run, + limit, + criteria, + batch_size, + only_missing, + snapshot_mode, + resume_from_snapshot, + max_reported_feeds, + simulate, + trace, + ) = get_parameters(payload) + return backfill_seals( + stable_feed_ids=stable_feed_ids, + start_date=start_date, + end_date=end_date, + days_back=days_back, + dry_run=dry_run, + limit=limit, + criteria=criteria, + batch_size=batch_size, + only_missing=only_missing, + snapshot_mode=snapshot_mode, + resume_from_snapshot=resume_from_snapshot, + max_reported_feeds=max_reported_feeds, + simulate=simulate, + trace=trace, + ) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py new file mode 100644 index 000000000..722f0f2a5 --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py @@ -0,0 +1,620 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Seal of Reliability backfill (issue #1763). + +Gives feeds with no seal state a starting one, so the nightly job (#1761) has a "yesterday" +to step from: cold-start each feed at `march_start`, replay the nightly evaluation forward a +day at a time to `end_date`, write only the final day. Marching is what builds the +path-dependent state (grace streaks, probation) the final state depends on. A dry run returns +the plan without writing. + +`march_start = max(start_date, feed.created_at)` — skips days before the feed existed, and is +what Stable counts its 180 days from. `end_date` is resolved once by the caller, never per +feed, so every feed of a run ends on the same day. + +Two limits, argued in misc/AI/seal_backfill_algorithm_1763.md: Official and Stable have no +history and are read at today's values; and the cold start's error is not bounded by the +window, so `days_back` is a cost/coverage default rather than a correctness guarantee. +""" + +import logging +import time as clock +from datetime import date, datetime, time, timedelta, timezone +from typing import Dict, List, Optional, Sequence, Set, Tuple + +from sqlalchemy import and_, or_, select +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.orm import Session + +from shared.database.database import with_db_session +from shared.database_gen.sqlacodegen_models import Gtfsfeed, SealCriterion + +from tasks.seal_of_reliability.context import ( + FeedSealContext, + batched, + collect_inputs, + is_seal_eligible, +) +from tasks.seal_of_reliability.backfill.simulation import ( + MAX_TRACE_ROWS, + check_simulated_write_allowed, + check_simulation_fits, + collapse_runs, + observe, + parse_simulation, + policy_for, + trace_row, +) +from shared.common.seal_criteria import CriterionStatus, SealCriterionName +from tasks.seal_of_reliability.seal_updater import ( + DEFAULT_BATCH_SIZE, + DEFAULT_MAX_REPORTED_FEEDS, + SEAL_TABLE, + SNAPSHOT_STATE_COLUMNS, + SNAPSHOT_TABLE, + _load_previous_seals, + _resolve_evaluators, + _roll_up_has_seal, + _upsert_criteria, + _upsert_criterion_snapshot, + _validate_requested_feed_ids, + is_partial_run, +) +from tasks.seal_of_reliability.state_machine import SealCriterionState, transition + +logger = logging.getLogger(__name__) + +# Days rather than months, so the arithmetic needs no calendar library: 365 is #1763's +# "12 months". +DEFAULT_DAYS_BACK: int = 365 + +# What to record in seal_criterion_snapshot: only the last day (per #1763), every simulated +# day (millions of rows over a year, but what lets #1803 resume inside the window), or none. +SNAPSHOT_MODES: Tuple[str, ...] = ("final", "all", "none") +DEFAULT_SNAPSHOT_MODE: str = "final" + +CRITERION_TABLE = SealCriterion.__table__ + + +def yesterday_utc() -> date: + """The default `end_date`: the last day that is fully over in UTC.""" + return datetime.now(timezone.utc).date() - timedelta(days=1) + + +def resolve_window( + start_date: Optional[date], + end_date: Optional[date], + days_back: int, +) -> Tuple[date, date]: + """Resolve the run-wide window once, so every feed of a run ends on the same day.""" + if days_back <= 0: + raise ValueError("days_back must be a positive integer") + + resolved_end = end_date or yesterday_utc() + resolved_start = start_date or (resolved_end - timedelta(days=days_back)) + + if resolved_start > resolved_end: + raise ValueError( + f"start_date ({resolved_start.isoformat()}) is after end_date " + f"({resolved_end.isoformat()})" + ) + return resolved_start, resolved_end + + +def march_start_for(feed: Gtfsfeed, start_date: date) -> date: + """The later of the window start and the feed's creation. + + Also the value Stable counts from, and what makes the cold start exact for a feed younger + than the window: it has no history before its creation to be wrong about. + """ + created = feed.created_at + if created is None: # NOT NULL in the schema; defensive only + return start_date + created_day = ( + created.astimezone(timezone.utc).date() + if created.tzinfo is not None + else created.date() + ) + return max(start_date, created_day) + + +def _feeds_with_seal_state(db_session: Session, feed_ids: Sequence[str]) -> Set[str]: + """Feeds that already have seal state, which `only_missing` excludes. + + Re-marching a feed the nightly job owns would write a simulation over real history. + """ + if not feed_ids: + return set() + rows = db_session.execute( + select(CRITERION_TABLE.c.feed_id) + .where(CRITERION_TABLE.c.feed_id.in_(list(feed_ids))) + .distinct() + ).all() + return {row.feed_id for row in rows} + + +def day_start(day: date) -> datetime: + """The `now` a simulated day is evaluated at: midnight UTC. + + Fixed so `snapshot_date_of(now)` is the day itself and probation's `_next_day_start(now)` + lands on the following midnight, with no rounding to reason about. + """ + return datetime.combine(day, time.min, tzinfo=timezone.utc) + + +def days_between(first: date, last: date) -> List[date]: + """Every day from `first` to `last`, ascending, both ends included.""" + return [first + timedelta(days=offset) for offset in range((last - first).days + 1)] + + +def _state_from_snapshot(row) -> SealCriterionState: + """Rebuild a `SealCriterionState` from one snapshot row. + + Columns come from the table rather than a list — the mirror of `_snapshot_row`, which + writes them — so a new column is read back without editing this, and fails loudly if + `SealCriterionState` has no field for it. + """ + values = {} + for column in SNAPSHOT_STATE_COLUMNS: + value = getattr(row, column) + if column in ("observed_status", "confirmed_status"): + value = CriterionStatus(value) + values[column] = value + return SealCriterionState( + feed_id=row.feed_id, + criterion=SealCriterionName(row.criterion), + **values, + ) + + +def _seed_states( + db_session: Session, + feeds: Sequence[Gtfsfeed], + windows: Dict[str, Tuple[date, date]], + resume_from_snapshot: bool, +) -> Dict[Tuple[str, str], SealCriterionState]: + """The state each (feed, criterion) enters its first simulated day with. + + Empty unless `resume_from_snapshot`, which seeds each pair from its latest snapshot + before that feed's march start — a complete state, so a cold start becomes a resume + (#1803). Pairs with no snapshot are absent, and cold-start as usual: a resume reaching + further back than the snapshots go degrades rather than fails. + """ + if not resume_from_snapshot or not feeds: + return {} + + # One query for the batch. Each feed has its own cut-off, hence the OR; DISTINCT ON + # keeps the latest row per pair. + cutoffs = [ + and_( + SNAPSHOT_TABLE.c.feed_id == feed.id, + SNAPSHOT_TABLE.c.snapshot_date < windows[feed.id][0], + ) + for feed in feeds + if feed.id in windows + ] + if not cutoffs: + return {} + + rows = db_session.execute( + select(SNAPSHOT_TABLE) + .where(or_(*cutoffs)) + .distinct(SNAPSHOT_TABLE.c.feed_id, SNAPSHOT_TABLE.c.criterion) + .order_by( + SNAPSHOT_TABLE.c.feed_id, + SNAPSHOT_TABLE.c.criterion, + SNAPSHOT_TABLE.c.snapshot_date.desc(), + ) + ).all() + return {(row.feed_id, row.criterion): _state_from_snapshot(row) for row in rows} + + +def _upsert_seals_from_backfill( + db_session: Session, + outcomes: Sequence[dict], + now: datetime, +) -> None: + """Write feed_reliability_seal for the marched feeds. + + Differs from the nightly `_upsert_seals` only in `created_at`, which is written as the + feed's march start (left at `DEFAULT now()`, Stable would fail on every simulated day and + the backfill would grant nothing) and is **insert-only**, so a re-backfill cannot reset a + countdown already running. + + `seal_earned_at` gets `end_date`. The march knows the day the roll-up flipped, but under + a cold start that is often day one — which would claim a feed earned its seal a year ago + on one simulated day. + """ + for outcome in outcomes: + row = { + "feed_id": outcome["feed_id"], + "has_seal": outcome["has_seal"], + "created_at": day_start(outcome["tracking_start"]), + "updated_at": now, + } + if outcome["granted"]: + row["seal_earned_at"] = now + elif outcome["revoked"]: + row["seal_lost_at"] = now + + statement = insert(SEAL_TABLE).values(**row) + update_set = { + "has_seal": statement.excluded.has_seal, + "updated_at": statement.excluded.updated_at, + } + # created_at is deliberately not in update_set — see the docstring. + if "seal_earned_at" in row: + update_set["seal_earned_at"] = statement.excluded.seal_earned_at + if "seal_lost_at" in row: + update_set["seal_lost_at"] = statement.excluded.seal_lost_at + db_session.execute( + statement.on_conflict_do_update( + index_elements=[SEAL_TABLE.c.feed_id], set_=update_set + ) + ) + + +def _longest_march(windows: Dict[str, Tuple[date, date]]) -> int: + """Days in the longest window of the run, both ends included. + + Feeds clamped to their own `created_at` march fewer, so this is an upper bound rather + than a length they all share. It is the report's `days`, and the range a simulated day + offset has to fall inside. Zero when nothing was selected. + """ + return max(((end - start).days + 1 for start, end in windows.values()), default=0) + + +def _march( + db_session: Session, + feeds: Sequence[Gtfsfeed], + windows: Dict[str, Tuple[date, date]], + evaluators: Sequence, + end_date: date, + snapshot_mode: str, + resume_from_snapshot: bool, + partial_run: bool, + simulation: Optional[Dict[str, Dict[int, CriterionStatus]]] = None, + trace: bool = False, + write: bool = True, +) -> dict: + """Replay the nightly evaluation day by day for one batch, and write the final day. + + The evaluation is the nightly job's, unmodified; what this adds is threading each day's + state into the next in memory, so a year costs one write per feed rather than 366. + Ascending order is the algorithm, not a convenience: each day feeds the next. + """ + if not feeds: + return { + "feeds": 0, + "criterion_rows": 0, + "snapshot_rows": 0, + "outcomes": [], + "trace": [], + } + + marched_days = days_between(min(start for start, _ in windows.values()), end_date) + + # One load per criterion for the whole range; per-day queries would be thousands. + inputs = collect_inputs(db_session, feeds, marched_days, evaluators) + + states = _seed_states(db_session, feeds, windows, resume_from_snapshot) + simulation = simulation or {} + snapshot_rows = 0 + trace_rows: List[dict] = [] + + for today in marched_days: + now = day_start(today) + # A feed whose march starts later is simply not evaluated yet: its window was + # clamped to its own created_at, and days before that have nothing to say about it. + active = [feed for feed in feeds if windows[feed.id][0] <= today] + if not active: + continue + + days_states: List[SealCriterionState] = [] + for feed in active: + # Every day-invariant field on the context is set here, not just the ones the + # criteria implemented today read: one left out defaults to None and the + # criterion reading it degrades to a silent UNKNOWN or a wrong FAIL rather than + # an error. `latest_dataset` and anything else that varies by day arrives + # through `inputs` instead. + ctx = FeedSealContext( + feed_id=feed.id, + now=now, + stable_id=feed.stable_id, + official=feed.official, + is_producer_url_unstable=feed.is_producer_url_unstable, + seasonal=feed.seasonal, + feed_created_at=feed.created_at, + inputs=inputs, + ) + offset = (today - windows[feed.id][0]).days + for evaluator in evaluators: + key = (feed.id, evaluator.name.value) + observation = observe(evaluator, ctx, simulation, offset) + # A simulation may lend a criterion a grace period or probation it does not + # have, which is the only way Official shows any debouncing at all. + grace, probation = policy_for(evaluator, simulation) + states[key] = transition( + prev=states.get(key), + observation=observation, + grace_period=grace, + probation_period=probation, + now=now, + feed_id=feed.id, + ) + days_states.append(states[key]) + if trace and len(trace_rows) < MAX_TRACE_ROWS: + trace_rows.append( + trace_row(feed, evaluator, offset, observation, states[key]) + ) + + if write and snapshot_mode == "all": + # The only mode that writes inside the loop, flushed per day so a year's march + # does not hold every day in memory. + _upsert_criterion_snapshot(db_session, days_states, today) + snapshot_rows += len(days_states) + db_session.commit() + + final_states = list(states.values()) + outcomes = _final_outcomes(db_session, feeds, windows, states, partial_run) + + if write: + _upsert_criteria(db_session, final_states, day_start(end_date)) + if snapshot_mode == "final": + _upsert_criterion_snapshot(db_session, final_states, end_date) + snapshot_rows += len(final_states) + if outcomes: + _upsert_seals_from_backfill(db_session, outcomes, day_start(end_date)) + db_session.commit() + + return { + "feeds": len(feeds), + "criterion_rows": len(final_states), + "snapshot_rows": snapshot_rows, + "outcomes": outcomes, + "trace": trace_rows, + } + + +def _final_outcomes( + db_session: Session, + feeds: Sequence[Gtfsfeed], + windows: Dict[str, Tuple[date, date]], + states: Dict[Tuple[str, str], SealCriterionState], + partial_run: bool, +) -> List[dict]: + """Roll `has_seal` up from the final day's state, one entry per marched feed. + + Skipped on a partial criteria run, as in `update_seals`: criteria that were not evaluated + cannot be judged. + """ + if partial_run: + return [] + + previous = _load_previous_seals(db_session, [feed.id for feed in feeds]) + outcomes = [] + for feed in feeds: + feed_states = { + criterion: state + for (owner_id, criterion), state in states.items() + if owner_id == feed.id + } + had_seal = bool(previous.get(feed.id)) + has_seal = _roll_up_has_seal(feed_states) + outcomes.append( + { + "feed_id": feed.id, + "stable_id": feed.stable_id, + "tracking_start": windows[feed.id][0], + "had_seal": had_seal, + "has_seal": has_seal, + # A first evaluation can grant but never revoke: nothing was held to lose. + "granted": has_seal and not had_seal, + "revoked": had_seal and not has_seal, + } + ) + return outcomes + + +@with_db_session +def backfill_seals( + db_session: Session, + stable_feed_ids: Sequence[str], + start_date: Optional[date] = None, + end_date: Optional[date] = None, + days_back: int = DEFAULT_DAYS_BACK, + dry_run: bool = True, + limit: Optional[int] = None, + criteria: Optional[Sequence[str]] = None, + batch_size: int = DEFAULT_BATCH_SIZE, + only_missing: bool = True, + snapshot_mode: str = DEFAULT_SNAPSHOT_MODE, + resume_from_snapshot: bool = False, + max_reported_feeds: int = DEFAULT_MAX_REPORTED_FEEDS, + simulate: Optional[dict] = None, + trace: bool = False, +) -> dict: + """Plan and run the backfill for an explicit list of feeds. + + Enumerating the catalogue is the producer's job, as with `update_seals`. Unknown or + ineligible ids are skipped with a warning; it raises only if none can be used. See + `backfill_seal_of_reliability` for the parameters as an operator passes them. + + `simulate` forces observed statuses on named days, and `trace` returns the state each + day left behind, always collapsed to one entry per unchanged stretch — a year of days is + mostly repetition, and no caller wanted it row by row. Both are inspection tools and + neither may write: see the dry_run check below. + + Returns a report; `days` is the longest march in the run, since feeds clamped to their + own `created_at` march fewer. + """ + started = clock.monotonic() + if not stable_feed_ids: + raise ValueError("stable_feed_ids is required and must be non-empty") + if simulate and not dry_run: + check_simulated_write_allowed(simulate, db_session) + if snapshot_mode not in SNAPSHOT_MODES: + raise ValueError( + f"Unknown snapshot_mode {snapshot_mode!r}. Known modes: {list(SNAPSHOT_MODES)}" + ) + if batch_size <= 0: + raise ValueError("batch_size must be a positive integer") + + window_start, window_end = resolve_window(start_date, end_date, days_back) + evaluators = _resolve_evaluators(criteria) + simulation = parse_simulation(simulate, evaluators) + + # By-id load then eligibility in Python, as `update_seals` does: tells "not found" from + # "found but ineligible" without a second query. + query = db_session.query(Gtfsfeed).filter( + Gtfsfeed.stable_id.in_(list(stable_feed_ids)) + ) + if limit is not None: + query = query.limit(limit) + feeds = query.all() + eligible = [feed for feed in feeds if is_seal_eligible(feed)] + + already_backfilled = ( + _feeds_with_seal_state(db_session, [feed.id for feed in eligible]) + if only_missing + else set() + ) + selected = [feed for feed in eligible if feed.id not in already_backfilled] + + _validate_requested_feed_ids( + stable_feed_ids, + found={feed.stable_id for feed in feeds}, + evaluated={feed.stable_id for feed in eligible}, + ) + + windows = { + feed.id: (march_start_for(feed, window_start), window_end) for feed in selected + } + longest_march = _longest_march(windows) + if simulation: + check_simulation_fits(simulation, longest_march) + + feed_plans = [ + { + "stable_id": feed.stable_id, + "march_start": windows[feed.id][0].isoformat(), + "end_date": window_end.isoformat(), + "days": (window_end - windows[feed.id][0]).days + 1, + # Also Stable's anchor, and what created_at gets on insert. + "tracking_start": windows[feed.id][0].isoformat(), + } + for feed in selected + ] + + partial_run = is_partial_run(evaluators) + + report = { + "message": ( + f"{'Planned' if dry_run else 'Ran'} a backfill of {len(selected)} feed(s) " + f"across {len(evaluators)} criterion/criteria, ending " + f"{window_end.isoformat()}." + ), + "dry_run": dry_run, + "start_date": window_start.isoformat(), + "end_date": window_end.isoformat(), + "days": longest_march, + "total_feeds": len(selected), + "skipped_already_backfilled": len(already_backfilled), + "criteria": [evaluator.name.value for evaluator in evaluators], + "partial_run": partial_run, + "only_missing": only_missing, + "snapshot_mode": snapshot_mode, + "resume_from_snapshot": resume_from_snapshot, + "batch_size": batch_size, + "criterion_rows_written": 0, + "snapshot_rows_written": 0, + "seals_granted": 0, + "seals_revoked": 0, + "seals_after_run": 0, + "granted_stable_ids": [], + "revoked_stable_ids": [], + } + + # A plain dry run stops at the plan. One asked to simulate or trace has to march — + # that is the whole point — so it marches with writing suppressed. + inspecting = bool(simulation) or trace + if not dry_run or inspecting: + outcomes: List[dict] = [] + trace_rows: List[dict] = [] + for batch in batched(selected, batch_size): + result = _march( + db_session, + batch, + windows, + evaluators, + window_end, + "none" if dry_run else snapshot_mode, + resume_from_snapshot, + partial_run, + simulation=simulation, + trace=trace, + write=not dry_run, + ) + if not dry_run: + report["criterion_rows_written"] += result["criterion_rows"] + report["snapshot_rows_written"] += result["snapshot_rows"] + outcomes.extend(result["outcomes"]) + trace_rows.extend(result["trace"]) + if trace: + report["trace"] = collapse_runs(trace_rows) + # Counted on the marched days, not on the collapsed entries: the cap is what the + # march stopped recording, and collapsing happens after. + report["trace_truncated"] = len(trace_rows) >= MAX_TRACE_ROWS + if simulation: + report["simulated"] = { + criterion: forced.as_reported() + for criterion, forced in simulation.items() + } + if not dry_run: + # The only provenance that exists: the response says the rows are fabricated, + # because the rows themselves cannot. + report["simulated_write"] = True + + granted = [outcome for outcome in outcomes if outcome["granted"]] + # Only reachable with only_missing=False, but reported anyway so the monitor's + # aggregate reads the same keys from both fan-outs. + revoked = [outcome for outcome in outcomes if outcome["revoked"]] + report["seals_granted"] = len(granted) + report["seals_revoked"] = len(revoked) + report["seals_after_run"] = sum( + 1 for outcome in outcomes if outcome["has_seal"] + ) + report["granted_stable_ids"] = [outcome["stable_id"] for outcome in granted] + report["revoked_stable_ids"] = [outcome["stable_id"] for outcome in revoked] + + if partial_run: + report["note"] = ( + "Partial criteria run: has_seal was not recalculated because the criteria " + "that were not evaluated cannot be judged." + ) + + report["elapsed_seconds"] = round(clock.monotonic() - started, 2) + report["feeds"] = feed_plans[:max_reported_feeds] + report["feeds_omitted"] = max(0, len(feed_plans) - max_reported_feeds) + + # Without `feeds`: Cloud Logging drops a LogEntry over 256 KB. + logger.info( + "Backfill %s: %s", + "plan" if dry_run else "complete", + {key: value for key, value in report.items() if key != "feeds"}, + ) + + return report diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_orchestrator.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_orchestrator.py new file mode 100644 index 000000000..1806e80a7 --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_orchestrator.py @@ -0,0 +1,184 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Cloud Tasks producer: fan the Seal of Reliability backfill out across the catalog (#1763). + +Enumerates the catalog and chunks it for `backfill_seal_of_reliability`, which only marches +an explicit list. The mechanism is `fanout.plan_fanout`, shared with the nightly producer. + +Three differences from the nightly run, all because a march is long where a nightly +evaluation is one day: `end_date` is resolved here once and passed to every worker (or two +workers either side of midnight would end on different days); batches are smaller and the +deadline longer; and `only_missing` is the eligibility predicate rather than a worker-side +filter, so a feed the nightly job owns never has a simulation written over it. + +Payload (all optional):: + + { + "dry_run": bool, # default True + "batch_size": int, # default 100 + "start_date": str | None, # ISO date, default end_date - days_back + "end_date": str | None, # ISO date, default yesterday UTC + "days_back": int, # default 365 + "criteria": [str] | None, # default None (every implemented criterion) + "limit": int | None, # cap total feeds considered, default None + "stable_feed_ids": [str] | None, # restrict eligibility to these ids, default None + "only_missing": bool, # default True + "snapshot_mode": str, # final | all | none, default final + "resume_from_snapshot": bool, # default False + "deadline_seconds": int, # default 7200 (2h) + "monitor_delay_seconds": int, # default 300 + } +""" + +import logging +from typing import Any, Dict, List, Optional + +from shared.database.database import with_db_session + +from tasks.seal_of_reliability.backfill.backfill_seal_of_reliability import _parse_day +from tasks.seal_of_reliability.backfill.seal_backfill import ( + DEFAULT_DAYS_BACK, + DEFAULT_SNAPSHOT_MODE, + SNAPSHOT_MODES, + resolve_window, +) +from tasks.seal_of_reliability.context import ( + count_eligible_feeds, + iter_eligible_stable_ids, +) +from tasks.seal_of_reliability.fanout import FanoutSpec, plan_fanout + +logger = logging.getLogger(__name__) + +# Distinct from the nightly run's, so the two never share a tracker. +SEAL_BACKFILL_TASK_NAME = "seal_backfill_run" + +# Smaller than the nightly 250: per-batch cost scales with days as well as feeds. +DEFAULT_BATCH_SIZE = 100 +DEFAULT_DEADLINE_SECONDS = 2 * 60 * 60 # 2h wall-clock cap for a run +DEFAULT_MONITOR_DELAY_SECONDS = 300 + +# The monitor is shared with the nightly run, so it has to be told which tracker to settle. +SPEC = FanoutSpec( + task_name=SEAL_BACKFILL_TASK_NAME, + worker_task="seal_backfill_worker", + run_id_prefix="seal-backfill", + task_prefix="seal-backfill", + log_name="seal_backfill_orchestrator", + monitor_extra={"task_name": SEAL_BACKFILL_TASK_NAME}, +) + + +def seal_backfill_orchestrator_handler(payload: dict) -> dict: + """Entry point for the `seal_backfill_orchestrator` task.""" + payload = payload or {} + snapshot_mode = payload.get("snapshot_mode", DEFAULT_SNAPSHOT_MODE) + if snapshot_mode not in SNAPSHOT_MODES: + raise ValueError( + f"Unknown snapshot_mode {snapshot_mode!r}. Known modes: {list(SNAPSHOT_MODES)}" + ) + + # Resolved here, so a bad date fails once at the producer rather than once per batch. + window_start, window_end = resolve_window( + _parse_day(payload.get("start_date"), "start_date"), + _parse_day(payload.get("end_date"), "end_date"), + int(payload.get("days_back", DEFAULT_DAYS_BACK)), + ) + + return _plan_run( + dry_run=bool(payload.get("dry_run", True)), + batch_size=int(payload.get("batch_size", DEFAULT_BATCH_SIZE)), + window_start=window_start, + window_end=window_end, + criteria=payload.get("criteria"), + limit=payload.get("limit"), + stable_feed_ids=payload.get("stable_feed_ids"), + only_missing=bool(payload.get("only_missing", True)), + snapshot_mode=snapshot_mode, + resume_from_snapshot=bool(payload.get("resume_from_snapshot", False)), + deadline_seconds=int(payload.get("deadline_seconds", DEFAULT_DEADLINE_SECONDS)), + monitor_delay_seconds=int( + payload.get("monitor_delay_seconds", DEFAULT_MONITOR_DELAY_SECONDS) + ), + ) + + +@with_db_session +def _plan_run( + dry_run: bool, + batch_size: int, + window_start, + window_end, + criteria: Optional[List[str]], + limit: Optional[int], + stable_feed_ids: Optional[List[str]], + only_missing: bool, + snapshot_mode: str, + resume_from_snapshot: bool, + deadline_seconds: int, + monitor_delay_seconds: int, + db_session=None, +) -> Dict[str, Any]: + """Resolve the feeds to backfill, chunk them, and (unless dry_run) fan the run out.""" + window = { + "start_date": window_start.isoformat(), + "end_date": window_end.isoformat(), + } + settings = { + "only_missing": only_missing, + "snapshot_mode": snapshot_mode, + "resume_from_snapshot": resume_from_snapshot, + } + + plan = plan_fanout( + db_session, + SPEC, + count_feeds=lambda session: count_eligible_feeds( + session, + stable_feed_ids=stable_feed_ids, + limit=limit, + exclude_backfilled=only_missing, + ), + iter_batches=lambda session, size: iter_eligible_stable_ids( + session, + size, + stable_feed_ids=stable_feed_ids, + limit=limit, + exclude_backfilled=only_missing, + ), + build_worker_payload=lambda run_id, batch_id, ids: { + "run_id": run_id, + "batch_id": batch_id, + "stable_feed_ids": ids, + "criteria": criteria, + **window, + **settings, + }, + run_params=lambda run_started_at: { + "dry_run": False, + "batch_size": batch_size, + "criteria": criteria, + **window, + **settings, + "run_started_at": run_started_at, + "deadline_seconds": deadline_seconds, + }, + batch_size=batch_size, + dry_run=dry_run, + monitor_delay_seconds=monitor_delay_seconds, + ) + # Echoed back so an operator can see what a dry run resolved to. + return {**plan, **window, **settings} diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_worker.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_worker.py new file mode 100644 index 000000000..e19f629c3 --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_worker.py @@ -0,0 +1,121 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Cloud Tasks worker: march one batch of the Seal of Reliability backfill (#1763). + +One task per batch, enqueued by `seal_backfill_orchestrator`. Calls `backfill_seals` for its +slice and reports to the shared `TaskExecutionTracker`. + +Redelivery is safe: `backfill_seals` upserts, the same window over the same sources gives the +same final state, and `created_at` is insert-only. Both dates arrive explicit — the window +belongs to the run, not to when a batch happened to execute. + +Payload:: + + { + "run_id": str, # required — TaskExecutionTracker run id + "batch_id": str, # required — e.g. "batch-0003" + "stable_feed_ids": [str], # required, non-empty + "start_date": str, # required, ISO date + "end_date": str, # required, ISO date + "criteria": [str] | None, # optional + "only_missing": bool, # optional, default True + "snapshot_mode": str, # optional, default final + "resume_from_snapshot": bool # optional, default False + } +""" + +import logging +from typing import Optional + +from shared.database.database import with_db_session +from shared.helpers.task_execution.task_execution_tracker import TaskExecutionTracker + +from tasks.seal_of_reliability.backfill.backfill_seal_of_reliability import _parse_day +from tasks.seal_of_reliability.backfill.seal_backfill import ( + DEFAULT_SNAPSHOT_MODE, + backfill_seals, +) +from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + SEAL_BACKFILL_TASK_NAME, +) + +logger = logging.getLogger(__name__) + + +def seal_backfill_worker_handler(payload: dict) -> dict: + """Entry point for the `seal_backfill_worker` task.""" + payload = payload or {} + run_id = payload.get("run_id") + batch_id = payload.get("batch_id") + stable_feed_ids = payload.get("stable_feed_ids") + if not run_id or not batch_id: + raise ValueError("run_id and batch_id are required") + if not stable_feed_ids: + raise ValueError("stable_feed_ids is required and must be non-empty") + + start_date = _parse_day(payload.get("start_date"), "start_date") + end_date = _parse_day(payload.get("end_date"), "end_date") + if start_date is None or end_date is None: + # Defaulting here could march to a different final day than a sibling batch. + raise ValueError("start_date and end_date are required") + + try: + result = backfill_seals( + stable_feed_ids=stable_feed_ids, + start_date=start_date, + end_date=end_date, + dry_run=False, + criteria=payload.get("criteria"), + only_missing=bool(payload.get("only_missing", True)), + snapshot_mode=payload.get("snapshot_mode", DEFAULT_SNAPSHOT_MODE), + resume_from_snapshot=bool(payload.get("resume_from_snapshot", False)), + ) + except ( + Exception + ) as error: # infra failure, or every id in the batch turned ineligible + logger.exception( + "seal_backfill_worker failed for run=%s batch=%s", run_id, batch_id + ) + _mark_entry(run_id, batch_id, error=str(error)) + raise + + _mark_entry(run_id, batch_id, result=result) + return {"status": "ok", "batch_id": batch_id, **result} + + +@with_db_session +def _mark_entry( + run_id: str, + batch_id: str, + result: Optional[dict] = None, + error: Optional[str] = None, + db_session=None, +) -> None: + """Record this batch in the run's tracker. + + The stored metadata is what `seal_orchestrator_monitor` aggregates, so its keys must + survive here. + """ + tracker = TaskExecutionTracker( + task_name=SEAL_BACKFILL_TASK_NAME, + run_id=run_id, + db_session=db_session, + ) + if error is None: + tracker.mark_completed(batch_id, metadata=result) + else: + tracker.mark_failed(batch_id, error_message=error) + db_session.commit() diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/simulation.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/simulation.py new file mode 100644 index 000000000..4fdcbede8 --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/simulation.py @@ -0,0 +1,463 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Forced per-day statuses and the day-by-day trace, for inspecting a backfill march. + +A simulated verdict in `seal_criterion` would be indistinguishable from an earned one — the +row carries no provenance — so writing one is refused by default, and a traced dry run +marches with writing suppressed. `check_simulated_write_allowed` holds the exception and its +conditions: an explicit payload flag, an environment on the allowlist, and not the production +tunnel port. Everything that decides whether a fabricated status may reach the tables lives in +this module, so the rule can be read in one place. + +Day offsets are counted from each feed's own march start, so day 0 is that feed's first +evaluated day: the one denied a grace period. Anchoring to the run's `start_date` instead +would point at days a younger feed never marched. +""" + +import logging +import os +from dataclasses import dataclass, field, fields as dataclass_fields +from datetime import timedelta +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +from sqlalchemy.orm import Session + +from shared.common.seal_criteria import CriterionStatus +from tasks.seal_of_reliability.evaluators.base import CriterionObservation +from tasks.seal_of_reliability.state_machine import SealCriterionState, phase + +logger = logging.getLogger(__name__) + +# Every field of the state a day leaves behind, minus the two that identify it — those are +# already on the row as `stable_id` and `criterion`. Taken from the dataclass rather than +# listed, so a field added to SealCriterionState shows up in the trace without touching +# this file, which is the same trick `seal_updater._snapshot_row` uses for the snapshots. +TRACED_STATE_FIELDS: Tuple[str, ...] = tuple( + f.name + for f in dataclass_fields(SealCriterionState) + if f.name not in ("feed_id", "criterion") +) + +# Fields that advance on their own inside a stretch where nothing actually happened, and so +# must not break a run when collapsing. `day` and `evaluated_at` move every day; +# `last_verdict_at` moves on every verdict; and during a confirmed failure streak +# `last_observed_failure_at`, `last_confirmed_failure_at` and `probation_start` are all +# re-stamped daily — probation_start to tomorrow, which is why the longest runs would never +# collapse if it counted. `reason` is prose and differs per simulated day. +# +# Everything else is part of the signature, so a field added to SealCriterionState breaks +# runs by default rather than being silently ignored: too eager beats invisible. +TICKING_FIELDS: frozenset = frozenset( + { + "day", + "evaluated_at", + "last_verdict_at", + "last_observed_failure_at", + "last_confirmed_failure_at", + "probation_start", + "reason", + } +) + +# Cap on the trace a single call returns. A year x a batch of feeds x six criteria would be +# a response no one can read and Cloud Logging would drop; a simulation is a small thing. +MAX_TRACE_ROWS: int = 2000 + +# Payload keys inside a criterion that do something other than name days. Status names are a +# closed set, so there is no collision. +GRACE_KEY = "grace_days" +PROBATION_KEY = "probation_days" +DEFAULT_KEY = "default" +_RESERVED_KEYS = (DEFAULT_KEY, GRACE_KEY, PROBATION_KEY) + +# Distinguishes "not given" — use the evaluator's own value — from an explicit null, which +# means the criterion has no such period. +_UNSET = object() + + +@dataclass(frozen=True) +class CriterionSimulation: + """What a payload sets for one criterion: a baseline, named days, and optionally policy. + + A scenario is usually "this criterion holds one status, except on these days". `default` + is that baseline, and the named days are the exceptions on top of it. Without a baseline + the unnamed days fall through to the real evaluator, which is only useful where the + source data says something: locally `fresh_coverage` has no dataset history to read and + every unnamed day comes back UNKNOWN, so a scenario built purely from exceptions never + leaves `never_evaluated`. + + Overriding the periods matters for a criterion that has none — `official` and `stable` + are point-in-time checks, so a forced failure confirms the same day and the trace shows + nothing about debouncing. `fresh_coverage` ships with 14 days of grace and 180 of + probation, so it needs no lending to exercise either. + + Only ever applied to a dry run, so no fabricated status or policy can reach the seal + tables. + """ + + days: Mapping[int, CriterionStatus] = field(default_factory=dict) + baseline: Optional[CriterionStatus] = None + grace_period: Optional[timedelta] = None + probation_period: Optional[timedelta] = None + grace_overridden: bool = False + probation_overridden: bool = False + + def status_on(self, offset: int) -> Optional[CriterionStatus]: + """The status this payload forces on `offset`, or None to ask the evaluator.""" + forced = self.days.get(offset) + return forced if forced is not None else self.baseline + + def grace_for(self, evaluator) -> Optional[timedelta]: + return self.grace_period if self.grace_overridden else evaluator.grace_period + + def probation_for(self, evaluator) -> Optional[timedelta]: + return ( + self.probation_period + if self.probation_overridden + else evaluator.probation_period + ) + + def as_reported(self) -> dict: + """The echo returned in the report, so a run states what it was told to pretend. + + Offsets are stringified rather than left as ints: they share the dict with + `grace_days` and `probation_days`, and Flask's jsonify sorts keys, which raises on a + dict mixing str and int. JSON object keys are strings anyway, so the response shape + is unchanged. + """ + echo: Dict[str, Any] = { + str(offset): status.value for offset, status in sorted(self.days.items()) + } + if self.baseline is not None: + echo[DEFAULT_KEY] = self.baseline.value + if self.grace_overridden: + echo[GRACE_KEY] = ( + self.grace_period.days if self.grace_period is not None else None + ) + if self.probation_overridden: + echo[PROBATION_KEY] = ( + self.probation_period.days + if self.probation_period is not None + else None + ) + return echo + + +def policy_for(evaluator, simulation) -> tuple: + """The (grace, probation) this run applies to `evaluator` — its own unless overridden.""" + forced = (simulation or {}).get(evaluator.name.value) + if forced is None: + return evaluator.grace_period, evaluator.probation_period + return forced.grace_for(evaluator), forced.probation_for(evaluator) + + +def _parse_period(value, criterion: str, key: str) -> Optional[timedelta]: + """A whole number of days, or null meaning the criterion has no such period.""" + if value is None: + return None + try: + days = int(value) + except (TypeError, ValueError): + raise ValueError( + f"{key} for {criterion!r} must be a whole number of days or null, got {value!r}" + ) + if days < 0: + raise ValueError( + f"{key} for {criterion!r} cannot be negative; got {days}. Use null to mean the " + f"criterion has no such period." + ) + return timedelta(days=days) + + +def parse_simulation( + simulate: Optional[dict], evaluators: Sequence +) -> Dict[str, CriterionSimulation]: + """Turn the `simulate` payload into criterion -> CriterionSimulation. + + Shape, offsets counted from each feed's own march start:: + + {"fresh_coverage": {"default": "pass", "fail": [3, 4], "unknown": [8]}} + + Offsets rather than dates because a scenario is about the shape of a history — "it fails + on day 3" — not about a calendar. Anchoring per feed rather than to the window start is + what makes day 0 the feed's first evaluation, the one denied a grace period. + + `default` is the status every unnamed day takes. Omit it and unnamed days fall through to + the real evaluator instead, which is what you want when the source data has something to + say and useless when it does not. + + `grace_days` and `probation_days` are optional and override the criterion's own values + for the run. Omit either to keep the evaluator's; pass null to mean it has none. + """ + if not simulate: + return {} + + known = {evaluator.name.value for evaluator in evaluators} + forced_statuses = { + status.value for status in CriterionStatus if status.is_verdict + } | { + CriterionStatus.UNKNOWN.value, + CriterionStatus.NOT_APPLICABLE.value, + } + + parsed: Dict[str, CriterionSimulation] = {} + for criterion, by_status in simulate.items(): + if criterion not in known: + raise ValueError( + f"Cannot simulate unknown criterion {criterion!r}. This run evaluates: " + f"{sorted(known)}" + ) + by_status = dict(by_status or {}) + grace = by_status.pop(GRACE_KEY, _UNSET) + probation = by_status.pop(PROBATION_KEY, _UNSET) + baseline = by_status.pop(DEFAULT_KEY, None) + if baseline is not None and baseline not in forced_statuses: + raise ValueError( + f"Cannot simulate {DEFAULT_KEY} status {baseline!r} for {criterion!r}. " + f"Valid: {sorted(forced_statuses)}" + ) + + days: Dict[int, CriterionStatus] = {} + for status, offsets in by_status.items(): + if status not in forced_statuses: + raise ValueError( + f"Cannot simulate status {status!r} for {criterion!r}. Valid: " + f"{sorted(forced_statuses)} — plus {list(_RESERVED_KEYS)}" + ) + for offset in offsets or []: + offset = int(offset) + if offset < 0: + raise ValueError( + f"Simulated day offsets are counted from the march start and cannot " + f"be negative; got {offset} for {criterion!r}" + ) + if offset in days and days[offset].value != status: + raise ValueError( + f"Day {offset} of {criterion!r} is simulated twice, as " + f"{days[offset].value!r} and {status!r}" + ) + days[offset] = CriterionStatus(status) + + parsed[criterion] = CriterionSimulation( + days=days, + baseline=(CriterionStatus(baseline) if baseline is not None else None), + grace_period=( + None if grace is _UNSET else _parse_period(grace, criterion, GRACE_KEY) + ), + probation_period=( + None + if probation is _UNSET + else _parse_period(probation, criterion, PROBATION_KEY) + ), + grace_overridden=grace is not _UNSET, + probation_overridden=probation is not _UNSET, + ) + return parsed + + +def check_simulation_fits( + simulation: Dict[str, Dict[int, CriterionStatus]], + longest_march: int, +) -> None: + """Reject offsets no march reaches, rather than letting them silently do nothing. + + A typo like day 400 in an eight-day window would otherwise look like it worked. + """ + if not longest_march: + # Nothing was selected, so blaming the offsets would send the reader to the wrong + # parameter entirely. `only_missing` excluding an already-backfilled feed is the + # usual cause. + raise ValueError( + "Nothing to simulate: no feed was selected for this run. If the feeds already " + "have seal state, only_missing (default true) excludes them — pass " + "only_missing=false to march them again." + ) + for criterion, forced in simulation.items(): + beyond = sorted(offset for offset in forced.days if offset >= longest_march) + if beyond: + raise ValueError( + f"Simulated day(s) {beyond} for {criterion!r} are past the end of every " + f"feed's march; the longest here is {longest_march} day(s), so valid " + f"offsets are 0..{max(longest_march - 1, 0)}" + ) + + +def observe(evaluator, ctx, simulation, offset: int) -> CriterionObservation: + """The criterion's own verdict, unless this day is simulated. + + A named day wins over the baseline, and with neither the day falls through to the real + evaluator — so a simulation ranges from real data with a couple of overrides to a wholly + synthetic history, depending on whether `default` was given. + """ + entry = simulation.get(evaluator.name.value) + forced = entry.status_on(offset) if entry else None + if forced is None: + return evaluator.evaluate(ctx) + named = entry.days.get(offset) is not None + return CriterionObservation( + criterion=evaluator.name, + observed_status=forced, + reason=( + f"simulated: {forced.value} on day {offset}" + if named + else f"simulated: {forced.value} by default" + ), + ) + + +def _as_day(value): + """Render a state value for the trace: statuses as their name, timestamps as their day. + + The march evaluates at midnight UTC, so a date loses nothing and reads better than a + full timestamp repeated down a year of rows. + """ + if isinstance(value, CriterionStatus): + return value.value + if hasattr(value, "date"): + return value.date().isoformat() + return value + + +def trace_row(feed, evaluator, offset: int, observation, state) -> dict: + """One day of one criterion: every seal_criterion field, plus where it came from. + + Carries the whole state rather than a summary, so a trace answers the same questions the + stored row would — when the current streak began, when a verdict was last reached — and + a reader never has to run the march again to see a field that was left out. + + Flask's jsonify sorts keys, so the order here is for reading the source, not the + response. + """ + row = { + "stable_id": feed.stable_id, + "criterion": evaluator.name.value, + "day": offset, + # No `date`: `evaluated_at` is the same day by construction, since the march + # evaluates every criterion once per day and `transition` stamps it every time. + "phase": phase(state).value, + "simulated": observation.reason.startswith("simulated:"), + "reason": observation.reason, + } + for name in TRACED_STATE_FIELDS: + row[name] = _as_day(getattr(state, name)) + return row + + +def _signature(row: dict) -> tuple: + """What makes a day different from the one before it, ignoring the ticking fields.""" + return tuple( + sorted((key, value) for key, value in row.items() if key not in TICKING_FIELDS) + ) + + +def collapse_runs(rows: Sequence[dict]) -> List[dict]: + """Collapse consecutive days in which nothing changed into one entry per run. + + A year of trace is mostly repetition — a criterion sits in one situation for weeks. Each + run is reported as its first day, its last day, and how many days sat between them, so + the boundaries stay exact while the middle collapses. + + Rows are grouped by feed and criterion first: the march emits them day-major, so + consecutive entries in the flat list are different feeds, not consecutive days. + """ + grouped: Dict[Tuple[str, str], List[dict]] = {} + for row in rows: + grouped.setdefault((row["stable_id"], row["criterion"]), []).append(row) + + collapsed: List[dict] = [] + for series in grouped.values(): + series.sort(key=lambda row: row["day"]) + run: List[dict] = [] + for row in series: + if run and _signature(run[-1]) == _signature(row): + run.append(row) + continue + if run: + collapsed.append(_as_run(run)) + run = [row] + if run: + collapsed.append(_as_run(run)) + return collapsed + + +def _as_run(run: Sequence[dict]) -> dict: + """One unchanged stretch: its first day, its last, and the count between them.""" + entry = {"days": len(run), "first": run[0]} + if len(run) > 1: + entry["last"] = run[-1] + entry["in_between"] = len(run) - 2 + return entry + + +# Environments in which a forced verdict may be written to the seal tables. Deliberately a +# closed list rather than "anything but prod": an ENVIRONMENT that is unset or misspelled +# refuses, so a deployment that forgets to set it cannot fabricate seal history. dev and qa are +# both in, by decision — they are where scenarios get exercised. Note dev and qa share one +# Cloud SQL instance, so a simulated write in either is one database name away from the other's +# data; the response flag and the warning log are the only marks a fabricated row leaves. +SIMULATED_WRITE_ENVIRONMENTS: Tuple[str, ...] = ("local", "dev", "qa", "test") + +# The local port production is reached on when a tunnel is up, by team convention. Refusing it +# is a guard rail, not a guarantee: the port is a property of how the tunnel was started rather +# than of the database, so a tunnel opened on another port walks straight past this. It is here +# because ENVIRONMENT describes the process, not the write target — a local run labelled `local` +# can be pointed at production through a tunnel and would otherwise pass every other check. The +# actual control is connecting to production as a read-only user. +PROD_TUNNEL_PORT: int = 9901 + + +def _connection_port(db_session: Session) -> Optional[int]: + """The port this session is connected on, or None if it cannot be determined.""" + try: + return db_session.get_bind().url.port + except Exception: # defensive: never let the guard rail itself break a run + logger.warning( + "Could not determine the database port for the simulated-write check" + ) + return None + + +def check_simulated_write_allowed(simulate: dict, db_session: Session) -> None: + """Refuse to write forced verdicts anywhere they could be mistaken for real history. + + A simulated verdict in `seal_criterion` is indistinguishable from an earned one — the row + carries no provenance — so where it may be written is decided here, by the deployment + rather than by the payload. Two conditions, both about the target: the environment has to + be one where fabricated data is expected, and the connection must not be production's + tunnel port. + """ + environment = os.getenv("ENVIRONMENT", "").strip().lower() + if environment not in SIMULATED_WRITE_ENVIRONMENTS: + raise ValueError( + f"a simulated write is refused with ENVIRONMENT={environment or 'unset'!r}: " + f"forced verdicts may only be written in {list(SIMULATED_WRITE_ENVIRONMENTS)}, " + f"and an unset environment is treated as production." + ) + + port = _connection_port(db_session) + if port == PROD_TUNNEL_PORT: + raise ValueError( + f"a simulated write is refused on port {PROD_TUNNEL_PORT}: that is the " + f"production database's tunnel port, whatever ENVIRONMENT={environment!r} claims." + ) + + logger.warning( + "SIMULATED WRITE in ENVIRONMENT=%s on port %s: forced statuses for %s are being " + "written to the seal tables. These rows are indistinguishable from earned ones.", + environment, + port, + sorted(simulate), + ) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py index 71380816d..f4be533ea 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py @@ -18,39 +18,39 @@ The evaluators are pure functions over a `FeedSealContext`, so every DB read for a batch of feeds happens here, in a fixed number of queries regardless of batch size. -Official, Stable and Fresh (future coverage) are implemented. Official and Stable read the -feed row alone; Fresh needs one bulk-loaded extra, the feed's latest dataset. Each new -criterion adds the fields it needs here plus, where they are not already on the feed row, one -bulk query to populate them: the latest dataset's validation report for Compliant, the day's -availability rows for Available, the full dataset coverage history for Fresh continuous -coverage. +A criterion's inputs reach it one of two ways, and the split is deliberate: + +* Day-invariant feed facts — `official`, `seasonal`, `is_producer_url_unstable`, + `created_at` — are fields on `FeedSealContext`, read straight off the feed row the caller + has already loaded. They cost no query and they have no history to read: the same value + answers every day of a backfill. Official and Stable need nothing else. +* Anything that varies by day is loaded by the criterion itself, through + `CriterionEvaluator.load_inputs`. Only the criterion knows what its own inputs look like, + and keeping that knowledge there is what stops this module from having to grow a field + and a query for every criterion added. Fresh (future coverage) is the first of these — the + feed's latest dataset is a different row on each day a march evaluates — and the day's + availability rows for Available, the latest validation report for Compliant and the full + coverage history for Fresh continuous coverage all belong there too. + +Official, Stable and Fresh (future coverage) are implemented; Available and Compliant are +the rest of #1784, and Fresh / continuous coverage is tracked by #1782. """ import itertools -from dataclasses import dataclass -from datetime import datetime -from typing import Dict, Iterator, List, Optional, Sequence +from dataclasses import dataclass, field +from datetime import date, datetime, timezone +from typing import Any, Dict, Iterator, List, Mapping, Optional, Sequence from sqlalchemy import select from sqlalchemy.orm import Session -from shared.database_gen.sqlacodegen_models import Feed, Gtfsdataset, Gtfsfeed - - -@dataclass(frozen=True) -class LatestDataset: - """ - The feed's latest dataset as of the run's `now`, and the fields criteria read off it. - """ - - dataset_id: str - downloaded_at: datetime - service_date_range_end: Optional[datetime] = None +from shared.common.seal_criteria import SealCriterionName +from shared.database_gen.sqlacodegen_models import Feed, Gtfsfeed, SealCriterion @dataclass class FeedSealContext: - """Everything the evaluators need for one feed. + """Everything the evaluators need for one feed on one day. Built by `build_contexts`. Evaluators read from this and never query. """ @@ -69,8 +69,22 @@ class FeedSealContext: # Stable: when the feed was first added to the database. feed_created_at: Optional[datetime] = None - # The feed's latest dataset as of `now` - resolved by `downloaded_at` vs `now` - latest_dataset: Optional[LatestDataset] = None + # Each criterion's own bulk-loaded inputs, keyed by criterion name — see + # `collect_inputs`. Opaque here: this module never looks inside a criterion's payload, + # and an evaluator reaches only its own through `inputs_for(self.name)`. + # + # The whole batch's inputs are shared by reference across every context, rather than + # sliced per feed and per day. Slicing would force every criterion into one storage + # shape, and it would copy a year of history once per (feed, day) during a backfill. + inputs: Mapping[SealCriterionName, Any] = field(default_factory=dict) + + def inputs_for(self, criterion: SealCriterionName) -> Any: + """This criterion's loaded inputs, or None if its loader returned nothing. + + None is the normal answer for a criterion whose inputs are day-invariant fields on + this context, so it means "nothing to load", not "the load failed". + """ + return self.inputs.get(criterion) # Feeds in these statuses, or not published, are never eligible for the seal. @@ -94,13 +108,20 @@ def is_seal_eligible(feed) -> bool: def _eligible_stable_ids_query( - db_session: Session, stable_feed_ids: Optional[Sequence[str]] = None + db_session: Session, + stable_feed_ids: Optional[Sequence[str]] = None, + exclude_backfilled: bool = False, ): """Base query: `stable_id` of every seal-eligible GTFS feed. `stable_feed_ids`, if given, narrows the candidate set without changing the predicate. Left as `None`, every eligible feed in the catalog is returned; this is what the seal orchestrator (issue #1800) uses to enumerate the full batch to fan out. + + `exclude_backfilled` drops feeds that already have seal state, which is the backfill + producer's candidate set (#1763): a feed the nightly job already owns has real history + to carry forward and must not have a simulation written over it. It lives here rather + than in the producer so both the count and the stream apply one predicate. """ query = db_session.query(Gtfsfeed.stable_id).filter( Feed.status.notin_(INELIGIBLE_STATUSES), @@ -108,6 +129,11 @@ def _eligible_stable_ids_query( ) if stable_feed_ids is not None: query = query.filter(Feed.stable_id.in_(list(stable_feed_ids))) + if exclude_backfilled: + has_state = select(SealCriterion.__table__.c.feed_id).where( + SealCriterion.__table__.c.feed_id == Feed.id + ) + query = query.filter(~has_state.exists()) return query @@ -115,9 +141,14 @@ def count_eligible_feeds( db_session: Session, stable_feed_ids: Optional[Sequence[str]] = None, limit: Optional[int] = None, + exclude_backfilled: bool = False, ) -> int: """Cheap `COUNT(*)` of eligible feeds — no rows loaded.""" - query = _eligible_stable_ids_query(db_session, stable_feed_ids=stable_feed_ids) + query = _eligible_stable_ids_query( + db_session, + stable_feed_ids=stable_feed_ids, + exclude_backfilled=exclude_backfilled, + ) if limit is not None: query = query.limit(limit) return query.count() @@ -128,6 +159,7 @@ def iter_eligible_stable_ids( batch_size: int, stable_feed_ids: Optional[Sequence[str]] = None, limit: Optional[int] = None, + exclude_backfilled: bool = False, ) -> Iterator[List[str]]: """Stream eligible feeds' `stable_id`s in chunks of at most `batch_size`. @@ -138,7 +170,11 @@ def iter_eligible_stable_ids( if batch_size <= 0: raise ValueError("batch_size must be a positive integer") query = ( - _eligible_stable_ids_query(db_session, stable_feed_ids=stable_feed_ids) + _eligible_stable_ids_query( + db_session, + stable_feed_ids=stable_feed_ids, + exclude_backfilled=exclude_backfilled, + ) .order_by(Gtfsfeed.stable_id) .execution_options(stream_results=True) ) @@ -152,87 +188,84 @@ def iter_eligible_stable_ids( yield chunk -def _load_latest_datasets( - db_session: Session, feed_ids: Sequence[str], now: datetime -) -> Dict[str, LatestDataset]: - """feed_id -> the feed's latest dataset as of `now`, for feeds that had one. +def snapshot_date_of(now: datetime) -> date: + """The UTC day a run evaluating at `now` belongs to. + + The day a run's snapshots are keyed under, and the day its criteria are evaluated + against. Naive values are read as UTC rather than rejected: the task entry points + normalize what an operator passes, but `update_seals` is also called directly. + """ + if now.tzinfo is None: + return now.date() + return now.astimezone(timezone.utc).date() - "Latest as of `now`" is the most recently downloaded dataset with - `downloaded_at <= now`. - A feed missing from the result had no dataset at all as of `now`. That is deliberately - distinct from a `LatestDataset` whose `service_date_range_end` is None, which had one - whose coverage was never extracted - the criteria read both as UNKNOWN but report which. +def collect_inputs( + db_session: Session, + feeds: Sequence[Gtfsfeed], + days: Sequence[date], + evaluators: Sequence, +) -> Dict[SealCriterionName, Any]: + """Ask each evaluator to bulk-load its own day-varying inputs for the whole batch. + + Called once per batch whatever the number of days: a criterion loads its full history + for `days` in one go and answers each day from memory afterwards. That is what holds the + query count proportional to the number of criteria rather than to feeds x days — the + difference between a handful of queries and several thousand once a backfill marches a + year (#1763). + + Args: + db_session: SQLAlchemy session. + feeds: The batch of feeds, already loaded and eligibility-checked by the caller. + days: Every UTC day that will be evaluated, ascending. One entry for a nightly run. + evaluators: The `CriterionEvaluator` instances this run will apply. Not annotated as + such because `evaluators.base` imports this module for `FeedSealContext`. + + Returns: + criterion name -> whatever that criterion's loader returned. Opaque to this module; + an evaluator reaches its own with `ctx.inputs_for(self.name)`. """ - if not feed_ids: - return {} - rows = db_session.execute( - select( - Gtfsdataset.feed_id, - Gtfsdataset.id, - Gtfsdataset.downloaded_at, - Gtfsdataset.service_date_range_end, - ) - .where( - Gtfsdataset.feed_id.in_(list(feed_ids)), - Gtfsdataset.downloaded_at.is_not(None), - Gtfsdataset.downloaded_at <= now, - ) - .distinct(Gtfsdataset.feed_id) - .order_by( - Gtfsdataset.feed_id, - Gtfsdataset.downloaded_at.desc(), - Gtfsdataset.id.desc(), - ) - ).all() return { - row.feed_id: LatestDataset( - dataset_id=row.id, - downloaded_at=row.downloaded_at, - service_date_range_end=row.service_date_range_end, - ) - for row in rows + evaluator.name: evaluator.load_inputs(db_session, feeds, days) + for evaluator in evaluators } def build_contexts( - db_session: Session, feeds: Sequence[Gtfsfeed], now: datetime + db_session: Session, + feeds: Sequence[Gtfsfeed], + now: datetime, + evaluators: Sequence, ) -> Dict[str, FeedSealContext]: - """Load everything the evaluators need for `feeds`, in a fixed number of queries. + """Build one context per feed for a single day — the nightly run's case. + + This is `collect_inputs` over a one-day range, plus the day-invariant feed fields. A + backfill marching a year calls `collect_inputs` once for the whole range and then builds + its contexts per day from that same result, so both paths load a criterion's inputs + through the criterion itself and there is only ever one place they come from. Args: - db_session: SQLAlchemy session. - feeds: The batch of feeds to load, already loaded (and eligibility-checked via + db_session: SQLAlchemy session, passed on to the evaluators' loaders. + feeds: The batch of feeds to build for, already loaded (and eligibility-checked via `is_seal_eligible`) by the caller — `update_seals`. now: The evaluation timestamp. + evaluators: The evaluators this run will apply. Required rather than defaulted: an + omitted list would leave every criterion with no inputs and quietly turn its + verdicts into UNKNOWN. Returns: feed_id -> FeedSealContext. - How to add a criterion's data. Two kinds: + Adding a criterion's data. Two kinds: 1. Already on the selected feed row (`official`, `created_at`, `seasonal`, - `is_producer_url_unstable`). Add the field to FeedSealContext and read it off `feed` - below. No query, no cost. - - 2. Needs its own query. Add the field, then a module-level `_load_*` helper that takes - the whole batch and returns a dict keyed by feed_id, and call it once here, as - `_load_latest_datasets` does. Keeping the query per batch rather than per feed - is what holds the query count proportional to the number of criteria instead of the - number of feeds. For example, Available (issue #1784) would add: - - def _load_availability_today(db_session, feed_ids, day_start) -> Dict[str, bool]: - '''feed_id -> whether any availability check succeeded since day_start. - Feeds absent from the result had no check at all, which the criterion reads - as "not evaluable" rather than "failing".''' - - called once as `availability = _load_availability_today(...)` and consumed per feed - as `availability_success_today=availability.get(feed.id, False)`. - """ - latest_datasets = _load_latest_datasets( - db_session, [feed.id for feed in feeds], now - ) + `is_producer_url_unstable`). Add the field to `FeedSealContext` and read it off `feed` + below. No query, no cost, and it answers for any day. + 2. Varies by day. Nothing changes here: override `load_inputs` on the criterion's own + evaluator and read it back in `_evaluate` with `ctx.inputs_for(self.name)`. + """ + inputs = collect_inputs(db_session, feeds, [snapshot_date_of(now)], evaluators) return { feed.id: FeedSealContext( feed_id=feed.id, @@ -242,7 +275,7 @@ def _load_availability_today(db_session, feed_ids, day_start) -> Dict[str, bool] is_producer_url_unstable=feed.is_producer_url_unstable, seasonal=feed.seasonal, feed_created_at=feed.created_at, - latest_dataset=latest_datasets.get(feed.id), + inputs=inputs, ) for feed in feeds } diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py index 349f33311..d2c9b18fc 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py @@ -17,10 +17,12 @@ `EVALUATORS` is the registry the job iterates. Official (issue #1783), Stable and Fresh / future coverage (issue #1784) are implemented; Available and Compliant are the rest of -#1784, and Fresh / continuous coverage is tracked by #1782. Adding one means a new subclass, -an entry here, and whatever fields it needs on `FeedSealContext`. Its windows are not declared -on the subclass: they come from the policy maps in `shared.common.seal_criteria`, which the -read API reads too. +#1784, and Fresh / continuous coverage is tracked by #1782. + +Adding one means a new subclass and an entry here, plus — for whatever inputs it needs — +either a day-invariant field on `FeedSealContext`, or its own `load_inputs` override when the +inputs vary by day. Its windows are not declared on the subclass: they come from the policy +maps in `shared.common.seal_criteria`, which the read API reads too. `seal_criterion_name` in the database already declares all six values, so a criterion can be added without a schema change. diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py index 5436557d6..5ae037e21 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py @@ -16,8 +16,10 @@ """Base class for the per-criterion evaluators.""" from dataclasses import dataclass -from datetime import timedelta -from typing import Optional, Tuple +from datetime import date, timedelta +from typing import Any, Optional, Sequence, Tuple + +from sqlalchemy.orm import Session from shared.common.seal_criteria import ( CriterionStatus, @@ -55,8 +57,12 @@ class CriterionEvaluator: which the read API reads too, so a criterion cannot debounce one way for the job and another way for the API. To change a window, change the map. - Evaluators never touch the database: all the data they need is on the context, loaded in - bulk by `context.build_contexts`. + Evaluators never touch the database at evaluation time: everything they need is already + on the context, either as a day-invariant feed field or as the inputs their own + `load_inputs` bulk-loaded. + + A criterion that has to look backwards owns that lookup itself, rather than the context + builder growing a field and a query per criterion. `load_inputs` is where it goes. """ name: SealCriterionName = None @@ -78,6 +84,35 @@ def probation_period(self) -> Optional[timedelta]: """ return probation_period_for(self.name) + def load_inputs( + self, + db_session: Session, + feeds: Sequence, + days: Sequence[date], + ) -> Any: + """Bulk-load this criterion's day-varying inputs for a whole batch of feeds, at once. + + Returns an object of the criterion's own choosing — nothing outside the criterion + looks inside it. The caller stashes it on every context in the batch, and `_evaluate` + reads it back with `ctx.inputs_for(self.name)`, indexing by `ctx.feed_id` and the + day of `ctx.now`. + + The default returns None, which is the right answer for a criterion whose inputs are + day-invariant fields already on the context — Official and Stable read the feed row + and have nothing of their own to load. + + Override it for any criterion that does, and load the whole of `days` in one query + rather than one query per day: a nightly run passes a single day, but a backfill + (#1763) passes a year, and a per-day query there turns a handful of queries into + several thousand. + + Args: + db_session: SQLAlchemy session. + feeds: The batch of feeds to load for, already loaded by the caller. + days: Every UTC day that will be evaluated, ascending. + """ + return None + def evaluate(self, ctx: FeedSealContext) -> CriterionObservation: """Evaluate the criterion and label the result with this evaluator's name. diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/fresh_coverage.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/fresh_coverage.py index c932cb4e1..2f628986a 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/fresh_coverage.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/fresh_coverage.py @@ -15,27 +15,191 @@ # """Fresh (future coverage) criterion: the latest dataset still covers the near future.""" -from typing import Tuple +from bisect import bisect_right +from dataclasses import dataclass +from datetime import date, datetime, time, timedelta, timezone +from typing import Dict, List, Optional, Sequence, Tuple + +from sqlalchemy import select +from sqlalchemy.orm import Session from shared.common.seal_criteria import ( FUTURE_COVERAGE_HORIZON, CriterionStatus, SealCriterionName, ) +from shared.database_gen.sqlacodegen_models import Gtfsdataset from tasks.seal_of_reliability.context import FeedSealContext from tasks.seal_of_reliability.evaluators.base import CriterionEvaluator +@dataclass(frozen=True) +class LatestDataset: + """One dataset row, reduced to the fields Fresh reads off it.""" + + dataset_id: str + downloaded_at: datetime + service_date_range_end: Optional[datetime] = None + + +class FreshCoverageInputs: + """Every feed's dataset history over a run's day range, ready for as-of lookups. + + Holds the whole batch's history rather than one dataset per feed, because "the latest + dataset" is a different row on each day a backfill marches. Built once per batch by + `FreshCoverageEvaluator.load_inputs` and shared by reference across every context. + + The per-feed lists are sorted by `downloaded_at` and the keys are kept alongside them, so + `as_of` is a binary search rather than a scan: a year's march asks this once per feed per + day, which is where a linear lookup would start to cost real time. + """ + + def __init__(self, history: Dict[str, List[LatestDataset]]): + self._downloaded_at: Dict[str, List[datetime]] = {} + self._datasets: Dict[str, List[LatestDataset]] = {} + for feed_id, datasets in history.items(): + # `dataset_id` breaks ties the same way `_load` orders them, so two datasets + # stamped at the same instant resolve to one answer rather than an arbitrary one. + datasets.sort( + key=lambda dataset: (dataset.downloaded_at, dataset.dataset_id) + ) + self._datasets[feed_id] = datasets + self._downloaded_at[feed_id] = [ + dataset.downloaded_at for dataset in datasets + ] + + def as_of(self, feed_id: str, moment: datetime) -> Optional[LatestDataset]: + """The feed's most recently downloaded dataset at `moment`, or None if it had none. + + None means the feed had no dataset at all by then - deliberately distinct from a + `LatestDataset` whose `service_date_range_end` is None, which had one whose coverage + was never extracted. The criterion reads both as UNKNOWN but reports which. + """ + keys = self._downloaded_at.get(feed_id) + if not keys: + return None + index = bisect_right(keys, moment) + if index == 0: + return None + return self._datasets[feed_id][index - 1] + + class FreshCoverageEvaluator(CriterionEvaluator): """`latest dataset.service_date_range_end >= now + 7 days`. This is the only implemented criterion that can return NOT_APPLICABLE. A seasonal feed is expected to have coverage that runs out between seasons, so the question "does this feed cover the next week" has no meaningful answer for it. + + It is also the first criterion whose inputs vary by day, so it loads them itself through + `load_inputs` rather than through a field on `FeedSealContext`: which dataset is "the + latest" changes on every day a backfill marches (#1763). """ name = SealCriterionName.FRESH_COVERAGE + def load_inputs( + self, + db_session: Session, + feeds: Sequence, + days: Sequence[date], + ) -> Optional[FreshCoverageInputs]: + """Every dataset the batch's feeds had over `days`, plus the one each carried in. + + Two queries for the whole batch and the whole range, never one per day: + + 1. The carry-in - one row per feed, the latest dataset downloaded strictly before the + range opens. Without it the first day of a march would see no dataset at all for a + feed whose most recent one predates the range, and Fresh would read that as UNKNOWN. + 2. Everything downloaded inside the range, which is what makes later days differ from + earlier ones. + + Bounding query 2 by the range rather than loading a feed's whole history is what keeps + the row count proportional to the days actually being evaluated. + """ + if not feeds or not days: + return FreshCoverageInputs({}) + + feed_ids = [feed.id for feed in feeds] + # The march evaluates at each day's start, while a nightly run evaluates part-way + # through its day, so the window closes at the end of the last day either way. + range_start = datetime.combine(min(days), time.min, tzinfo=timezone.utc) + range_end = datetime.combine( + max(days), time.min, tzinfo=timezone.utc + ) + timedelta(days=1) + + history: Dict[str, List[LatestDataset]] = {} + for feed_id, dataset in self._carry_in( + db_session, feed_ids, range_start + ) + self._in_range(db_session, feed_ids, range_start, range_end): + history.setdefault(feed_id, []).append(dataset) + return FreshCoverageInputs(history) + + @staticmethod + def _columns(): + return ( + Gtfsdataset.feed_id, + Gtfsdataset.id, + Gtfsdataset.downloaded_at, + Gtfsdataset.service_date_range_end, + ) + + @classmethod + def _rows_to_datasets(cls, rows) -> List[Tuple[str, LatestDataset]]: + return [ + ( + row.feed_id, + LatestDataset( + dataset_id=row.id, + downloaded_at=row.downloaded_at, + service_date_range_end=row.service_date_range_end, + ), + ) + for row in rows + ] + + @classmethod + def _carry_in( + cls, db_session: Session, feed_ids: Sequence[str], range_start: datetime + ) -> List[Tuple[str, LatestDataset]]: + """One row per feed: its latest dataset from before the range opened.""" + rows = db_session.execute( + select(*cls._columns()) + .where( + Gtfsdataset.feed_id.in_(list(feed_ids)), + Gtfsdataset.downloaded_at.is_not(None), + Gtfsdataset.downloaded_at < range_start, + ) + .distinct(Gtfsdataset.feed_id) + .order_by( + Gtfsdataset.feed_id, + Gtfsdataset.downloaded_at.desc(), + Gtfsdataset.id.desc(), + ) + ).all() + return cls._rows_to_datasets(rows) + + @classmethod + def _in_range( + cls, + db_session: Session, + feed_ids: Sequence[str], + range_start: datetime, + range_end: datetime, + ) -> List[Tuple[str, LatestDataset]]: + """Every dataset downloaded while the range was open.""" + rows = db_session.execute( + select(*cls._columns()) + .where( + Gtfsdataset.feed_id.in_(list(feed_ids)), + Gtfsdataset.downloaded_at.is_not(None), + Gtfsdataset.downloaded_at >= range_start, + Gtfsdataset.downloaded_at < range_end, + ) + .order_by(Gtfsdataset.feed_id, Gtfsdataset.downloaded_at, Gtfsdataset.id) + ).all() + return cls._rows_to_datasets(rows) + def _evaluate(self, ctx: FeedSealContext) -> Tuple[CriterionStatus, str]: # Applicability is a property of the feed, so it is settled before the inputs are # looked at: a seasonal feed's missing dataset is not an UNKNOWN worth reporting. @@ -45,12 +209,24 @@ def _evaluate(self, ctx: FeedSealContext) -> Tuple[CriterionStatus, str]: "the feed is seasonal, so future coverage is not required", ) + inputs = ctx.inputs_for(self.name) + if inputs is None: + # Not a data condition: the context was built without running this criterion's + # loader. Said out loud in the reason rather than passed off as a missing dataset, + # because the two look identical in the stored row and only this one is a bug. + return ( + CriterionStatus.UNKNOWN, + "fresh_coverage inputs were never loaded for this run - the context was " + "built without calling load_inputs", + ) + # Two different missing inputs, kept apart so the report says which: no dataset at # all as of this run, or one whose coverage was never extracted. - if ctx.latest_dataset is None: + latest = inputs.as_of(ctx.feed_id, ctx.now) + if latest is None: return CriterionStatus.UNKNOWN, "the feed has no latest dataset" - coverage_end = ctx.latest_dataset.service_date_range_end + coverage_end = latest.service_date_range_end if coverage_end is None: return ( CriterionStatus.UNKNOWN, diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/fanout.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/fanout.py new file mode 100644 index 000000000..d0380a597 --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/fanout.py @@ -0,0 +1,298 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""The Cloud Tasks fan-out both seal producers share. + +The nightly run (#1800) and the backfill (#1763) differ in what they send a worker and which +feeds they select, but the mechanism between those two points is identical: count, chunk, +register the run, enqueue a worker per batch, reconcile the count against what the stream +actually yielded, enqueue one monitor. That reconciliation is the subtle part — it is what +stops a batch sitting `triggered` until the deadline — and having it in one place is the +reason this module exists. + +A producer supplies a `FanoutSpec` (the names and queues it uses) plus three callables: how +to count its feeds, how to stream them, and how to build a worker payload. Everything else is +here. +""" + +import json +import logging +import math +import os +import re +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional, Sequence + +from shared.database.database import with_db_session +from shared.helpers.task_execution.task_execution_tracker import TaskExecutionTracker + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class FanoutSpec: + """What distinguishes one producer's fan-out from the other's. + + `monitor_extra` is merged into the monitor's payload. The nightly run needs nothing + there; the backfill passes its tracker `task_name`, since the two share one monitor. + """ + + task_name: str # TaskExecutionTracker task_name for the run + worker_task: str # in-body task name of the worker to enqueue + run_id_prefix: str # run ids are "-" + task_prefix: str # Cloud Tasks names are "--" + log_name: str # what this producer calls itself in logs + queue_env: str = "SEAL_ORCHESTRATOR_QUEUE" + monitor_queue_env: str = "SEAL_ORCHESTRATOR_MONITOR_QUEUE" + monitor_task: str = "seal_orchestrator_monitor" + monitor_extra: Mapping[str, Any] = field(default_factory=dict) + + +def safe_task_name(name: str) -> str: + return re.sub(r"[^a-zA-Z0-9_-]", "-", name)[:500] + + +def enqueue_task( + *, + in_body_task: str, + payload: dict, + queue_env: str, + task_name: str, + schedule_seconds: int = 0, +) -> bool: + """Enqueue a Cloud Task targeting the tasks_executor function. + + Returns True on enqueue (or already-exists), False when misconfigured. + """ + project = os.getenv("PROJECT_ID") + queue = os.getenv(queue_env) + gcp_region = os.getenv("GCP_REGION") + environment = os.getenv("ENVIRONMENT") + if not all([project, queue, gcp_region, environment]): + logger.warning( + "enqueue_task: missing env (PROJECT_ID/GCP_REGION/ENVIRONMENT/%s) — " + "skipping enqueue of %s", + queue_env, + task_name, + ) + return False + + try: + from google.cloud import tasks_v2 + from google.protobuf import timestamp_pb2 + from shared.common.gcp_utils import create_http_task_with_name + + url = ( + f"https://{gcp_region}-{project}.cloudfunctions.net/" + f"tasks_executor-{environment}" + ) + body = json.dumps({"task": in_body_task, "payload": payload}).encode() + + schedule_time: Optional[Any] = None + if schedule_seconds > 0: + run_at = datetime.now(timezone.utc) + timedelta(seconds=schedule_seconds) + schedule_time = timestamp_pb2.Timestamp() + schedule_time.FromDatetime(run_at.replace(tzinfo=None)) + + create_http_task_with_name( + client=tasks_v2.CloudTasksClient(), + body=body, + url=url, + project_id=project, + gcp_region=gcp_region, + queue_name=queue, + task_name=task_name, + task_time=schedule_time, + http_method=tasks_v2.HttpMethod.POST, + ) + return True + except Exception as e: # pragma: no cover - network/env dependent + if "already exists" in str(e).lower() or "ALREADY_EXISTS" in str(e): + logger.info("enqueue_task: task %s already exists — skipping", task_name) + return True + logger.warning("enqueue_task: could not enqueue %s: %s", task_name, e) + return False + + +@with_db_session +def start_run( + task_name: str, + run_id: str, + batch_ids: List[str], + run_params: dict, + db_session=None, +) -> None: + """Register the run and one tracked entry per batch.""" + tracker = TaskExecutionTracker( + task_name=task_name, run_id=run_id, db_session=db_session + ) + tracker.start_run(total_count=len(batch_ids), params=run_params) + for batch_id in batch_ids: + tracker.mark_triggered(batch_id) + db_session.commit() + + +@with_db_session +def mark_enqueue_failed( + task_name: str, + run_id: str, + batch_id: str, + error_message: str = "enqueue failed", + db_session=None, +) -> None: + tracker = TaskExecutionTracker( + task_name=task_name, run_id=run_id, db_session=db_session + ) + tracker.mark_failed(batch_id, error_message=error_message) + db_session.commit() + + +def new_run_id(spec: FanoutSpec, started_at: datetime) -> str: + return f"{spec.run_id_prefix}-{started_at.strftime('%Y%m%dT%H%M%S')}" + + +def _reconcile( + spec: FanoutSpec, + run_id: str, + batch_ids: List[str], + consumed: int, + stream: Iterator[List[str]], +) -> None: + """Settle the difference between the plan-time count and what the stream yielded. + + Both directions come from the same cause: the count and the stream are separate queries, + so eligibility can move in the gap between them. + """ + if consumed < len(batch_ids): + # Fewer chunks than planned. The leftovers are already `triggered` from start_run + # and would otherwise sit there until the deadline failed the whole run. + missing = batch_ids[consumed:] + logger.error( + "%s: run=%s stream yielded %d batch(es), expected %d — marking %d failed: %s", + spec.log_name, + run_id, + consumed, + len(batch_ids), + len(missing), + missing, + ) + for batch_id in missing: + mark_enqueue_failed( + spec.task_name, + run_id, + batch_id, + error_message="no eligible-feed data for this batch (count/stream mismatch)", + ) + return + + # zip() with batch_ids (a list) first never calls next() on the stream for a final + # round once batch_ids is exhausted, so this reflects what is genuinely left over. + extra_chunk = next(stream, None) + if extra_chunk is not None: + # More chunks than planned: feeds became newly eligible in the gap. Log-only — + # self-healing would mean mutating total_count after start_run fixed it, for a + # race whose only consequence is a feed waiting for the next run. + logger.error( + "%s: run=%s stream had more batches than the plan-time count of %d expected " + "(>=%d additional feed(s) seen) — those feeds were not processed this run", + spec.log_name, + run_id, + len(batch_ids), + len(extra_chunk), + ) + + +def plan_fanout( + db_session, + spec: FanoutSpec, + *, + count_feeds: Callable[[Any], int], + iter_batches: Callable[[Any, int], Iterator[List[str]]], + build_worker_payload: Callable[[str, str, Sequence[str]], dict], + run_params: Callable[[str], dict], + batch_size: int, + dry_run: bool, + monitor_delay_seconds: int, +) -> Dict[str, Any]: + """Count, chunk, register, enqueue and reconcile. Returns the plan. + + `run_params` is a callable rather than a dict so a producer can fold in `run_started_at`, + which is decided here. + + On a dry run — or when nothing is eligible — nothing is registered and nothing is + enqueued, so the returned plan is purely informational. + """ + if batch_size <= 0: + raise ValueError("batch_size must be a positive integer") + + run_started_at = datetime.now(timezone.utc) + total_feeds = count_feeds(db_session) + num_batches = math.ceil(total_feeds / batch_size) if total_feeds else 0 + run_id = new_run_id(spec, run_started_at) + + logger.info( + "%s: run=%s total_feeds=%d batch_size=%d batches=%d dry_run=%s", + spec.log_name, + run_id, + total_feeds, + batch_size, + num_batches, + dry_run, + ) + + plan = { + "run_id": run_id, + "total_feeds": total_feeds, + "batch_size": batch_size, + "batches": num_batches, + "enqueued": 0, + "dry_run": dry_run, + } + if dry_run or not num_batches: + return plan + + batch_ids = [f"batch-{index:04d}" for index in range(num_batches)] + start_run(spec.task_name, run_id, batch_ids, run_params(run_started_at.isoformat())) + + enqueued = 0 + consumed = 0 + stream = iter_batches(db_session, batch_size) + for batch_id, batch_stable_ids in zip(batch_ids, stream): + consumed += 1 + if enqueue_task( + in_body_task=spec.worker_task, + payload=build_worker_payload(run_id, batch_id, batch_stable_ids), + queue_env=spec.queue_env, + task_name=safe_task_name(f"{spec.task_prefix}-{run_id}-{batch_id}"), + ): + enqueued += 1 + else: + # Dead on arrival: don't leave it `triggered` until the deadline. + mark_enqueue_failed(spec.task_name, run_id, batch_id) + + _reconcile(spec, run_id, batch_ids, consumed, stream) + + # Single barrier task, delayed so it does not fire before any worker has run. + enqueue_task( + in_body_task=spec.monitor_task, + payload={"run_id": run_id, **spec.monitor_extra}, + queue_env=spec.monitor_queue_env, + task_name=safe_task_name(f"{spec.task_prefix}-monitor-{run_id}"), + schedule_seconds=monitor_delay_seconds, + ) + + plan["enqueued"] = enqueued + return plan diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator.py index a73aca639..085b3d47b 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator.py @@ -17,23 +17,14 @@ """Cloud Tasks producer: fan the nightly Seal of Reliability evaluation out to per-batch workers (issue #1800). -`update_seal_of_reliability` only ever evaluates an explicit `stable_feed_ids` list — -there is no run-the-whole-catalogue mode, and pure-SQL evaluation for the whole catalog -in one invocation would eventually hit `tasks_executor`'s own timeout as the catalog -grows. This producer is what enumerates the catalog and chunks it: +`update_seal_of_reliability` only evaluates an explicit `stable_feed_ids` list, and one +invocation over the whole catalog would eventually hit `tasks_executor`'s timeout. This +enumerates the catalog and chunks it; the mechanism itself lives in `fanout.plan_fanout`, +shared with the backfill producer (#1763). - 1. resolves every seal-eligible GTFS feed (same eligibility predicate `update_seals` - itself applies via `context.is_seal_eligible` — see `context.iter_eligible_stable_ids`); - 2. splits the stable_ids into batches of `batch_size`; - 3. registers the run + one entry per batch in TaskExecutionTracker and enqueues one - `seal_orchestrator_worker` Cloud Task per batch; - 4. enqueues a single `seal_orchestrator_monitor` barrier task. - -Batches, not feeds, are the tracked unit: seal evaluation is pure DB work with no -per-feed side effect requiring isolation (unlike notification dispatch, where each -subscription needs an independent send + claim), so one Cloud Task per ~250 feeds -keeps the daily invocation count low while still removing the single-invocation -timeout ceiling entirely. +Batches, not feeds, are the tracked unit: seal evaluation is pure DB work with no per-feed +side effect requiring isolation, so one Cloud Task per ~250 feeds keeps the daily invocation +count low while removing the single-invocation timeout ceiling. Payload (all optional):: @@ -49,56 +40,50 @@ } """ -import json import logging -import math -import os -import re -from datetime import datetime, timezone from typing import Any, Dict, List, Optional from shared.database.database import with_db_session -from shared.helpers.task_execution.task_execution_tracker import TaskExecutionTracker from tasks.seal_of_reliability.context import ( count_eligible_feeds, iter_eligible_stable_ids, ) +from tasks.seal_of_reliability.fanout import FanoutSpec, plan_fanout logger = logging.getLogger(__name__) -# TaskExecutionTracker task_name for a seal orchestrator run (fan-out workers + -# monitor all key off this plus a per-run run_id). +# TaskExecutionTracker task_name for a seal orchestrator run (fan-out workers + monitor +# all key off this plus a per-run run_id). SEAL_ORCHESTRATOR_TASK_NAME = "seal_orchestrator_run" DEFAULT_BATCH_SIZE = 250 DEFAULT_DEADLINE_SECONDS = 60 * 60 # 1h wall-clock cap for a run DEFAULT_MONITOR_DELAY_SECONDS = 60 +SPEC = FanoutSpec( + task_name=SEAL_ORCHESTRATOR_TASK_NAME, + worker_task="seal_orchestrator_worker", + run_id_prefix="seal", + task_prefix="seal-orchestrator", + log_name="seal_orchestrator", +) + def seal_orchestrator_handler(payload: dict) -> dict: """Entry point for the `seal_orchestrator` task.""" payload = payload or {} - dry_run = bool(payload.get("dry_run", True)) - batch_size = int(payload.get("batch_size", DEFAULT_BATCH_SIZE)) - criteria = payload.get("criteria") - now = payload.get("now") - limit = payload.get("limit") - stable_feed_ids = payload.get("stable_feed_ids") - deadline_seconds = int(payload.get("deadline_seconds", DEFAULT_DEADLINE_SECONDS)) - monitor_delay_seconds = int( - payload.get("monitor_delay_seconds", DEFAULT_MONITOR_DELAY_SECONDS) - ) - return _plan_run( - dry_run=dry_run, - batch_size=batch_size, - criteria=criteria, - now=now, - limit=limit, - stable_feed_ids=stable_feed_ids, - deadline_seconds=deadline_seconds, - monitor_delay_seconds=monitor_delay_seconds, + dry_run=bool(payload.get("dry_run", True)), + batch_size=int(payload.get("batch_size", DEFAULT_BATCH_SIZE)), + criteria=payload.get("criteria"), + now=payload.get("now"), + limit=payload.get("limit"), + stable_feed_ids=payload.get("stable_feed_ids"), + deadline_seconds=int(payload.get("deadline_seconds", DEFAULT_DEADLINE_SECONDS)), + monitor_delay_seconds=int( + payload.get("monitor_delay_seconds", DEFAULT_MONITOR_DELAY_SECONDS) + ), ) @@ -115,233 +100,31 @@ def _plan_run( db_session=None, ) -> Dict[str, Any]: """Resolve eligible feeds, chunk them, and (unless dry_run) fan the run out.""" - if batch_size <= 0: - raise ValueError("batch_size must be a positive integer") - - run_started_at = datetime.now(timezone.utc) - total_feeds = count_eligible_feeds( - db_session, stable_feed_ids=stable_feed_ids, limit=limit - ) - num_batches = math.ceil(total_feeds / batch_size) if total_feeds else 0 - run_id = f"seal-{run_started_at.strftime('%Y%m%dT%H%M%S')}" - - logger.info( - "seal_orchestrator: run=%s total_feeds=%d batch_size=%d batches=%d dry_run=%s", - run_id, - total_feeds, - batch_size, - num_batches, - dry_run, - ) - - if dry_run or not num_batches: - return { - "run_id": run_id, - "total_feeds": total_feeds, - "batch_size": batch_size, - "batches": num_batches, - "enqueued": 0, - "dry_run": dry_run, - } - - run_params = { - "dry_run": False, - "batch_size": batch_size, - "criteria": criteria, - "now": now, - "run_started_at": run_started_at.isoformat(), - "deadline_seconds": deadline_seconds, - } - batch_ids = [f"batch-{index:04d}" for index in range(num_batches)] - _start_run(run_id, batch_ids, run_params) - - enqueued = 0 - consumed = 0 - stable_id_batches = iter_eligible_stable_ids( - db_session, batch_size, stable_feed_ids=stable_feed_ids, limit=limit - ) - for batch_id, batch_stable_ids in zip(batch_ids, stable_id_batches): - consumed += 1 - worker_payload = { + return plan_fanout( + db_session, + SPEC, + count_feeds=lambda session: count_eligible_feeds( + session, stable_feed_ids=stable_feed_ids, limit=limit + ), + iter_batches=lambda session, size: iter_eligible_stable_ids( + session, size, stable_feed_ids=stable_feed_ids, limit=limit + ), + build_worker_payload=lambda run_id, batch_id, ids: { "run_id": run_id, "batch_id": batch_id, - "stable_feed_ids": batch_stable_ids, + "stable_feed_ids": ids, "criteria": criteria, "now": now, - } - if _enqueue( - in_body_task="seal_orchestrator_worker", - payload=worker_payload, - queue_env="SEAL_ORCHESTRATOR_QUEUE", - task_name=_safe_task_name(f"seal-orchestrator-{run_id}-{batch_id}"), - ): - enqueued += 1 - else: - # Dead on arrival: don't leave this batch as `triggered` for the monitor - # to only notice once the deadline passes. - _mark_enqueue_failed(run_id, batch_id) - - if consumed < len(batch_ids): - # The eligible-feed stream (a separately executed query) yielded fewer chunks - # than count_eligible_feeds implied at plan time — eligibility narrowed in the - # gap between the two queries. The leftover batch_ids are already `triggered` - # (via _start_run) but would otherwise never be enqueued or reported, sitting - # stuck until deadline_seconds forces the whole run to `failed`. Fail them - # immediately and visibly instead. - missing = batch_ids[consumed:] - logger.error( - "seal_orchestrator: run=%s eligible-feed stream yielded %d batch(es), " - "expected %d from the plan-time count — marking %d batch(es) failed: %s", - run_id, - consumed, - len(batch_ids), - len(missing), - missing, - ) - for batch_id in missing: - _mark_enqueue_failed( - run_id, - batch_id, - error_message="no eligible-feed data for this batch (count/stream mismatch)", - ) - else: - # zip() with batch_ids (a plain list) first never calls next() on - # stable_id_batches for a final round once batch_ids is exhausted, so this - # reliably reflects whatever the stream still has left, with no off-by-one. - extra_chunk = next(stable_id_batches, None) - if extra_chunk is not None: - # Opposite direction: more chunks than the plan-time count implied (feeds - # became newly eligible in the gap). Log-only: making this self-healing - # would mean mutating TaskExecutionTracker's total_count after _start_run - # already fixed it, for a narrow race window whose only consequence is one - # feed waiting until the next nightly run. - logger.error( - "seal_orchestrator: run=%s eligible-feed stream had more batches than " - "the plan-time count of %d expected (>=%d additional feed(s) seen) — " - "some newly-eligible feeds were not evaluated this run; the next " - "scheduled run will pick them up", - run_id, - len(batch_ids), - len(extra_chunk), - ) - - # Single barrier/summary task; polls until the run drains, then reports. - # Delayed slightly so it doesn't fire before any worker has had a chance to run. - _enqueue( - in_body_task="seal_orchestrator_monitor", - payload={"run_id": run_id}, - queue_env="SEAL_ORCHESTRATOR_MONITOR_QUEUE", - task_name=_safe_task_name(f"seal-orchestrator-monitor-{run_id}"), - schedule_seconds=monitor_delay_seconds, - ) - - return { - "run_id": run_id, - "total_feeds": total_feeds, - "batch_size": batch_size, - "batches": num_batches, - "enqueued": enqueued, - "dry_run": False, - } - - -@with_db_session -def _start_run( - run_id: str, - batch_ids: List[str], - run_params: dict, - db_session=None, -) -> None: - """Register the run and one tracked entry per batch.""" - tracker = TaskExecutionTracker( - task_name=SEAL_ORCHESTRATOR_TASK_NAME, - run_id=run_id, - db_session=db_session, - ) - tracker.start_run(total_count=len(batch_ids), params=run_params) - for batch_id in batch_ids: - tracker.mark_triggered(batch_id) - db_session.commit() - - -@with_db_session -def _mark_enqueue_failed( - run_id: str, - batch_id: str, - error_message: str = "enqueue failed", - db_session=None, -) -> None: - tracker = TaskExecutionTracker( - task_name=SEAL_ORCHESTRATOR_TASK_NAME, - run_id=run_id, - db_session=db_session, + }, + run_params=lambda run_started_at: { + "dry_run": False, + "batch_size": batch_size, + "criteria": criteria, + "now": now, + "run_started_at": run_started_at, + "deadline_seconds": deadline_seconds, + }, + batch_size=batch_size, + dry_run=dry_run, + monitor_delay_seconds=monitor_delay_seconds, ) - tracker.mark_failed(batch_id, error_message=error_message) - db_session.commit() - - -def _safe_task_name(name: str) -> str: - return re.sub(r"[^a-zA-Z0-9_-]", "-", name)[:500] - - -def _enqueue( - *, - in_body_task: str, - payload: dict, - queue_env: str, - task_name: str, - schedule_seconds: int = 0, -) -> bool: - """Enqueue a Cloud Task targeting the tasks_executor function. - - Returns True on enqueue (or already-exists), False when misconfigured. - """ - project = os.getenv("PROJECT_ID") - queue = os.getenv(queue_env) - gcp_region = os.getenv("GCP_REGION") - environment = os.getenv("ENVIRONMENT") - if not all([project, queue, gcp_region, environment]): - logger.warning( - "_enqueue: missing env (PROJECT_ID/GCP_REGION/ENVIRONMENT/%s) — " - "skipping enqueue of %s", - queue_env, - task_name, - ) - return False - - try: - from google.cloud import tasks_v2 - from google.protobuf import timestamp_pb2 - from datetime import timedelta - from shared.common.gcp_utils import create_http_task_with_name - - url = ( - f"https://{gcp_region}-{project}.cloudfunctions.net/" - f"tasks_executor-{environment}" - ) - body = json.dumps({"task": in_body_task, "payload": payload}).encode() - - schedule_time: Optional[Any] = None - if schedule_seconds > 0: - run_at = datetime.now(timezone.utc) + timedelta(seconds=schedule_seconds) - schedule_time = timestamp_pb2.Timestamp() - schedule_time.FromDatetime(run_at.replace(tzinfo=None)) - - create_http_task_with_name( - client=tasks_v2.CloudTasksClient(), - body=body, - url=url, - project_id=project, - gcp_region=gcp_region, - queue_name=queue, - task_name=task_name, - task_time=schedule_time, - http_method=tasks_v2.HttpMethod.POST, - ) - return True - except Exception as e: # pragma: no cover - network/env dependent - if "already exists" in str(e).lower() or "ALREADY_EXISTS" in str(e): - logger.info("_enqueue: task %s already exists — skipping", task_name) - return True - logger.warning("_enqueue: could not enqueue %s: %s", task_name, e) - return False diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator_monitor.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator_monitor.py index a7168be18..12bebf504 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator_monitor.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator_monitor.py @@ -31,9 +31,15 @@ a clear failure: the whole point of tracking start/end is to know when a nightly run did NOT fully update the seal for every feed. +The same monitor settles the backfill fan-out (#1763). Everything it does — poll, honour +the deadline, aggregate each batch's stored report — is identical for both; only the +TaskExecutionTracker `task_name` differs, so it is a payload parameter rather than a second +copy of this file. + Payload:: - { "run_id": str } # required + { "run_id": str, # required + "task_name": str } # optional, defaults to the nightly run's task name """ import logging @@ -61,19 +67,32 @@ _SETTLED_STATUSES = (STATUS_COMPLETED, STATUS_FAILED) +# Numeric keys summed across a run's batches. A batch that does not report one contributes +# zero, which is what lets the nightly and backfill fan-outs share this aggregation. +_SUMMED_KEYS = ( + "total_feeds", + "criterion_rows_written", + "snapshot_rows_written", + "seals_granted", + "seals_revoked", +) + def seal_orchestrator_monitor_handler(payload: dict) -> dict: """Entry point for the `seal_orchestrator_monitor` task.""" - run_id = (payload or {}).get("run_id") + payload = payload or {} + run_id = payload.get("run_id") if not run_id: raise ValueError("run_id is required") - return _monitor(run_id) + return _monitor(run_id, payload.get("task_name") or SEAL_ORCHESTRATOR_TASK_NAME) @with_db_session -def _monitor(run_id: str, db_session=None) -> dict: +def _monitor( + run_id: str, task_name: str = SEAL_ORCHESTRATOR_TASK_NAME, db_session=None +) -> dict: tracker = TaskExecutionTracker( - task_name=SEAL_ORCHESTRATOR_TASK_NAME, + task_name=task_name, run_id=run_id, db_session=db_session, ) @@ -89,7 +108,7 @@ def _monitor(run_id: str, db_session=None) -> dict: # report the same aggregate (read-only, no mutation) rather than a bare status string: # this is the only way to see a settled run's feed-processing totals after the fact. if summary["run_status"] in _SETTLED_STATUSES: - aggregated = _aggregate_batches(db_session, run_id) + aggregated = _aggregate_batches(db_session, run_id, task_name) return { "run_id": run_id, "status": ( @@ -121,7 +140,7 @@ def _monitor(run_id: str, db_session=None) -> dict: f"run {run_id} still in progress: {summary['triggered']} batch(es) pending" ) - aggregated = _aggregate_batches(db_session, run_id) + aggregated = _aggregate_batches(db_session, run_id, task_name) incomplete = summary["triggered"] # > 0 only if the deadline was reached first final_status = ( STATUS_FAILED if summary["failed"] > 0 or incomplete > 0 else STATUS_COMPLETED @@ -152,32 +171,29 @@ def _monitor(run_id: str, db_session=None) -> dict: return result -def _aggregate_batches(db_session, run_id: str) -> Dict[str, Any]: +def _aggregate_batches(db_session, run_id: str, task_name: str) -> Dict[str, Any]: """Sum each completed batch's stored `update_seals` report into one run-level report.""" rows = ( db_session.query(TaskExecutionLog.metadata_) .filter( - TaskExecutionLog.task_name == SEAL_ORCHESTRATOR_TASK_NAME, + TaskExecutionLog.task_name == task_name, TaskExecutionLog.run_id == run_id, TaskExecutionLog.metadata_.isnot(None), ) .all() ) - total_feeds = 0 - criterion_rows_written = 0 - seals_granted = 0 - seals_revoked = 0 + # snapshot_rows_written is only ever reported by a backfill batch; a nightly batch + # simply has no such key and contributes zero. + totals = dict.fromkeys(_SUMMED_KEYS, 0) granted_stable_ids: list = [] revoked_stable_ids: list = [] for (metadata,) in rows: if not metadata: continue - total_feeds += metadata.get("total_feeds", 0) or 0 - criterion_rows_written += metadata.get("criterion_rows_written", 0) or 0 - seals_granted += metadata.get("seals_granted", 0) or 0 - seals_revoked += metadata.get("seals_revoked", 0) or 0 + for key in _SUMMED_KEYS: + totals[key] += metadata.get(key, 0) or 0 granted_stable_ids.extend(metadata.get("granted_stable_ids") or []) revoked_stable_ids.extend(metadata.get("revoked_stable_ids") or []) @@ -186,10 +202,9 @@ def _aggregate_batches(db_session, run_id: str) -> Dict[str, Any]: ) return { - "total_feeds_evaluated": total_feeds, - "criterion_rows_written": criterion_rows_written, - "seals_granted": seals_granted, - "seals_revoked": seals_revoked, + # Kept under its historical name; the others carry the key the batch reported. + "total_feeds_evaluated": totals["total_feeds"], + **{key: totals[key] for key in _SUMMED_KEYS if key != "total_feeds"}, "granted_stable_ids": granted_stable_ids[:MAX_REPORTED_IDS], "revoked_stable_ids": revoked_stable_ids[:MAX_REPORTED_IDS], "ids_omitted": ids_omitted, diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py index 37807c1d2..70a211538 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py @@ -54,6 +54,7 @@ batched, build_contexts, is_seal_eligible, + snapshot_date_of, ) from tasks.seal_of_reliability.evaluators import EVALUATORS from tasks.seal_of_reliability.state_machine import ( @@ -96,6 +97,16 @@ def _resolve_evaluators(criteria: Optional[Sequence[str]]) -> List: return [evaluator for evaluator in EVALUATORS if evaluator.name.value in wanted] +def is_partial_run(evaluators: Sequence) -> bool: + """Whether `evaluators` is only part of the registry, so has_seal cannot be rolled up. + + Kept here, next to the registry it compares against, so that both the nightly job and + the backfill answer the question the same way — and so a test patching `EVALUATORS` in + this module alone moves both. + """ + return len(evaluators) < len(EVALUATORS) + + def _validate_requested_feed_ids( requested: Sequence[str], found: Set[str], @@ -243,17 +254,6 @@ def _upsert_criteria( ) -def snapshot_date_of(now: datetime) -> date: - """The UTC day a run evaluating at `now` takes its snapshots under. - - Naive values are read as UTC rather than rejected: the entry point normalizes what an - operator passes, but `update_seals` is also called directly. - """ - if now.tzinfo is None: - return now.date() - return now.astimezone(timezone.utc).date() - - def _snapshot_row(state: SealCriterionState, snapshot_date: date) -> dict: """One seal_criterion_snapshot row: the key, then the state columns read off by name. @@ -379,7 +379,7 @@ def update_seals( started = time.monotonic() now = now or datetime.now(timezone.utc) evaluators = _resolve_evaluators(criteria) - partial_run = len(evaluators) < len(EVALUATORS) + partial_run = is_partial_run(evaluators) # Plain by-id load: no eligibility predicate here, since these ids were already # explicitly requested. Eligibility is checked in Python below, on the loaded rows. @@ -414,7 +414,7 @@ def update_seals( for batch in batched(eligible_feeds, batch_size): batch_ids = [feed.id for feed in batch] - contexts = build_contexts(db_session, batch, now) + contexts = build_contexts(db_session, batch, now, evaluators) previous_states = _load_previous_states(db_session, batch_ids) previous_seals = _load_previous_seals(db_session, batch_ids) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py index 8a3a1405f..a39f81e1d 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py @@ -30,6 +30,10 @@ privilege, and a criterion serving a penalty for an earlier confirmed failure has forfeited it. That coupling is what makes IN_GRACE_PERIOD and ON_PROBATION mutually exclusive. +The grace period runs from the day the streak began, not in days the criterion was seen +failing, so it can expire on a day with no reading. It is confirmed then regardless: the +evidence is the failures already observed. + The seal (step 5, in seal_updater) requires every criterion in service to be a confirmed pass and not on probation. """ @@ -205,19 +209,46 @@ def transition( observed_status = observation.observed_status if not observed_status.is_verdict: - # UNKNOWN keeps the stored confirmed_status so the criterion stays in the roll-up - # with its last verdict; NOT_APPLICABLE overwrites it so the criterion leaves the - # roll-up. Neither touches probation or the failure timestamps, and neither moves - # last_verdict_at — no verdict was produced. - confirmed_status = ( - CriterionStatus.NOT_APPLICABLE - if observed_status is CriterionStatus.NOT_APPLICABLE - else base.confirmed_status + # NOT_APPLICABLE leaves the roll-up. Its penalty survives, should it come back. + if observed_status is CriterionStatus.NOT_APPLICABLE: + return replace( + base, + observed_status=observed_status, + confirmed_status=CriterionStatus.NOT_APPLICABLE, + evaluated_at=now, + ) + + # UNKNOWN keeps the stored verdict, unless the streak has already outlived its + # grace: today's missing reading does not undo the failures behind it. Only the first + # such day confirms, or probation would advance on days nobody measured. + outlived_grace = ( + grace_period is not None + and base.confirmed_status is not CriterionStatus.FAIL + and base.first_observed_failure_at is not None + and base.last_verdict_at is not None + and now - base.first_observed_failure_at >= grace_period ) + if outlived_grace: + # Stamped at the last observed failure, not today: today has nothing to point at. + last_seen = base.last_observed_failure_at + return replace( + base, + observed_status=observed_status, + confirmed_status=CriterionStatus.FAIL, + evaluated_at=now, + last_confirmed_failure_at=last_seen, + probation_start=( + _next_day_start(last_seen) + if probation_period is not None + else base.probation_start + ), + ) + + # No branch here moves last_verdict_at: the check ran, but returned no verdict. return replace( base, observed_status=observed_status, - confirmed_status=confirmed_status, + confirmed_status=base.confirmed_status, evaluated_at=now, ) diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_backfill_matrix.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_backfill_matrix.py new file mode 100644 index 000000000..c01c0d625 --- /dev/null +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_backfill_matrix.py @@ -0,0 +1,306 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""End-state matrix for the backfill march (#1763): feed age x observation pattern. + +Every cell runs one real backfill over a 365-day window and asserts the three things that +survive it — `confirmed_status`, whether probation is open, and whether the feed holds the +seal. The point is not the state machine itself (covered day by day in +test_scripted_evaluator.py) but that a *march* of the right length lands on the right answer. + +Feed age is the second axis because the march start is clamped to `created_at`, so a younger +feed marches fewer days. Where that matters it is called out per cell: a criterion needs 180 +clean days to serve probation, and a feed younger than that simply cannot finish it inside +its own march however clean it is. + +Day 0 of each scenario is that feed's own march start, not the window start. + +**The criterion under test is not the real Official.** Every cell runs with `EVALUATORS` +patched to a single `ScriptedEvaluator`, which files its rows under the `official` name but +carries a 30-day grace period and the standard 180-day probation — for the duration of the +test only. The real `OfficialEvaluator` has neither and would flip the same day the flag +moves, making every row of this table identical and the age axis meaningless. The whole +matrix is about the debouncing mechanisms, so it has to supply a criterion that has them; +see test_scripted_evaluator.py. +""" + +import unittest +from datetime import date, datetime, timedelta, timezone + +from sqlalchemy import select +from unittest.mock import patch + +from shared.database.database import with_db_session +from shared.database_gen.sqlacodegen_models import FeedReliabilitySeal, SealCriterion +from tasks.seal_of_reliability.backfill.seal_backfill import backfill_seals +from shared.common.seal_criteria import ( + PROBATION_PERIOD, + CriterionStatus, + SealCriterionName, +) +from test_scripted_evaluator import ( + TEST_GRACE, + Script, + ScriptedEvaluator, + cleanup, + seed_feed, +) +from test_shared.test_utils.database_utils import default_db_url + +PREFIX = "seal_mx_" + +WINDOW_DAYS = 365 +END = date(2026, 6, 1) +START = END - timedelta(days=WINDOW_DAYS) + +GRACE_DAYS = TEST_GRACE.days # 30 +PROBATION_DAYS = PROBATION_PERIOD.days # 180 + +# How long before `END` each band's feed was created. The march start is the later of the +# window start and the creation date, so only `old` is clamped by the window. +BANDS = { + "old": 800, # older than the window: marches all 366 days + "middle": 270, # inside the window, comfortably past the probation period + "young": 90, # inside the window, and shorter than probation +} + +PASS = CriterionStatus.PASS.value +FAIL = CriterionStatus.FAIL.value + +# Each pattern is a function of the march length, returning the failing day offsets. +PATTERNS = { + # Fails from the very first day and never recovers. + "all_fail": lambda n: range(0, n), + # Passes from the very first day and never fails. + "all_pass": lambda n: (), + # One bad first day, clean ever after. The first evaluation gets no grace, so it + # confirms — and recovery from a confirmed failure opens probation on day 1. + "fail_first_then_clean": lambda n: (0,), + # Clean, then fails from day 5 to the end. Confirms once the streak outlasts grace. + "clean_then_fails_to_the_end": lambda n: range(5, n), + # A single failing day, well inside the grace period. + "absorbed_blip": lambda n: (20,), + # A confirmed failure repaired too late for probation to be served by `end_date`. + "late_recovery": lambda n: range(n - 40, n - 4), + # The same failure repaired early enough that probation *may* be served, depending on + # how many days the feed's march actually has left. + "early_recovery": lambda n: range(5, 5 + GRACE_DAYS + 6), +} + +# (pattern, band) -> (confirmed_status, probation_open, has_seal) +EXPECTED = { + ("all_fail", "old"): (FAIL, True, False), + ("all_fail", "middle"): (FAIL, True, False), + ("all_fail", "young"): (FAIL, True, False), + ("all_pass", "old"): (PASS, False, True), + ("all_pass", "middle"): (PASS, False, True), + ("all_pass", "young"): (PASS, False, True), + # Probation opens on day 1 and needs 180 clean days. Only the young feed runs out of + # march before it can serve them. + ("fail_first_then_clean", "old"): (PASS, False, True), + ("fail_first_then_clean", "middle"): (PASS, False, True), + ("fail_first_then_clean", "young"): (PASS, True, False), + ("clean_then_fails_to_the_end", "old"): (FAIL, True, False), + ("clean_then_fails_to_the_end", "middle"): (FAIL, True, False), + ("clean_then_fails_to_the_end", "young"): (FAIL, True, False), + ("absorbed_blip", "old"): (PASS, False, True), + ("absorbed_blip", "middle"): (PASS, False, True), + ("absorbed_blip", "young"): (PASS, False, True), + ("late_recovery", "old"): (PASS, True, False), + ("late_recovery", "middle"): (PASS, True, False), + ("late_recovery", "young"): (PASS, True, False), + ("early_recovery", "old"): (PASS, False, True), + ("early_recovery", "middle"): (PASS, False, True), + # Same repair, same clean run afterwards — but 90 days of march cannot contain a + # 180-day probation, so the young feed ends still serving it. + ("early_recovery", "young"): (PASS, True, False), +} + + +def _created_at(age_days: int) -> datetime: + return datetime.combine( + END - timedelta(days=age_days), datetime.min.time(), tzinfo=timezone.utc + ) + + +def _march_start(age_days: int) -> date: + return max(START, END - timedelta(days=age_days)) + + +def _march_length(age_days: int) -> int: + return (END - _march_start(age_days)).days + 1 + + +class TestBackfillEndStateMatrix(unittest.TestCase): + """One real backfill per cell, asserting the state it leaves behind.""" + + @with_db_session(db_url=default_db_url) + def setUp(self, db_session): + cleanup(db_session, PREFIX) + + @with_db_session(db_url=default_db_url) + def tearDown(self, db_session): + cleanup(db_session, PREFIX) + + @staticmethod + @with_db_session(db_url=default_db_url) + def _seed(stable_id, age_days, db_session=None): + seed_feed(db_session, stable_id, _created_at(age_days)) + db_session.commit() + + @staticmethod + @with_db_session(db_url=default_db_url) + def _final_state(stable_id, db_session=None): + criterion = db_session.execute( + select(SealCriterion.__table__).where( + SealCriterion.__table__.c.feed_id == stable_id, + SealCriterion.__table__.c.criterion == SealCriterionName.OFFICIAL.value, + ) + ).one() + seal = db_session.execute( + select(FeedReliabilitySeal.__table__).where( + FeedReliabilitySeal.__table__.c.feed_id == stable_id + ) + ).one() + return ( + criterion.confirmed_status, + criterion.probation_start is not None, + bool(seal.has_seal), + ) + + def _run_cell(self, pattern_name: str, band: str): + age = BANDS[band] + length = _march_length(age) + stable_id = f"{PREFIX}{band}_{pattern_name}"[:255] + + self._seed(stable_id, age) + script = Script.from_offsets( + _march_start(age), failing=PATTERNS[pattern_name](length) + ) + with patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", + [ScriptedEvaluator(script)], + ): + backfill_seals( + stable_feed_ids=[stable_id], + start_date=START, + end_date=END, + dry_run=False, + ) + return self._final_state(stable_id) + + def test_every_cell(self): + for (pattern_name, band), expected in sorted(EXPECTED.items()): + with self.subTest(pattern=pattern_name, band=band): + self.assertEqual( + self._run_cell(pattern_name, band), + expected, + f"{pattern_name} / {band}: expected " + f"(confirmed, probation_open, has_seal) = {expected}", + ) + + def test_the_bands_really_do_march_different_lengths(self): + """Guards the matrix: if the clamp broke, every band would march the same window.""" + self.assertEqual(_march_length(BANDS["old"]), WINDOW_DAYS + 1) + self.assertEqual(_march_length(BANDS["middle"]), BANDS["middle"] + 1) + self.assertEqual(_march_length(BANDS["young"]), BANDS["young"] + 1) + self.assertLess(_march_length(BANDS["young"]), PROBATION_DAYS) + + +class TestProbationBoundaryAcrossAges(unittest.TestCase): + """The exact age at which a feed becomes able to serve probation inside its own march. + + A bad first day opens probation on day 1, which clears on the first passing day at or + after day 1 + 180. So the feed needs a march reaching day 181 — one created 181 days + before `end_date` clears it on the very last day, and one created 180 days before does + not. Nothing else in the suite pins this. + """ + + @with_db_session(db_url=default_db_url) + def setUp(self, db_session): + cleanup(db_session, PREFIX) + + @with_db_session(db_url=default_db_url) + def tearDown(self, db_session): + cleanup(db_session, PREFIX) + + def _seal_after_bad_first_day(self, age_days: int) -> bool: + stable_id = f"{PREFIX}boundary_{age_days}" + TestBackfillEndStateMatrix._seed(stable_id, age_days) + script = Script.from_offsets(_march_start(age_days), failing=[0]) + with patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", + [ScriptedEvaluator(script)], + ): + backfill_seals( + stable_feed_ids=[stable_id], + start_date=START, + end_date=END, + dry_run=False, + ) + return TestBackfillEndStateMatrix._final_state(stable_id)[2] + + def test_one_day_short_of_serving_probation(self): + self.assertFalse(self._seal_after_bad_first_day(PROBATION_DAYS)) + + def test_exactly_long_enough_to_serve_probation(self): + self.assertTrue(self._seal_after_bad_first_day(PROBATION_DAYS + 1)) + + +class TestMarchEndingInsideTheGracePeriod(unittest.TestCase): + """A march can run out before a failure streak outlasts its grace period. + + A feed failing every observed day since day 5 still ends holding the seal, because the + 30-day grace period has not expired by `end_date`. Correct, and the one case where a + shorter march is *more* generous rather than less. + """ + + @with_db_session(db_url=default_db_url) + def setUp(self, db_session): + cleanup(db_session, PREFIX) + + @with_db_session(db_url=default_db_url) + def tearDown(self, db_session): + cleanup(db_session, PREFIX) + + def test_a_newborn_feed_keeps_the_seal_mid_streak(self): + age = 20 # marches 21 days; the streak from day 5 is 15 days old at the end + stable_id = f"{PREFIX}newborn" + TestBackfillEndStateMatrix._seed(stable_id, age) + script = Script.from_offsets( + _march_start(age), failing=range(5, _march_length(age)) + ) + with patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", + [ScriptedEvaluator(script)], + ): + backfill_seals( + stable_feed_ids=[stable_id], + start_date=START, + end_date=END, + dry_run=False, + ) + + confirmed, probation_open, has_seal = TestBackfillEndStateMatrix._final_state( + stable_id + ) + self.assertEqual(confirmed, PASS, "the streak is still inside the grace period") + self.assertFalse(probation_open) + self.assertTrue(has_seal) + self.assertLess(_march_length(age), 5 + GRACE_DAYS) + + +if __name__ == "__main__": + unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_scripted_evaluator.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_scripted_evaluator.py new file mode 100644 index 000000000..a6969eb01 --- /dev/null +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_scripted_evaluator.py @@ -0,0 +1,428 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""A day-scripted stand-in evaluator, and the tests proving it drives. + +Only Official is implemented, and it has neither a grace period nor probation. So nothing +otherwise exercises the path-dependent behaviour a backfill (#1763) exists to reconstruct: a +failure streak debounced by a grace period, a confirmed failure, the probation that follows +recovery, and an UNKNOWN day that freezes the lot. + +**It borrows the `official` enum value on purpose.** Official already has an evaluator, so no +future criterion implementation can collide with this fixture — unlike borrowing `compliant` +or `available`, whose real evaluators are still to be written (#1782, #1784). The grace and +probation values below are the harness's, chosen to exercise the state machine; the real +Official has neither, and nothing here should be read as its policy. + +Why scripted by day rather than driven through the database, as the stand-ins in +`test_seal_updater_db.py` are: those read `ctx.official` and a test moves them by issuing an +UPDATE between runs. A backfill marches its days in memory with no writes in between, so a +criterion it can drive has to answer from `ctx.now` alone. `Script` is that — sets of failing, +unknown and not-applicable days fixed up front, replayed by advancing the clock. +""" + +import unittest +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import date, datetime, timedelta, timezone +from typing import FrozenSet, Iterable, Optional, Tuple +from unittest.mock import patch + +from sqlalchemy import delete, select + +from shared.database.database import with_db_session +from shared.database_gen.sqlacodegen_models import Feed, Gtfsfeed, SealCriterion +from shared.common.seal_criteria import ( + PROBATION_PERIOD, + CriterionStatus, + SealCriterionName, +) +from tasks.seal_of_reliability.evaluators import CriterionEvaluator, OfficialEvaluator +from tasks.seal_of_reliability.seal_updater import update_seals +from test_shared.test_utils.database_utils import default_db_url + +# The harness's debouncing values, not any criterion's published policy. 30 days is long +# enough that a streak has to be deliberate to outlast it, and short enough that a test can +# step over the boundary without marching a year. +TEST_GRACE = timedelta(days=30) + +DAY_ZERO = datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc) + +PREFIX = "seal_sc_" +FEED = f"{PREFIX}scripted" + +# Distinguishes "not given" from an explicit None, which means "this criterion has no grace +# period" and is a value a test may legitimately want to pass. +_UNSET = object() + + +def day(offset: int) -> datetime: + """The run timestamp `offset` days after day zero.""" + return DAY_ZERO + timedelta(days=offset) + + +@dataclass(frozen=True) +class Script: + """What the stand-in answers on each day, fixed before the run starts. + + Days named in none of the three sets pass. Built from offsets against an anchor — a + feed's march start, usually — so a scenario reads in the same relative terms whatever + day the window actually begins on. + """ + + failing: FrozenSet[date] = field(default_factory=frozenset) + unknown: FrozenSet[date] = field(default_factory=frozenset) + not_applicable: FrozenSet[date] = field(default_factory=frozenset) + + @classmethod + def from_offsets( + cls, + anchor: date, + failing: Iterable[int] = (), + unknown: Iterable[int] = (), + not_applicable: Iterable[int] = (), + ) -> "Script": + def days(offsets): + return frozenset(anchor + timedelta(days=int(o)) for o in offsets) + + return cls(days(failing), days(unknown), days(not_applicable)) + + def status_on(self, today: date) -> CriterionStatus: + """Precedence matters: an input we could not read is never a failure.""" + if today in self.unknown: + return CriterionStatus.UNKNOWN + if today in self.not_applicable: + return CriterionStatus.NOT_APPLICABLE + if today in self.failing: + return CriterionStatus.FAIL + return CriterionStatus.PASS + + +class ScriptedEvaluator(CriterionEvaluator): + """A stand-in whose verdict is a pure function of the day being evaluated. + + It files its rows under the `official` criterion — see the module docstring for why that + name and not one of the unimplemented ones. + + **Its debouncing is not Official's.** The real `OfficialEvaluator` has `grace_period` and + `probation_period` both `None`: it is a point-in-time check that flips the same day the + flag moves, in either direction. This stand-in gives that name a 30-day grace period and + the standard 180-day probation *for the duration of the test only*, because those are the + mechanisms a backfill has to reconstruct and no implemented criterion has them yet. + + Nothing here changes the real evaluator. `registry()` patches the whole `EVALUATORS` + list for the length of a `with` block, so the substitution cannot leak past it. + """ + + name = SealCriterionName.OFFICIAL + # Both windows are declared here, not inherited. `CriterionEvaluator` resolves them from + # the policy maps by `name`, and `official` is exempt from both — so a stand-in borrowing + # that name gets None for anything it does not state. Declaring them also keeps them + # plain class attributes, which is what lets `__init__` shadow them per instance: the + # base class's versions are read-only properties and cannot be assigned to. + grace_period = TEST_GRACE + probation_period = PROBATION_PERIOD + + def __init__( + self, + script: Script, + criterion: Optional[SealCriterionName] = None, + grace_period=_UNSET, + probation_period=_UNSET, + ): + self.script = script + # Instance attributes shadow the class ones the job reads, so a test can vary the + # policy per case without subclassing. + if criterion is not None: + self.name = criterion + if grace_period is not _UNSET: + self.grace_period = grace_period + if probation_period is not _UNSET: + self.probation_period = probation_period + + def _evaluate(self, ctx) -> Tuple[CriterionStatus, str]: + today = ctx.now.astimezone(timezone.utc).date() + status = self.script.status_on(today) + return status, f"scripted: {status.value} on {today}" + + +@contextmanager +def registry(evaluator: ScriptedEvaluator): + """Run with the stand-in as the only criterion. + + Sole occupant on purpose: `update_seals` treats a shorter evaluator list as a partial run + and skips the has_seal roll-up, so patching the registry itself rather than filtering it + is what keeps the seal in play. Patched in `seal_updater`, which is where both the nightly + job and the backfill resolve the registry from. + """ + with patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", + [evaluator], + ): + yield + + +def cleanup(db_session, prefix: str = PREFIX): + """Delete from `feed`, not `gtfsfeed`. + + Gtfsfeed is a joined-table subclass, so deleting the subclass leaves the parent row and + the next insert collides on feed_pkey. The seal tables are ON DELETE CASCADE. + """ + db_session.execute(delete(Feed).where(Feed.stable_id.like(f"{prefix}%"))) + db_session.commit() + + +def seed_feed(db_session, stable_id: str, created_at: datetime, official=True): + db_session.add( + Gtfsfeed( + id=stable_id, + stable_id=stable_id, + data_type="gtfs", + status="active", + operational_status="published", + official=official, + created_at=created_at, + producer_url=f"https://example.com/{stable_id}.zip", + ) + ) + db_session.flush() + + +class ScriptedEvaluatorTestCase(unittest.TestCase): + """Seeds one eligible feed and replays scripted days against it.""" + + @with_db_session(db_url=default_db_url) + def setUp(self, db_session): + cleanup(db_session) + seed_feed(db_session, FEED, DAY_ZERO - timedelta(days=400)) + db_session.commit() + + @with_db_session(db_url=default_db_url) + def tearDown(self, db_session): + cleanup(db_session) + + @staticmethod + def run_days(script: Script, offsets) -> None: + """Evaluate the feed once per day, in order, writing each day's state. + + A march done the slow way — through the database, one `update_seals` call per day — + which is exactly what #1763's in-memory march has to reproduce. + """ + with registry(ScriptedEvaluator(script)): + for offset in offsets: + update_seals(stable_feed_ids=[FEED], dry_run=False, now=day(offset)) + + @staticmethod + @with_db_session(db_url=default_db_url) + def state(db_session=None): + return db_session.execute( + select(SealCriterion.__table__).where( + SealCriterion.__table__.c.feed_id == FEED, + SealCriterion.__table__.c.criterion == SealCriterionName.OFFICIAL.value, + ) + ).one() + + +class TestScriptDrivesTheEvaluator(unittest.TestCase): + """The fixture itself, with no database in the way.""" + + class _Ctx: + def __init__(self, now): + self.now = now + self.feed_id = FEED + self.stable_id = FEED + + def observed(self, script: Script, offset: int) -> CriterionStatus: + return ( + ScriptedEvaluator(script).evaluate(self._Ctx(day(offset))).observed_status + ) + + def test_unnamed_days_pass(self): + script = Script.from_offsets(DAY_ZERO.date(), failing=[3]) + self.assertIs(self.observed(script, 0), CriterionStatus.PASS) + + def test_named_days_fail(self): + script = Script.from_offsets(DAY_ZERO.date(), failing=[3]) + self.assertIs(self.observed(script, 3), CriterionStatus.FAIL) + + def test_a_range_is_inclusive_of_both_ends(self): + script = Script.from_offsets(DAY_ZERO.date(), failing=range(5, 8)) + for offset, expected in ( + (4, CriterionStatus.PASS), + (5, CriterionStatus.FAIL), + (7, CriterionStatus.FAIL), + (8, CriterionStatus.PASS), + ): + with self.subTest(offset=offset): + self.assertIs(self.observed(script, offset), expected) + + def test_unknown_wins_over_failing(self): + """An input we could not read is not a failure, whatever else the script says.""" + script = Script.from_offsets(DAY_ZERO.date(), failing=[3], unknown=[3]) + self.assertIs(self.observed(script, 3), CriterionStatus.UNKNOWN) + + def test_not_applicable_wins_over_failing(self): + script = Script.from_offsets(DAY_ZERO.date(), failing=[3], not_applicable=[3]) + self.assertIs(self.observed(script, 3), CriterionStatus.NOT_APPLICABLE) + + def test_it_borrows_official_so_nothing_future_can_collide(self): + self.assertIs(ScriptedEvaluator(Script()).name, SealCriterionName.OFFICIAL) + + def test_policy_is_the_harness_own_and_overridable(self): + default = ScriptedEvaluator(Script()) + self.assertEqual(default.grace_period, TEST_GRACE) + self.assertEqual(default.probation_period, PROBATION_PERIOD) + + custom = ScriptedEvaluator(Script(), grace_period=None, probation_period=None) + self.assertIsNone(custom.grace_period) + self.assertIsNone(custom.probation_period) + + def test_the_stand_in_debounces_where_the_real_official_does_not(self): + """Pins the divergence, so it is a fact rather than a comment. + + The real Official is a point-in-time check with neither mechanism. Every scenario + built on this fixture depends on the stand-in having both — so if Official were ever + given a grace period for real, this fails and says which assumption moved. + """ + # Instances, not the classes: both windows are properties resolving `name` against + # the policy maps, so reading them off the class returns the property object. + self.assertIsNone(OfficialEvaluator().grace_period) + self.assertIsNone(OfficialEvaluator().probation_period) + + stand_in = ScriptedEvaluator(Script()) + self.assertIs(stand_in.name, OfficialEvaluator.name) + self.assertIsNotNone(stand_in.grace_period) + self.assertIsNotNone(stand_in.probation_period) + + def test_the_substitution_does_not_outlive_the_context(self): + """`registry()` swaps the whole list, so nothing leaks into a later test.""" + from tasks.seal_of_reliability import seal_updater + + before = list(seal_updater.EVALUATORS) + with registry(ScriptedEvaluator(Script())): + self.assertEqual(len(seal_updater.EVALUATORS), 1) + self.assertEqual(list(seal_updater.EVALUATORS), before) + + +class TestGracePeriod(ScriptedEvaluatorTestCase): + def test_a_short_streak_is_absorbed(self): + """29 failing days is inside the 30-day grace period, so the status holds.""" + script = Script.from_offsets(DAY_ZERO.date(), failing=range(1, 30)) + self.run_days(script, range(0, 30)) + + row = self.state() + self.assertEqual(row.observed_status, CriterionStatus.FAIL.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.PASS.value, + "still inside the grace period on day 29", + ) + self.assertIsNone(row.probation_start, "an absorbed failure opens no probation") + + def test_a_streak_past_the_grace_period_confirms(self): + script = Script.from_offsets(DAY_ZERO.date(), failing=range(1, 41)) + self.run_days(script, range(0, 41)) + + row = self.state() + self.assertEqual(row.confirmed_status, CriterionStatus.FAIL.value) + self.assertIsNotNone(row.last_confirmed_failure_at) + + def test_the_first_evaluation_gets_no_grace(self): + """A criterion that has never passed has no track record to hold.""" + self.run_days(Script.from_offsets(DAY_ZERO.date(), failing=[0]), [0]) + self.assertEqual(self.state().confirmed_status, CriterionStatus.FAIL.value) + + +class TestProbationFollowsRecovery(ScriptedEvaluatorTestCase): + def test_recovery_from_a_confirmed_failure_opens_probation(self): + script = Script.from_offsets(DAY_ZERO.date(), failing=range(1, 41)) + self.run_days(script, list(range(0, 42))) # day 41 is the repair + + row = self.state() + self.assertEqual( + row.confirmed_status, CriterionStatus.PASS.value, "the check passes again" + ) + self.assertIsNotNone( + row.probation_start, "but it is serving probation for the failure" + ) + + def test_probation_suspends_the_grace_period(self): + """One bad day during probation confirms at once, and restarts the count. + + Off probation, a single failing day sits well inside the 30-day grace period and + would confirm nothing. This is the ratchet that makes a cold start's error persist. + """ + script = Script.from_offsets(DAY_ZERO.date(), failing=list(range(1, 41)) + [60]) + self.run_days(script, list(range(0, 62))) + + row = self.state() + self.assertEqual( + row.last_confirmed_failure_at.astimezone(timezone.utc).date(), + day(60).date(), + "the single day confirmed because probation had suspended the grace period", + ) + self.assertEqual( + row.probation_start.astimezone(timezone.utc).date(), + day(61).date(), + "and probation restarted from the day after it", + ) + + def test_probation_clears_once_served(self): + script = Script.from_offsets(DAY_ZERO.date(), failing=range(1, 41)) + served = 41 + PROBATION_PERIOD.days + self.run_days(script, [*range(0, 42), served]) + + self.assertIsNone( + self.state().probation_start, "the full stretch has been served" + ) + + +class TestNoVerdictDays(ScriptedEvaluatorTestCase): + def test_an_unknown_day_leaves_the_verdict_standing(self): + script = Script.from_offsets(DAY_ZERO.date(), unknown=[1]) + self.run_days(script, [0, 1]) + + row = self.state() + self.assertEqual(row.observed_status, CriterionStatus.UNKNOWN.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.PASS.value, + "a missing input must never read as a failure", + ) + + def test_an_unknown_day_does_not_advance_a_failure_streak(self): + """The streak keeps its start, so the grace period is not quietly extended.""" + script = Script.from_offsets(DAY_ZERO.date(), failing=range(1, 6), unknown=[3]) + self.run_days(script, range(0, 6)) + + self.assertEqual( + self.state().first_observed_failure_at.astimezone(timezone.utc).date(), + day(1).date(), + ) + + def test_a_not_applicable_day_withdraws_the_criterion(self): + script = Script.from_offsets(DAY_ZERO.date(), not_applicable=[1]) + self.run_days(script, [0, 1]) + + row = self.state() + self.assertEqual( + row.confirmed_status, + CriterionStatus.NOT_APPLICABLE.value, + "it leaves the roll-up rather than being frozen in it", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py new file mode 100644 index 000000000..b5a7bb15d --- /dev/null +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py @@ -0,0 +1,1002 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Tests for the Seal of Reliability backfill (#1763): parameters, window, and plan. + +The day march is not implemented, so these cover the invocation surface — what the payload +resolves to, which feeds are selected, and that a non-dry run refuses rather than reporting +a success that wrote nothing. +""" + +import json +import os +import unittest +from dataclasses import dataclass +from datetime import date, datetime, timedelta, timezone +from typing import Optional +from unittest.mock import patch + +from sqlalchemy import delete, insert, select + +from shared.database.database import with_db_session +from shared.database_gen.sqlacodegen_models import ( + Feed, + FeedReliabilitySeal, + Gtfsfeed, + SealCriterion, + SealCriterionSnapshot, +) +from tasks.seal_of_reliability.backfill.backfill_seal_of_reliability import ( + _parse_day, + backfill_seal_of_reliability_handler, + get_parameters, +) +from tasks.seal_of_reliability.backfill.seal_backfill import ( + DEFAULT_DAYS_BACK, + backfill_seals, + day_start, + days_between, + march_start_for, + resolve_window, + yesterday_utc, +) +from shared.common.seal_criteria import SealCriterionName +from tasks.seal_of_reliability.seal_updater import update_seals +from test_scripted_evaluator import Script, ScriptedEvaluator +from test_shared.test_utils.database_utils import default_db_url + +PREFIX = "seal_bf_" +OLD = f"{PREFIX}old" # created well before any window we test +YOUNG = f"{PREFIX}young" # created inside the window, so its march is clamped +DEPRECATED = f"{PREFIX}deprecated" # not seal-eligible +ALREADY = f"{PREFIX}already" # already has seal state + +END = date(2026, 6, 1) +START = date(2025, 6, 1) + +NOW = datetime(2026, 6, 2, 12, 0, tzinfo=timezone.utc) +OLD_CREATED = NOW - timedelta(days=800) +YOUNG_CREATED = NOW - timedelta(days=90) + + +@dataclass +class _FeedStub: + """Just enough of a feed row for `march_start_for`, which reads only `created_at`.""" + + created_at: Optional[datetime] + + +class TestParseDay(unittest.TestCase): + def test_plain_iso_date(self): + self.assertEqual(_parse_day("2026-01-31", "start_date"), date(2026, 1, 31)) + + def test_timestamp_is_accepted_and_truncated(self): + """An operator pasting the nightly task's `now` should not hit a parse error. + + The march is day-granular, so the time of day is dropped either way. + """ + for value in ("2026-01-31T12:00:00", "2026-01-31T12:00:00+00:00"): + with self.subTest(value=value): + self.assertEqual(_parse_day(value, "end_date"), date(2026, 1, 31)) + + def test_absent_stays_none(self): + self.assertIsNone(_parse_day(None, "start_date")) + + def test_garbage_names_the_field(self): + with self.assertRaises(ValueError) as caught: + _parse_day("last tuesday", "start_date") + self.assertIn("start_date", str(caught.exception)) + + +class TestGetParameters(unittest.TestCase): + def test_defaults(self): + ( + stable_feed_ids, + start_date, + end_date, + days_back, + dry_run, + limit, + criteria, + batch_size, + only_missing, + snapshot_mode, + resume_from_snapshot, + max_reported_feeds, + simulate, + trace, + ) = get_parameters({"stable_feed_ids": ["a"]}) + + self.assertEqual(stable_feed_ids, ["a"]) + self.assertIsNone(start_date) + self.assertIsNone(end_date) + self.assertEqual(days_back, DEFAULT_DAYS_BACK) + self.assertTrue(dry_run, "a backfill must not write unless asked to") + self.assertIsNone(limit) + self.assertIsNone(criteria) + self.assertEqual(batch_size, 200) + self.assertTrue(only_missing, "#1763 backfills feeds that have no state yet") + self.assertEqual(snapshot_mode, "final") + self.assertFalse(resume_from_snapshot) + self.assertEqual(max_reported_feeds, 50) + self.assertIsNone(simulate) + self.assertFalse(trace, "a run must not pay for a trace unless asked") + + def test_empty_payload_does_not_raise_here(self): + """Validation belongs to the engine, so the parser stays a plain reader.""" + self.assertIsNone(get_parameters({})[0]) + + +class TestResolveWindow(unittest.TestCase): + def test_both_given_are_kept(self): + self.assertEqual(resolve_window(START, END, DEFAULT_DAYS_BACK), (START, END)) + + def test_end_defaults_to_yesterday(self): + _, resolved_end = resolve_window(START, None, DEFAULT_DAYS_BACK) + self.assertEqual(resolved_end, yesterday_utc()) + + def test_start_defaults_to_days_back_from_end(self): + resolved_start, _ = resolve_window(None, END, 365) + self.assertEqual(resolved_start, END - timedelta(days=365)) + + def test_start_after_end_is_rejected(self): + with self.assertRaises(ValueError) as caught: + resolve_window(END + timedelta(days=1), END, DEFAULT_DAYS_BACK) + self.assertIn("after end_date", str(caught.exception)) + + def test_non_positive_days_back_is_rejected(self): + with self.assertRaises(ValueError): + resolve_window(None, END, 0) + + +class TestMarchStart(unittest.TestCase): + def test_feed_older_than_the_window_starts_at_the_window(self): + feed = _FeedStub(created_at=datetime(2020, 1, 1, tzinfo=timezone.utc)) + self.assertEqual(march_start_for(feed, START), START) + + def test_feed_younger_than_the_window_starts_at_its_creation(self): + """A feed with no history before its creation gets an exact cold start, not a guess.""" + created = datetime(2025, 9, 15, 8, 30, tzinfo=timezone.utc) + feed = _FeedStub(created_at=created) + self.assertEqual(march_start_for(feed, START), date(2025, 9, 15)) + + def test_naive_created_at_is_read_as_utc(self): + feed = _FeedStub(created_at=datetime(2025, 9, 15, 8, 30)) + self.assertEqual(march_start_for(feed, START), date(2025, 9, 15)) + + def test_missing_created_at_falls_back_to_the_window(self): + self.assertEqual(march_start_for(_FeedStub(created_at=None), START), START) + + +def _seed_feed(db_session, feed_id, created_at, status="active"): + db_session.add( + Gtfsfeed( + id=feed_id, + stable_id=feed_id, + data_type="gtfs", + status=status, + operational_status="published", + official=True, + created_at=created_at, + producer_url=f"https://example.com/{feed_id}.zip", + ) + ) + db_session.flush() + + +def _cleanup(db_session): + """Delete from `feed`, not `gtfsfeed`. + + Gtfsfeed is a joined-table subclass, so deleting the subclass leaves the parent row and + the next insert collides on feed_pkey. The seal tables are ON DELETE CASCADE. + """ + db_session.execute(delete(Feed).where(Feed.stable_id.like(f"{PREFIX}%"))) + db_session.commit() + + +class BackfillDbTestCase(unittest.TestCase): + @with_db_session(db_url=default_db_url) + def setUp(self, db_session): + _cleanup(db_session) + _seed_feed(db_session, OLD, OLD_CREATED) + _seed_feed(db_session, YOUNG, YOUNG_CREATED) + _seed_feed(db_session, DEPRECATED, OLD_CREATED, status="deprecated") + _seed_feed(db_session, ALREADY, OLD_CREATED) + db_session.execute( + insert(SealCriterion.__table__).values( + feed_id=ALREADY, criterion="official" + ) + ) + db_session.commit() + + @with_db_session(db_url=default_db_url) + def tearDown(self, db_session): + _cleanup(db_session) + + +class TestBackfillPlan(BackfillDbTestCase): + def test_dry_run_reports_the_resolved_window(self): + report = backfill_seals( + stable_feed_ids=[OLD], start_date=START, end_date=END, dry_run=True + ) + self.assertTrue(report["dry_run"]) + self.assertEqual(report["criterion_rows_written"], 0) + self.assertEqual(report["start_date"], START.isoformat()) + self.assertEqual(report["end_date"], END.isoformat()) + self.assertEqual(report["days"], (END - START).days + 1) + self.assertEqual(report["total_feeds"], 1) + + def test_each_feed_marches_from_its_own_start(self): + """The window start is run-wide; the march start is per feed.""" + report = backfill_seals( + stable_feed_ids=[OLD, YOUNG], start_date=START, end_date=END, dry_run=True + ) + by_id = {entry["stable_id"]: entry for entry in report["feeds"]} + + self.assertEqual(by_id[OLD]["march_start"], START.isoformat()) + self.assertEqual(by_id[YOUNG]["march_start"], YOUNG_CREATED.date().isoformat()) + self.assertLess(by_id[YOUNG]["days"], by_id[OLD]["days"]) + + def test_march_start_is_also_the_stable_anchor(self): + report = backfill_seals( + stable_feed_ids=[YOUNG], start_date=START, end_date=END, dry_run=True + ) + entry = report["feeds"][0] + self.assertEqual(entry["tracking_start"], entry["march_start"]) + + def test_ineligible_feeds_are_left_out(self): + report = backfill_seals( + stable_feed_ids=[OLD, DEPRECATED], + start_date=START, + end_date=END, + dry_run=True, + ) + self.assertEqual( + [entry["stable_id"] for entry in report["feeds"]], + [OLD], + ) + + def test_only_missing_skips_a_feed_that_already_has_state(self): + report = backfill_seals( + stable_feed_ids=[OLD, ALREADY], + start_date=START, + end_date=END, + dry_run=True, + ) + self.assertEqual(report["total_feeds"], 1) + self.assertEqual(report["skipped_already_backfilled"], 1) + self.assertEqual([entry["stable_id"] for entry in report["feeds"]], [OLD]) + + def test_only_missing_false_re_backfills(self): + report = backfill_seals( + stable_feed_ids=[OLD, ALREADY], + start_date=START, + end_date=END, + dry_run=True, + only_missing=False, + ) + self.assertEqual(report["total_feeds"], 2) + self.assertEqual(report["skipped_already_backfilled"], 0) + + +class TestBackfillValidation(BackfillDbTestCase): + def test_empty_feed_list_is_rejected(self): + with self.assertRaises(ValueError): + backfill_seals(stable_feed_ids=[], dry_run=True) + + def test_unknown_snapshot_mode_is_rejected(self): + with self.assertRaises(ValueError) as caught: + backfill_seals( + stable_feed_ids=[OLD], dry_run=True, snapshot_mode="occasionally" + ) + self.assertIn("occasionally", str(caught.exception)) + + def test_unknown_criteria_are_rejected(self): + with self.assertRaises(ValueError) as caught: + backfill_seals(stable_feed_ids=[OLD], dry_run=True, criteria=["punctual"]) + self.assertIn("punctual", str(caught.exception)) + + def test_dry_run_writes_nothing(self): + backfill_seals( + stable_feed_ids=[OLD], start_date=START, end_date=END, dry_run=True + ) + self.assertEqual(criterion_rows(OLD), {}) + self.assertIsNone(seal_row(OLD)) + + def test_handler_threads_the_payload_through(self): + report = backfill_seal_of_reliability_handler( + { + "stable_feed_ids": [OLD], + "start_date": START.isoformat(), + "end_date": END.isoformat(), + "snapshot_mode": "all", + "resume_from_snapshot": True, + } + ) + self.assertEqual(report["snapshot_mode"], "all") + self.assertTrue(report["resume_from_snapshot"]) + self.assertEqual(report["start_date"], START.isoformat()) + + +MARCHED = f"{PREFIX}marched" +REPLAYED = f"{PREFIX}replayed" + +# The march tests below run with EVALUATORS patched to a single ScriptedEvaluator. It files +# its rows under the `official` criterion but carries a 30-day grace period and the standard +# 180-day probation, for the duration of each test only — the real OfficialEvaluator has +# neither. Those two mechanisms are what a backfill has to reconstruct, and no implemented +# criterion has them yet; see test_scripted_evaluator.py. +# +# A short window, so the equivalence test replays a tractable number of days through the +# database. The failing run is long enough to outlast that 30-day grace period, so the +# comparison covers a confirmed failure and the probation that follows it. +MARCH_START = date(2026, 1, 1) +MARCH_END = date(2026, 3, 15) + +STATE_COLUMNS = ( + "observed_status", + "confirmed_status", + "evaluated_at", + "last_verdict_at", + "first_observed_failure_at", + "last_observed_failure_at", + "last_confirmed_failure_at", + "probation_start", +) + + +def _script_for(offsets_from_march_start): + """A Script whose failing days are offsets from MARCH_START.""" + return Script.from_offsets(MARCH_START, failing=offsets_from_march_start) + + +@with_db_session(db_url=default_db_url) +def criterion_rows(stable_id, db_session=None): + rows = db_session.execute( + select(SealCriterion.__table__).where( + SealCriterion.__table__.c.feed_id == stable_id + ) + ).all() + return {row.criterion: row for row in rows} + + +@with_db_session(db_url=default_db_url) +def seal_row(stable_id, db_session=None): + return db_session.execute( + select(FeedReliabilitySeal.__table__).where( + FeedReliabilitySeal.__table__.c.feed_id == stable_id + ) + ).one_or_none() + + +@with_db_session(db_url=default_db_url) +def snapshot_days(stable_id, db_session=None): + rows = db_session.execute( + select(SealCriterionSnapshot.__table__.c.snapshot_date).where( + SealCriterionSnapshot.__table__.c.feed_id == stable_id + ) + ).all() + return sorted({row.snapshot_date for row in rows}) + + +def trace_day(report, day): + """The marched day `day`, found inside the collapsed trace. + + A trace reports one entry per unchanged stretch, so a day in the middle of one is + represented by that stretch's `first` row — every field asserted on here is part of the + signature the stretch collapsed on, and so is identical across it. The closing day carries + its own row, returned as `last`. + """ + for entry in report["trace"]: + first = entry["first"] + last = entry.get("last", first) + if first["day"] <= day <= last["day"]: + return last if day == last["day"] else first + raise AssertionError(f"day {day} is not in the trace") + + +def trace_last(report): + """The final marched day of the trace.""" + entry = report["trace"][-1] + return entry.get("last", entry["first"]) + + +class MarchTestCase(unittest.TestCase): + """Two identically-aged feeds: one marched in memory, one replayed through the database.""" + + @with_db_session(db_url=default_db_url) + def setUp(self, db_session): + _cleanup(db_session) + _seed_feed(db_session, MARCHED, OLD_CREATED) + _seed_feed(db_session, REPLAYED, OLD_CREATED) + db_session.commit() + + @with_db_session(db_url=default_db_url) + def tearDown(self, db_session): + _cleanup(db_session) + + @staticmethod + def registry(script): + """Patch the registry `_resolve_evaluators` reads, which is the one both paths use.""" + return patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", + [ScriptedEvaluator(script)], + ) + + @staticmethod + def march(stable_id, script, **kwargs): + with MarchTestCase.registry(script): + return backfill_seals( + stable_feed_ids=[stable_id], + start_date=MARCH_START, + end_date=MARCH_END, + dry_run=False, + **kwargs, + ) + + @staticmethod + def replay_through_db(stable_id, script): + """The same days, evaluated one `update_seals` call at a time. + + Uses the same midnight-UTC timestamps the march uses, so any difference in the final + state is the march's doing and not a difference in `now`. + """ + with MarchTestCase.registry(script): + for day in days_between(MARCH_START, MARCH_END): + update_seals( + stable_feed_ids=[stable_id], dry_run=False, now=day_start(day) + ) + + def state_of(self, stable_id): + row = criterion_rows(stable_id)[SealCriterionName.OFFICIAL.value] + return {column: getattr(row, column) for column in STATE_COLUMNS} + + +class TestMarchMatchesTheDatabaseReplay(MarchTestCase): + def test_a_clean_run_agrees(self): + script = _script_for([]) + self.march(MARCHED, script) + self.replay_through_db(REPLAYED, script) + self.assertEqual(self.state_of(MARCHED), self.state_of(REPLAYED)) + + def test_a_confirmed_failure_and_its_probation_agree(self): + """The case the backfill exists for: state that depends on the whole path.""" + script = _script_for(range(10, 46)) + self.march(MARCHED, script) + self.replay_through_db(REPLAYED, script) + + marched = self.state_of(MARCHED) + self.assertEqual(marched, self.state_of(REPLAYED)) + self.assertIsNotNone( + marched["last_confirmed_failure_at"], + "the 36-day streak must have outlasted the 30-day grace period", + ) + self.assertIsNotNone( + marched["probation_start"], "and recovery must have opened probation" + ) + + def test_an_absorbed_blip_agrees(self): + script = _script_for([20]) + self.march(MARCHED, script) + self.replay_through_db(REPLAYED, script) + + marched = self.state_of(MARCHED) + self.assertEqual(marched, self.state_of(REPLAYED)) + self.assertIsNone( + marched["last_confirmed_failure_at"], + "one day is well inside the grace period", + ) + + +class TestMarchWrites(MarchTestCase): + def test_only_the_final_day_is_snapshotted_by_default(self): + self.march(MARCHED, _script_for([20])) + self.assertEqual(snapshot_days(MARCHED), [MARCH_END]) + + def test_snapshot_mode_all_records_every_day(self): + self.march(MARCHED, _script_for([20]), snapshot_mode="all") + self.assertEqual(snapshot_days(MARCHED), days_between(MARCH_START, MARCH_END)) + + def test_snapshot_mode_none_records_nothing(self): + self.march(MARCHED, _script_for([20]), snapshot_mode="none") + self.assertEqual(snapshot_days(MARCHED), []) + + def test_the_seal_row_created_at_is_the_march_start(self): + """Left at its DEFAULT now(), Stable would fail on every simulated day.""" + self.march(MARCHED, _script_for([])) + self.assertEqual( + seal_row(MARCHED).created_at.astimezone(timezone.utc).date(), MARCH_START + ) + + def test_created_at_survives_a_re_backfill(self): + """Insert-only: a re-run must not reset a countdown already running.""" + self.march(MARCHED, _script_for([])) + first = seal_row(MARCHED).created_at + + with self.registry(_script_for([])): + backfill_seals( + stable_feed_ids=[MARCHED], + start_date=MARCH_START + timedelta(days=30), + end_date=MARCH_END, + dry_run=False, + only_missing=False, + ) + self.assertEqual(seal_row(MARCHED).created_at, first) + + def test_seal_earned_at_is_the_end_of_the_window(self): + report = self.march(MARCHED, _script_for([])) + self.assertEqual(report["seals_granted"], 1) + self.assertEqual( + seal_row(MARCHED).seal_earned_at.astimezone(timezone.utc).date(), MARCH_END + ) + + def test_the_report_counts_what_was_written(self): + report = self.march(MARCHED, _script_for([20])) + self.assertEqual(report["criterion_rows_written"], 1) + self.assertEqual(report["snapshot_rows_written"], 1) + self.assertEqual(report["granted_stable_ids"], [MARCHED]) + self.assertFalse(report["dry_run"]) + + +class TestResumeFromSnapshot(MarchTestCase): + def test_a_resume_starts_from_the_stored_snapshot(self): + """Seeded from the day before, the march inherits an open probation. + + Without the seed the same window is a clean cold start, so the difference is + entirely the snapshot's doing. + """ + # A first march that ends on probation, snapshotting every day. + self.march(MARCHED, _script_for(range(10, 46)), snapshot_mode="all") + self.assertIsNotNone(self.state_of(MARCHED)["probation_start"]) + + # Resume the tail of the window, with no failures in it at all. + with self.registry(_script_for([])): + backfill_seals( + stable_feed_ids=[MARCHED], + start_date=MARCH_START + timedelta(days=60), + end_date=MARCH_END, + dry_run=False, + only_missing=False, + resume_from_snapshot=True, + ) + + self.assertIsNotNone( + self.state_of(MARCHED)["probation_start"], + "the probation carried over from the seeded snapshot", + ) + + def test_without_the_flag_the_same_window_cold_starts(self): + self.march(MARCHED, _script_for(range(10, 46)), snapshot_mode="all") + + with self.registry(_script_for([])): + backfill_seals( + stable_feed_ids=[MARCHED], + start_date=MARCH_START + timedelta(days=60), + end_date=MARCH_END, + dry_run=False, + only_missing=False, + ) + + self.assertIsNone( + self.state_of(MARCHED)["probation_start"], + "a cold start carries no probation forward", + ) + + +class TestSimulateAndTrace(MarchTestCase): + """Forced per-day statuses, and the day-by-day trace they are there to make visible.""" + + def simulate(self, stable_id, **kwargs): + with self.registry(_script_for([])): + return backfill_seals( + stable_feed_ids=[stable_id], + start_date=MARCH_START, + end_date=MARCH_END, + dry_run=True, + **kwargs, + ) + + def test_a_simulated_run_never_writes(self): + """The reason simulate forces dry_run: a forced verdict in seal_criterion would be + indistinguishable from an earned one.""" + report = self.simulate(MARCHED, simulate={"official": {"fail": [0, 1]}}) + self.assertEqual(criterion_rows(MARCHED), {}) + self.assertIsNone(seal_row(MARCHED)) + self.assertEqual(report["criterion_rows_written"], 0) + + def _simulated_write(self, **kwargs): + with self.registry(_script_for([])): + return backfill_seals( + stable_feed_ids=[MARCHED], + start_date=MARCH_START, + end_date=MARCH_END, + dry_run=False, + only_missing=False, + simulate={"official": {"fail": [0]}}, + **kwargs, + ) + + def test_a_simulated_write_is_allowed_in_dev_with_the_flag(self): + """The override exists so a debounced state can be written and read back locally.""" + with patch.dict(os.environ, {"ENVIRONMENT": "dev"}): + report = self._simulated_write() + self.assertFalse(report["dry_run"]) + self.assertTrue( + report["simulated_write"], "the response is the only provenance there is" + ) + + def test_a_simulated_write_is_refused_in_production(self): + with patch.dict(os.environ, {"ENVIRONMENT": "prod"}): + with self.assertRaises(ValueError) as caught: + self._simulated_write() + self.assertIn("prod", str(caught.exception)) + + def test_a_simulated_write_is_refused_on_the_prod_tunnel_port(self): + """ENVIRONMENT describes the process; the port is the only hint about the target.""" + with patch.dict(os.environ, {"ENVIRONMENT": "local"}): + with patch( + "tasks.seal_of_reliability.backfill.simulation._connection_port", + return_value=9901, + ): + with self.assertRaises(ValueError) as caught: + self._simulated_write() + self.assertIn("9901", str(caught.exception)) + + def test_an_unset_environment_is_treated_as_production(self): + """Fail closed: a deployment that forgets ENVIRONMENT must not fabricate history.""" + with patch.dict(os.environ): + os.environ.pop("ENVIRONMENT", None) + with self.assertRaises(ValueError) as caught: + self._simulated_write() + self.assertIn("unset", str(caught.exception)) + + def test_a_forced_failure_reaches_the_state_machine(self): + """Day 0 is the first evaluation, so it gets no grace and confirms immediately.""" + report = self.simulate( + MARCHED, simulate={"official": {"fail": [0]}}, trace=True + ) + day_zero = trace_day(report, 0) + self.assertEqual(day_zero["observed_status"], "fail") + self.assertEqual(day_zero["confirmed_status"], "fail") + self.assertTrue(day_zero["simulated"]) + + def test_unnamed_days_fall_through_to_the_real_evaluator(self): + """A simulation is real data with overrides, not a synthetic run.""" + report = self.simulate( + MARCHED, simulate={"official": {"fail": [0]}}, trace=True + ) + day_one = trace_day(report, 1) + self.assertEqual(day_one["observed_status"], "pass") + self.assertFalse(day_one["simulated"]) + + def test_a_streak_past_the_grace_period_confirms_then_serves_probation(self): + """Days 1-39 fail, then the feed recovers — the whole arc in one trace. + + The stand-in's grace period is 30 days and the streak starts on day 1, so day 31 is + the first confirmed failure. Recovery on day 40 clears the status but opens + probation, which 34 remaining days cannot serve. + """ + report = self.simulate( + MARCHED, + simulate={"official": {"fail": list(range(1, 40))}}, + trace=True, + ) + self.assertEqual( + trace_day(report, 30)["confirmed_status"], "pass", "last day of grace" + ) + self.assertEqual( + trace_day(report, 31)["confirmed_status"], "fail", "grace outlasted" + ) + + last = trace_last(report) + self.assertEqual(last["observed_status"], "pass") + self.assertEqual(last["confirmed_status"], "pass", "recovered") + self.assertEqual(last["phase"], "on_probation") + self.assertIsNotNone(last["probation_start"]) + + def test_a_trace_row_carries_every_seal_criterion_field(self): + """The trace is the stored row plus provenance, not a summary of it. + + Derived from SealCriterionState, so a field added there appears here without anyone + remembering to widen the trace — and this fails if it ever stops being derived. + """ + from dataclasses import fields as dataclass_fields + + from tasks.seal_of_reliability.state_machine import SealCriterionState + + report = self.simulate( + MARCHED, simulate={"official": {"fail": [1]}}, trace=True + ) + row = trace_day(report, 0) + + expected = {f.name for f in dataclass_fields(SealCriterionState)} - { + "feed_id", + "criterion", + } + self.assertTrue( + expected.issubset(row), + f"trace is missing state fields: {sorted(expected - set(row))}", + ) + for name in ("day", "phase", "simulated", "reason", "criterion"): + self.assertIn(name, row) + self.assertNotIn( + "date", row, "dropped: evaluated_at is the same day by construction" + ) + + def test_the_carried_state_tracks_the_streak(self): + """first_observed_failure_at is set while failing and cleared on recovery.""" + report = self.simulate( + MARCHED, simulate={"official": {"fail": [1, 2]}}, trace=True + ) + days = {day: trace_day(report, day) for day in (0, 1, 2, 3)} + + self.assertIsNone(days[0]["first_observed_failure_at"]) + self.assertEqual(days[1]["first_observed_failure_at"], days[1]["evaluated_at"]) + self.assertEqual( + days[2]["first_observed_failure_at"], + days[1]["evaluated_at"], + "the streak keeps its start", + ) + self.assertIsNone(days[3]["first_observed_failure_at"], "cleared on recovery") + self.assertEqual( + days[3]["last_observed_failure_at"], + days[2]["evaluated_at"], + "but the history is never cleared", + ) + + def test_the_trace_accounts_for_every_marched_day(self): + """Collapsed stretches tile the march: contiguous, in order, none missing.""" + report = self.simulate(MARCHED, trace=True) + marched = (MARCH_END - MARCH_START).days + 1 + + next_expected = 0 + for entry in report["trace"]: + first = entry["first"] + last = entry.get("last", first) + self.assertEqual(first["day"], next_expected, "stretches are contiguous") + self.assertEqual(entry["days"], last["day"] - first["day"] + 1) + next_expected = last["day"] + 1 + self.assertEqual(next_expected, marched, "every marched day is accounted for") + + def test_the_trace_is_offset_from_the_feed_own_march_start(self): + """Day 0 is the feed's first evaluated day, not the window start.""" + report = self.simulate(MARCHED, trace=True) + first = trace_day(report, 0) + self.assertEqual(first["day"], 0) + self.assertEqual(first["evaluated_at"], MARCH_START.isoformat()) + + def test_an_unknown_criterion_is_rejected(self): + with self.assertRaises(ValueError) as caught: + self.simulate(MARCHED, simulate={"punctual": {"fail": [0]}}) + self.assertIn("punctual", str(caught.exception)) + + def test_an_unknown_status_is_rejected(self): + with self.assertRaises(ValueError) as caught: + self.simulate(MARCHED, simulate={"official": {"broken": [0]}}) + self.assertIn("broken", str(caught.exception)) + + def test_a_negative_offset_is_rejected(self): + with self.assertRaises(ValueError) as caught: + self.simulate(MARCHED, simulate={"official": {"fail": [-1]}}) + self.assertIn("negative", str(caught.exception)) + + def test_an_offset_past_the_march_is_rejected(self): + """A typo like day 400 in a short window would otherwise silently do nothing.""" + with self.assertRaises(ValueError) as caught: + self.simulate(MARCHED, simulate={"official": {"fail": [9999]}}) + self.assertIn("9999", str(caught.exception)) + + def test_the_report_echoes_what_was_simulated(self): + report = self.simulate( + MARCHED, simulate={"official": {"fail": [2], "unknown": [4]}} + ) + # String keys: the echo shares its dict with grace_days/probation_days, and jsonify + # sorts keys, which raises on a dict mixing str and int. + self.assertEqual(report["simulated"]["official"], {"2": "fail", "4": "unknown"}) + + def test_a_plain_dry_run_still_stops_at_the_plan(self): + """Only simulate or trace makes a dry run pay for the march.""" + report = self.simulate(MARCHED) + self.assertNotIn("trace", report) + + +class TestSimulatedPolicy(MarchTestCase): + """Lending a criterion a grace period and probation it does not have. + + The stand-in already has both, so these use `grace_days`/`probation_days` to *remove* + and to *change* them — the same mechanism that lets Official, which has neither, show + debouncing in a simulation. + """ + + def simulate(self, **kwargs): + with self.registry(_script_for([])): + return backfill_seals( + stable_feed_ids=[MARCHED], + start_date=MARCH_START, + end_date=MARCH_END, + dry_run=True, + trace=True, + **kwargs, + ) + + @staticmethod + def _day(report, day): + return trace_day(report, day) + + def test_a_lent_grace_period_absorbs_a_failure(self): + """Without an override this criterion confirms on day 1; with 14 days it holds.""" + report = self.simulate(simulate={"official": {"grace_days": 14, "fail": [1]}}) + day_one = self._day(report, 1) + self.assertEqual(day_one["observed_status"], "fail") + self.assertEqual(day_one["confirmed_status"], "pass", "held by the lent grace") + self.assertEqual(day_one["phase"], "in_grace_period") + + def test_a_removed_grace_period_confirms_immediately(self): + """null means the criterion has none, which is Official's real behaviour.""" + report = self.simulate(simulate={"official": {"grace_days": None, "fail": [1]}}) + self.assertEqual(self._day(report, 1)["confirmed_status"], "fail") + + def test_the_lent_grace_period_expires_on_schedule(self): + report = self.simulate( + simulate={"official": {"grace_days": 14, "fail": list(range(1, 20))}} + ) + self.assertEqual( + self._day(report, 14)["confirmed_status"], "pass", "last day inside grace" + ) + self.assertEqual( + self._day(report, 15)["confirmed_status"], "fail", "grace outlasted" + ) + + def test_a_lent_probation_opens_on_recovery(self): + report = self.simulate( + simulate={ + "official": { + "grace_days": None, + "probation_days": 180, + "fail": [1], + } + } + ) + recovered = self._day(report, 2) + self.assertEqual(recovered["confirmed_status"], "pass") + self.assertEqual(recovered["phase"], "on_probation") + + def test_a_removed_probation_never_opens_one(self): + report = self.simulate( + simulate={ + "official": { + "grace_days": None, + "probation_days": None, + "fail": [1], + } + } + ) + recovered = self._day(report, 2) + self.assertEqual(recovered["confirmed_status"], "pass") + self.assertEqual(recovered["phase"], "steady") + self.assertIsNone(recovered["probation_start"]) + + def test_a_shorter_probation_is_served_sooner(self): + report = self.simulate( + simulate={ + "official": {"grace_days": None, "probation_days": 5, "fail": [1]} + } + ) + # Probation opens on day 2 and clears once five days have passed. + self.assertEqual(self._day(report, 6)["phase"], "on_probation") + self.assertEqual(self._day(report, 7)["phase"], "steady") + + def test_omitting_the_keys_keeps_the_criterion_own_policy(self): + """The stand-in's own 30-day grace, untouched — a streak from day 1 confirms on 31.""" + report = self.simulate(simulate={"official": {"fail": list(range(1, 40))}}) + self.assertEqual(self._day(report, 30)["confirmed_status"], "pass") + self.assertEqual(self._day(report, 31)["confirmed_status"], "fail") + + def test_the_report_echoes_the_lent_policy(self): + report = self.simulate( + simulate={ + "official": {"grace_days": 14, "probation_days": 180, "fail": [1]} + } + ) + echoed = report["simulated"]["official"] + self.assertEqual(echoed["grace_days"], 14) + self.assertEqual(echoed["probation_days"], 180) + self.assertEqual(echoed["1"], "fail") + # This echo is the shape that used to 500: offsets and policy keys in one dict, which + # only fails once Flask serializes it, so assert it survives that too. + self.assertEqual(json.loads(json.dumps(echoed, sort_keys=True)), echoed) + + def test_a_negative_period_is_rejected(self): + with self.assertRaises(ValueError) as caught: + self.simulate(simulate={"official": {"grace_days": -1}}) + self.assertIn("negative", str(caught.exception)) + + def test_a_non_numeric_period_is_rejected(self): + with self.assertRaises(ValueError) as caught: + self.simulate(simulate={"official": {"probation_days": "a fortnight"}}) + self.assertIn("probation_days", str(caught.exception)) + + def test_a_lent_policy_never_writes_in_production(self): + """Same rule as forced verdicts: a fabricated policy must not reach real tables.""" + with patch.dict(os.environ, {"ENVIRONMENT": "prod"}), self.assertRaises( + ValueError + ): + with self.registry(_script_for([])): + backfill_seals( + stable_feed_ids=[MARCHED], + start_date=MARCH_START, + end_date=MARCH_END, + dry_run=False, + simulate={"official": {"grace_days": 14}}, + ) + + +class TestSimulatedBaseline(MarchTestCase): + """`default` is what every unnamed day observes; named days are exceptions on top. + + Without it, unnamed days fall through to the evaluator — which says nothing useful for a + criterion whose source data is absent, as `fresh_coverage` is on a local database. + """ + + def simulate(self, **kwargs): + # The stand-in passes every day, so a `fail` baseline can only come from the payload. + with self.registry(_script_for([])): + return backfill_seals( + stable_feed_ids=[MARCHED], + start_date=MARCH_START, + end_date=MARCH_END, + dry_run=True, + trace=True, + **kwargs, + ) + + def test_a_baseline_replaces_the_evaluator_on_every_unnamed_day(self): + report = self.simulate(simulate={"official": {"default": "fail"}}) + for day in (0, 1, 5): + row = trace_day(report, day) + self.assertEqual(row["observed_status"], "fail", f"day {day}") + self.assertTrue(row["simulated"]) + self.assertIn("by default", row["reason"]) + + def test_a_named_day_overrides_the_baseline(self): + report = self.simulate(simulate={"official": {"default": "fail", "pass": [2]}}) + self.assertEqual(trace_day(report, 1)["observed_status"], "fail") + day_two = trace_day(report, 2) + self.assertEqual(day_two["observed_status"], "pass", "the exception wins") + self.assertIn("on day 2", day_two["reason"]) + + def test_without_a_baseline_unnamed_days_still_fall_through(self): + report = self.simulate(simulate={"official": {"fail": [2]}}) + self.assertFalse(trace_day(report, 1)["simulated"], "the evaluator answered") + + def test_the_report_echoes_the_baseline(self): + report = self.simulate(simulate={"official": {"default": "fail", "pass": [2]}}) + echoed = report["simulated"]["official"] + self.assertEqual(echoed["default"], "fail") + self.assertEqual(echoed["2"], "pass") + + def test_an_unknown_baseline_status_is_rejected(self): + with self.assertRaises(ValueError) as caught: + self.simulate(simulate={"official": {"default": "excellent"}}) + self.assertIn("default", str(caught.exception)) + + def test_a_baseline_alone_is_enough_to_march(self): + """No named day, so nothing to range-check — the baseline still forces the march.""" + report = self.simulate(simulate={"official": {"default": "unknown"}}) + self.assertEqual(trace_day(report, 0)["observed_status"], "unknown") + + +if __name__ == "__main__": + unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill_fanout.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill_fanout.py new file mode 100644 index 000000000..855c6bdea --- /dev/null +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill_fanout.py @@ -0,0 +1,350 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Unit tests for the seal backfill Cloud Tasks fan-out (issue #1763). + +Mirrors test_seal_orchestrator_fanout.py: the producer/worker orchestration with the DB and +Cloud Tasks boundaries mocked. The march itself is covered against the real database by +test_seal_backfill.py. +""" + +import unittest +from datetime import date +from unittest.mock import patch + +_FANOUT = "tasks.seal_of_reliability.fanout" +_PLAN = "tasks.seal_of_reliability.backfill.seal_backfill_orchestrator" +_WORKER = "tasks.seal_of_reliability.backfill.seal_backfill_worker" + +START = date(2025, 6, 1) +END = date(2026, 6, 1) + + +class TestBackfillOrchestrator(unittest.TestCase): + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) + @patch(f"{_PLAN}.iter_eligible_stable_ids") + @patch(f"{_PLAN}.count_eligible_feeds", return_value=5) + def test_enqueues_worker_per_batch_plus_monitor( + self, count_mock, iter_mock, enqueue_mock, start_run_mock + ): + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + iter_mock.return_value = iter( + [["mdb-1", "mdb-2"], ["mdb-3", "mdb-4"], ["mdb-5"]] + ) + + result = seal_backfill_orchestrator_handler( + { + "dry_run": False, + "batch_size": 2, + "start_date": START.isoformat(), + "end_date": END.isoformat(), + } + ) + + start_run_mock.assert_called_once() + in_body = [c.kwargs["in_body_task"] for c in enqueue_mock.call_args_list] + self.assertEqual(in_body.count("seal_backfill_worker"), 3) + self.assertEqual(in_body.count("seal_orchestrator_monitor"), 1) + self.assertEqual(result["total_feeds"], 5) + self.assertEqual(result["batches"], 3) + self.assertEqual(result["enqueued"], 3) + + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) + @patch(f"{_PLAN}.iter_eligible_stable_ids") + @patch(f"{_PLAN}.count_eligible_feeds", return_value=2) + def test_every_worker_gets_the_same_explicit_window( + self, count_mock, iter_mock, enqueue_mock, start_run_mock + ): + """The whole reason the producer resolves the window. + + Left to each worker to default, two started either side of midnight would march to + different final days. + """ + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + iter_mock.return_value = iter([["mdb-1"], ["mdb-2"]]) + seal_backfill_orchestrator_handler( + { + "dry_run": False, + "batch_size": 1, + "start_date": START.isoformat(), + "end_date": END.isoformat(), + } + ) + + windows = [ + (c.kwargs["payload"]["start_date"], c.kwargs["payload"]["end_date"]) + for c in enqueue_mock.call_args_list + if c.kwargs["in_body_task"] == "seal_backfill_worker" + ] + self.assertEqual(windows, [(START.isoformat(), END.isoformat())] * 2) + + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) + @patch(f"{_PLAN}.iter_eligible_stable_ids") + @patch(f"{_PLAN}.count_eligible_feeds", return_value=1) + def test_the_monitor_is_told_which_tracker_to_settle( + self, count_mock, iter_mock, enqueue_mock, start_run_mock + ): + """The monitor is shared with the nightly run, so the task_name has to travel.""" + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + SEAL_BACKFILL_TASK_NAME, + seal_backfill_orchestrator_handler, + ) + + iter_mock.return_value = iter([["mdb-1"]]) + seal_backfill_orchestrator_handler( + {"dry_run": False, "end_date": END.isoformat()} + ) + + monitor = next( + c + for c in enqueue_mock.call_args_list + if c.kwargs["in_body_task"] == "seal_orchestrator_monitor" + ) + self.assertEqual( + monitor.kwargs["payload"]["task_name"], SEAL_BACKFILL_TASK_NAME + ) + + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) + @patch(f"{_PLAN}.iter_eligible_stable_ids") + @patch(f"{_PLAN}.count_eligible_feeds", return_value=3) + def test_only_missing_narrows_the_candidate_set( + self, count_mock, iter_mock, enqueue_mock, start_run_mock + ): + """#1763's scope: feeds with no stored state. It is the eligibility predicate.""" + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + iter_mock.return_value = iter([["mdb-1", "mdb-2", "mdb-3"]]) + seal_backfill_orchestrator_handler( + {"dry_run": False, "end_date": END.isoformat()} + ) + + self.assertTrue(count_mock.call_args.kwargs["exclude_backfilled"]) + self.assertTrue(iter_mock.call_args.kwargs["exclude_backfilled"]) + + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) + @patch(f"{_PLAN}.iter_eligible_stable_ids") + @patch(f"{_PLAN}.count_eligible_feeds", return_value=3) + def test_only_missing_false_widens_it( + self, count_mock, iter_mock, enqueue_mock, start_run_mock + ): + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + iter_mock.return_value = iter([["mdb-1", "mdb-2", "mdb-3"]]) + seal_backfill_orchestrator_handler( + {"dry_run": False, "end_date": END.isoformat(), "only_missing": False} + ) + + self.assertFalse(count_mock.call_args.kwargs["exclude_backfilled"]) + + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task") + @patch(f"{_PLAN}.iter_eligible_stable_ids") + @patch(f"{_PLAN}.count_eligible_feeds", return_value=4) + def test_dry_run_enqueues_nothing( + self, count_mock, iter_mock, enqueue_mock, start_run_mock + ): + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + result = seal_backfill_orchestrator_handler({"end_date": END.isoformat()}) + + enqueue_mock.assert_not_called() + start_run_mock.assert_not_called() + self.assertTrue(result["dry_run"]) + self.assertEqual(result["enqueued"], 0) + self.assertEqual(result["total_feeds"], 4) + + @patch(f"{_FANOUT}.mark_enqueue_failed") + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=False) + @patch(f"{_PLAN}.iter_eligible_stable_ids") + @patch(f"{_PLAN}.count_eligible_feeds", return_value=2) + def test_a_failed_enqueue_fails_its_batch_immediately( + self, count_mock, iter_mock, enqueue_mock, start_run_mock, failed_mock + ): + """Otherwise the batch sits `triggered` until the deadline forces the run failed.""" + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + iter_mock.return_value = iter([["mdb-1"], ["mdb-2"]]) + result = seal_backfill_orchestrator_handler( + {"dry_run": False, "batch_size": 1, "end_date": END.isoformat()} + ) + + self.assertEqual(result["enqueued"], 0) + self.assertEqual(failed_mock.call_count, 2) + + def test_a_bad_window_fails_at_the_producer(self): + """One failure at the producer beats the same failure once per batch.""" + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + with self.assertRaises(ValueError): + seal_backfill_orchestrator_handler( + { + "start_date": END.isoformat(), + "end_date": START.isoformat(), + } + ) + + def test_an_unknown_snapshot_mode_fails_at_the_producer(self): + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + with self.assertRaises(ValueError): + seal_backfill_orchestrator_handler({"snapshot_mode": "occasionally"}) + + +class TestBackfillWorker(unittest.TestCase): + @patch(f"{_WORKER}._mark_entry") + @patch(f"{_WORKER}.backfill_seals", return_value={"total_feeds": 2}) + def test_marks_the_batch_completed(self, backfill_mock, mark_mock): + from tasks.seal_of_reliability.backfill.seal_backfill_worker import ( + seal_backfill_worker_handler, + ) + + result = seal_backfill_worker_handler( + { + "run_id": "r1", + "batch_id": "batch-0000", + "stable_feed_ids": ["mdb-1", "mdb-2"], + "start_date": START.isoformat(), + "end_date": END.isoformat(), + } + ) + + self.assertEqual(result["status"], "ok") + self.assertFalse(backfill_mock.call_args.kwargs["dry_run"]) + self.assertEqual(backfill_mock.call_args.kwargs["start_date"], START) + self.assertEqual(backfill_mock.call_args.kwargs["end_date"], END) + mark_mock.assert_called_once() + self.assertEqual(mark_mock.call_args.kwargs["result"], {"total_feeds": 2}) + + @patch(f"{_WORKER}._mark_entry") + @patch(f"{_WORKER}.backfill_seals", side_effect=RuntimeError("db down")) + def test_marks_the_batch_failed_and_re_raises(self, backfill_mock, mark_mock): + """Re-raised so Cloud Tasks retries; the tracker entry records the reason.""" + from tasks.seal_of_reliability.backfill.seal_backfill_worker import ( + seal_backfill_worker_handler, + ) + + with self.assertRaises(RuntimeError): + seal_backfill_worker_handler( + { + "run_id": "r1", + "batch_id": "batch-0000", + "stable_feed_ids": ["mdb-1"], + "start_date": START.isoformat(), + "end_date": END.isoformat(), + } + ) + self.assertIn("db down", mark_mock.call_args.kwargs["error"]) + + def test_a_worker_never_defaults_the_window(self): + """The run's window belongs to the run, not to when a batch happened to execute.""" + from tasks.seal_of_reliability.backfill.seal_backfill_worker import ( + seal_backfill_worker_handler, + ) + + with self.assertRaises(ValueError) as caught: + seal_backfill_worker_handler( + { + "run_id": "r1", + "batch_id": "batch-0000", + "stable_feed_ids": ["mdb-1"], + "start_date": START.isoformat(), + } + ) + self.assertIn("end_date", str(caught.exception)) + + def test_required_fields_are_checked(self): + from tasks.seal_of_reliability.backfill.seal_backfill_worker import ( + seal_backfill_worker_handler, + ) + + for payload in ( + {"batch_id": "b", "stable_feed_ids": ["mdb-1"]}, + {"run_id": "r", "stable_feed_ids": ["mdb-1"]}, + {"run_id": "r", "batch_id": "b", "stable_feed_ids": []}, + ): + with self.subTest(payload=payload): + with self.assertRaises(ValueError): + seal_backfill_worker_handler(payload) + + +class TestSharedMonitor(unittest.TestCase): + def test_it_defaults_to_the_nightly_tracker(self): + from tasks.seal_of_reliability.orchestrator.seal_orchestrator import ( + SEAL_ORCHESTRATOR_TASK_NAME, + ) + from tasks.seal_of_reliability.orchestrator import seal_orchestrator_monitor + + with patch.object(seal_orchestrator_monitor, "_monitor") as monitor_mock: + seal_orchestrator_monitor.seal_orchestrator_monitor_handler( + {"run_id": "r1"} + ) + self.assertEqual(monitor_mock.call_args.args[1], SEAL_ORCHESTRATOR_TASK_NAME) + + def test_it_settles_the_backfill_tracker_when_told_to(self): + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + SEAL_BACKFILL_TASK_NAME, + ) + from tasks.seal_of_reliability.orchestrator import seal_orchestrator_monitor + + with patch.object(seal_orchestrator_monitor, "_monitor") as monitor_mock: + seal_orchestrator_monitor.seal_orchestrator_monitor_handler( + {"run_id": "r1", "task_name": SEAL_BACKFILL_TASK_NAME} + ) + self.assertEqual(monitor_mock.call_args.args[1], SEAL_BACKFILL_TASK_NAME) + + +class TestBatchSizeDefault(unittest.TestCase): + def test_backfill_batches_are_smaller_than_nightly_ones(self): + """A batch marches a year per feed, so its cost scales with days as well as feeds.""" + from tasks.seal_of_reliability.backfill import seal_backfill_orchestrator + from tasks.seal_of_reliability.orchestrator import seal_orchestrator + + self.assertLess( + seal_backfill_orchestrator.DEFAULT_BATCH_SIZE, + seal_orchestrator.DEFAULT_BATCH_SIZE, + ) + self.assertGreater( + seal_backfill_orchestrator.DEFAULT_DEADLINE_SECONDS, + seal_orchestrator.DEFAULT_DEADLINE_SECONDS, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/orchestrator/test_seal_monitor_aggregation.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/orchestrator/test_seal_monitor_aggregation.py new file mode 100644 index 000000000..4705999dd --- /dev/null +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/orchestrator/test_seal_monitor_aggregation.py @@ -0,0 +1,262 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""`_aggregate_batches` against real TaskExecutionLog rows. + +Every test in test_seal_orchestrator_fanout.py patches this function out — it needs a real +session, and those tests drive the monitor with a MagicMock. Mocking it hides both whether it +sums correctly and whether it sums the right keys, so it is covered here instead. + +It is what turns each batch's stored report into the run-level one, which after a manual +backfill is the only thing an operator sees. +""" + +import unittest + +from sqlalchemy import delete + +from shared.database.database import with_db_session +from shared.database_gen.sqlacodegen_models import TaskExecutionLog +from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + SEAL_BACKFILL_TASK_NAME, +) +from tasks.seal_of_reliability.orchestrator.seal_orchestrator import ( + SEAL_ORCHESTRATOR_TASK_NAME, +) +from tasks.seal_of_reliability.orchestrator.seal_orchestrator_monitor import ( + MAX_REPORTED_IDS, + _aggregate_batches, + _parse_iso, +) +from test_shared.test_utils.database_utils import default_db_url + +RUN = "seal-agg-test-run" +OTHER_RUN = "seal-agg-test-other" + + +def _batch(task_name, run_id, entity_id, metadata): + return TaskExecutionLog( + task_name=task_name, + run_id=run_id, + entity_id=entity_id, + status="completed", + metadata_=metadata, + ) + + +class AggregationTestCase(unittest.TestCase): + @with_db_session(db_url=default_db_url) + def setUp(self, db_session): + self._cleanup(db_session) + + @with_db_session(db_url=default_db_url) + def tearDown(self, db_session): + self._cleanup(db_session) + + @staticmethod + def _cleanup(db_session): + db_session.execute( + delete(TaskExecutionLog).where( + TaskExecutionLog.run_id.in_([RUN, OTHER_RUN]) + ) + ) + db_session.commit() + + @staticmethod + @with_db_session(db_url=default_db_url) + def _seed(entries, db_session=None): + for entry in entries: + db_session.add(entry) + db_session.commit() + + @staticmethod + @with_db_session(db_url=default_db_url) + def _aggregate(task_name=SEAL_ORCHESTRATOR_TASK_NAME, run_id=RUN, db_session=None): + return _aggregate_batches(db_session, run_id, task_name) + + +class TestAggregateBatches(AggregationTestCase): + def test_sums_across_batches(self): + self._seed( + [ + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, + RUN, + "batch-0000", + { + "total_feeds": 10, + "criterion_rows_written": 20, + "seals_granted": 2, + "seals_revoked": 1, + "granted_stable_ids": ["a", "b"], + "revoked_stable_ids": ["c"], + }, + ), + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, + RUN, + "batch-0001", + { + "total_feeds": 5, + "criterion_rows_written": 7, + "seals_granted": 1, + "seals_revoked": 0, + "granted_stable_ids": ["d"], + }, + ), + ] + ) + + result = self._aggregate() + self.assertEqual(result["total_feeds_evaluated"], 15) + self.assertEqual(result["criterion_rows_written"], 27) + self.assertEqual(result["seals_granted"], 3) + self.assertEqual(result["seals_revoked"], 1) + self.assertEqual(sorted(result["granted_stable_ids"]), ["a", "b", "d"]) + self.assertEqual(result["revoked_stable_ids"], ["c"]) + self.assertEqual(result["ids_omitted"], 0) + + def test_a_backfill_snapshot_count_survives_to_the_run_report(self): + """The key a backfill batch reports and a nightly one does not. + + It was being dropped at aggregation, so a backfill's snapshot count never reached + the operator who triggered the run. + """ + self._seed( + [ + _batch( + SEAL_BACKFILL_TASK_NAME, + RUN, + "batch-0000", + {"total_feeds": 3, "snapshot_rows_written": 18}, + ), + _batch( + SEAL_BACKFILL_TASK_NAME, + RUN, + "batch-0001", + {"total_feeds": 2, "snapshot_rows_written": 12}, + ), + ] + ) + + result = self._aggregate(task_name=SEAL_BACKFILL_TASK_NAME) + self.assertEqual(result["snapshot_rows_written"], 30) + self.assertEqual(result["total_feeds_evaluated"], 5) + + def test_a_nightly_batch_contributes_zero_for_keys_it_never_reports(self): + """One aggregation serves both fan-outs, so a missing key must not raise.""" + self._seed( + [ + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, + RUN, + "batch-0000", + {"total_feeds": 4, "criterion_rows_written": 4}, + ) + ] + ) + + result = self._aggregate() + self.assertEqual(result["snapshot_rows_written"], 0) + + def test_it_only_sums_its_own_run(self): + self._seed( + [ + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, RUN, "batch-0000", {"total_feeds": 4} + ), + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, + OTHER_RUN, + "batch-0000", + {"total_feeds": 99}, + ), + ] + ) + self.assertEqual(self._aggregate()["total_feeds_evaluated"], 4) + + def test_it_only_sums_its_own_task_name(self): + """The two fan-outs share this function; a backfill run must not absorb a nightly one.""" + self._seed( + [ + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, RUN, "batch-0000", {"total_feeds": 4} + ), + _batch(SEAL_BACKFILL_TASK_NAME, RUN, "batch-0001", {"total_feeds": 50}), + ] + ) + self.assertEqual(self._aggregate()["total_feeds_evaluated"], 4) + self.assertEqual( + self._aggregate(task_name=SEAL_BACKFILL_TASK_NAME)["total_feeds_evaluated"], + 50, + ) + + def test_a_batch_with_no_metadata_is_skipped(self): + self._seed( + [ + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, RUN, "batch-0000", {"total_feeds": 4} + ), + _batch(SEAL_ORCHESTRATOR_TASK_NAME, RUN, "batch-0001", {}), + ] + ) + self.assertEqual(self._aggregate()["total_feeds_evaluated"], 4) + + def test_a_run_with_no_batches_aggregates_to_zero(self): + result = self._aggregate() + self.assertEqual(result["total_feeds_evaluated"], 0) + self.assertEqual(result["granted_stable_ids"], []) + self.assertEqual(result["ids_omitted"], 0) + + def test_the_id_lists_are_capped_and_the_overflow_counted(self): + """The seal tables hold every transition; this only bounds the response size.""" + granted = [f"mdb-{n}" for n in range(MAX_REPORTED_IDS + 5)] + self._seed( + [ + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, + RUN, + "batch-0000", + {"granted_stable_ids": granted}, + ) + ] + ) + + result = self._aggregate() + self.assertEqual(len(result["granted_stable_ids"]), MAX_REPORTED_IDS) + self.assertEqual(result["ids_omitted"], 5) + + +class TestParseIso(unittest.TestCase): + """The deadline check silently loses its guard if this returns None, so pin the branches.""" + + def test_absent_is_none(self): + self.assertIsNone(_parse_iso(None)) + self.assertIsNone(_parse_iso("")) + + def test_unparseable_is_none_rather_than_raising(self): + self.assertIsNone(_parse_iso("not a timestamp")) + + def test_naive_is_read_as_utc(self): + parsed = _parse_iso("2026-06-01T12:00:00") + self.assertEqual(parsed.tzinfo, __import__("datetime").timezone.utc) + + def test_offset_is_preserved(self): + self.assertIsNotNone(_parse_iso("2026-06-01T12:00:00+02:00").tzinfo) + + +if __name__ == "__main__": + unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/orchestrator/test_seal_orchestrator_fanout.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/orchestrator/test_seal_orchestrator_fanout.py index 843175556..4184afc4c 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/orchestrator/test_seal_orchestrator_fanout.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/orchestrator/test_seal_orchestrator_fanout.py @@ -32,12 +32,13 @@ # seal_orchestrator (producer) # --------------------------------------------------------------------------- +_FANOUT = "tasks.seal_of_reliability.fanout" _PLAN = "tasks.seal_of_reliability.orchestrator.seal_orchestrator" class TestSealOrchestratorHandler(unittest.TestCase): - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=5) def test_enqueues_worker_per_batch_plus_monitor( @@ -64,8 +65,8 @@ def test_enqueues_worker_per_batch_plus_monitor( self.assertEqual(result["enqueued"], 3) self.assertFalse(result["dry_run"]) - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=1) def test_dynamic_task_names_use_prefix( @@ -82,8 +83,8 @@ def test_dynamic_task_names_use_prefix( self.assertTrue(all(n.startswith("seal-orchestrator-") for n in names)) self.assertTrue(any(n.startswith("seal-orchestrator-monitor-") for n in names)) - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=2) def test_dry_run_enqueues_nothing( @@ -101,8 +102,8 @@ def test_dry_run_enqueues_nothing( self.assertEqual(result["enqueued"], 0) self.assertEqual(result["batches"], 2) - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=0) def test_no_eligible_feeds_enqueues_nothing( @@ -120,15 +121,16 @@ def test_no_eligible_feeds_enqueues_nothing( self.assertEqual(result["enqueued"], 0) self.assertEqual(result["batches"], 0) - @patch(f"{_PLAN}._mark_enqueue_failed") - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue") + @patch(f"{_FANOUT}.mark_enqueue_failed") + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task") @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=1) def test_failed_enqueue_marks_batch_failed_immediately( self, count_mock, iter_mock, enqueue_mock, start_run_mock, mark_failed_mock ): from tasks.seal_of_reliability.orchestrator.seal_orchestrator import ( + SEAL_ORCHESTRATOR_TASK_NAME, seal_orchestrator_handler, ) @@ -138,12 +140,13 @@ def test_failed_enqueue_marks_batch_failed_immediately( seal_orchestrator_handler({"dry_run": False, "batch_size": 1}) mark_failed_mock.assert_called_once() - call_args = mark_failed_mock.call_args[0] - self.assertTrue(call_args[0].startswith("seal-")) - self.assertEqual(call_args[1], "batch-0000") + task_name, run_id, batch_id = mark_failed_mock.call_args[0] + self.assertEqual(task_name, SEAL_ORCHESTRATOR_TASK_NAME) + self.assertTrue(run_id.startswith("seal-")) + self.assertEqual(batch_id, "batch-0000") - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue") + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task") @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds") def test_non_positive_batch_size_raises( @@ -163,9 +166,9 @@ def test_non_positive_batch_size_raises( enqueue_mock.assert_not_called() start_run_mock.assert_not_called() - @patch(f"{_PLAN}._mark_enqueue_failed") - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.mark_enqueue_failed") + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=6) def test_stream_yields_fewer_batches_than_planned_marks_leftover_failed( @@ -176,6 +179,7 @@ def test_stream_yields_fewer_batches_than_planned_marks_leftover_failed( leftover pre-registered batch_id must be failed immediately, not left `triggered` for the monitor's deadline to eventually notice.""" from tasks.seal_of_reliability.orchestrator.seal_orchestrator import ( + SEAL_ORCHESTRATOR_TASK_NAME, seal_orchestrator_handler, ) @@ -187,16 +191,18 @@ def test_stream_yields_fewer_batches_than_planned_marks_leftover_failed( mark_failed_mock.assert_called_once() call_args, call_kwargs = mark_failed_mock.call_args - self.assertTrue(call_args[0].startswith("seal-")) - self.assertEqual(call_args[1], "batch-0002") + task_name, run_id, batch_id = call_args + self.assertEqual(task_name, SEAL_ORCHESTRATOR_TASK_NAME) + self.assertTrue(run_id.startswith("seal-")) + self.assertEqual(batch_id, "batch-0002") self.assertEqual( call_kwargs["error_message"], "no eligible-feed data for this batch (count/stream mismatch)", ) - @patch(f"{_PLAN}._mark_enqueue_failed") - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.mark_enqueue_failed") + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=2) def test_stream_yields_more_batches_than_planned_logs_and_does_not_mark_failed( @@ -213,13 +219,14 @@ def test_stream_yields_more_batches_than_planned_logs_and_does_not_mark_failed( # chunk. iter_mock.return_value = iter([["mdb-1"], ["mdb-2"], ["mdb-3"]]) - with self.assertLogs( - "tasks.seal_of_reliability.orchestrator.seal_orchestrator", level="ERROR" - ) as log_ctx: + with self.assertLogs(_FANOUT, level="ERROR") as log_ctx: result = seal_orchestrator_handler({"dry_run": False, "batch_size": 1}) mark_failed_mock.assert_not_called() - self.assertTrue(any("newly-eligible" in msg for msg in log_ctx.output)) + self.assertTrue( + any("more batches than the plan-time count" in m for m in log_ctx.output) + ) + self.assertTrue(any("seal_orchestrator" in m for m in log_ctx.output)) self.assertEqual(result["enqueued"], 2) diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py index a37940102..9f496cc37 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py @@ -16,7 +16,7 @@ """Unit tests for the seal criterion evaluators. No database.""" import unittest -from datetime import datetime, timedelta, timezone +from datetime import date, datetime, timedelta, timezone from shared.common.seal_criteria import ( FUTURE_COVERAGE_HORIZON, @@ -25,7 +25,11 @@ CriterionStatus, SealCriterionName, ) -from tasks.seal_of_reliability.context import FeedSealContext, LatestDataset +from tasks.seal_of_reliability.context import FeedSealContext, collect_inputs +from tasks.seal_of_reliability.evaluators.fresh_coverage import ( + FreshCoverageInputs, + LatestDataset, +) from tasks.seal_of_reliability.evaluators import ( EVALUATORS, CriterionEvaluator, @@ -149,6 +153,68 @@ def test_reason_names_the_offending_value(self): self.assertIn("None", result.reason) +class TestLoadInputs(unittest.TestCase): + """The `load_inputs` hook, and how what it loads reaches `_evaluate`.""" + + def test_no_criterion_queries_for_an_empty_batch(self): + """A criterion reading only day-invariant context fields has nothing to load. + + None means "nothing to load", not "the load failed" — Official and Stable read the + feed row off the context and never need a query. Fresh overrides the loader, because + which dataset is "the latest" changes on every day a march evaluates, but it must + still answer an empty batch without touching the session. + + The session is `object()` on purpose: any evaluator that queried here would raise. + """ + for evaluator in EVALUATORS: + with self.subTest(criterion=evaluator.name): + loaded = evaluator.load_inputs(object(), [], [NOW.date()]) + if isinstance(evaluator, FreshCoverageEvaluator): + self.assertIsInstance(loaded, FreshCoverageInputs) + else: + self.assertIsNone(loaded) + + def test_each_criterion_is_asked_once_for_the_whole_batch(self): + """One call per criterion, carrying every feed and every day. + + This is the property the backfill depends on: a criterion loading per day instead + would turn a year's march into several thousand queries. + """ + calls = [] + + class Recording(CriterionEvaluator): + name = SealCriterionName.AVAILABLE + + def load_inputs(self, db_session, feeds, days): + calls.append((tuple(feeds), tuple(days))) + return {"loaded": True} + + def _evaluate(self, ctx): + return CriterionStatus.PASS, "recorded" + + feeds = ["feed-1", "feed-2"] + days = [date(2026, 5, 30), date(2026, 5, 31), NOW.date()] + inputs = collect_inputs(object(), feeds, days, [Recording()]) + + self.assertEqual(calls, [(("feed-1", "feed-2"), tuple(days))]) + self.assertEqual(inputs, {SealCriterionName.AVAILABLE: {"loaded": True}}) + + def test_a_criterion_reaches_only_its_own_inputs(self): + ctx = _ctx( + inputs={ + SealCriterionName.AVAILABLE: "available-inputs", + SealCriterionName.COMPLIANT: "compliant-inputs", + } + ) + self.assertEqual( + ctx.inputs_for(SealCriterionName.AVAILABLE), "available-inputs" + ) + self.assertIsNone(ctx.inputs_for(SealCriterionName.OFFICIAL)) + + def test_context_defaults_to_no_inputs(self): + self.assertIsNone(_ctx().inputs_for(SealCriterionName.AVAILABLE)) + + class TestStable(unittest.TestCase): """`feed.created_at <= now - 180 days` and the producer URL is not flagged unstable.""" @@ -247,15 +313,35 @@ class TestFreshCoverage(unittest.TestCase): """`latest dataset.service_date_range_end >= now + 7 days`.""" @staticmethod - def _dataset(coverage_end): - return LatestDataset( - dataset_id="mdb-1-202606010000", - downloaded_at=NOW - timedelta(days=1), - service_date_range_end=coverage_end, + def _inputs(coverage_end): + """The criterion's own loaded inputs, holding one dataset for `feed-1`.""" + return FreshCoverageInputs( + { + "feed-1": [ + LatestDataset( + dataset_id="mdb-1-202606010000", + downloaded_at=NOW - timedelta(days=1), + service_date_range_end=coverage_end, + ) + ] + } ) - def _fresh_ctx(self, coverage_end=NOW + timedelta(days=90), **overrides): - defaults = {"latest_dataset": self._dataset(coverage_end)} + def _fresh_ctx( + self, coverage_end=NOW + timedelta(days=90), dataset=True, **overrides + ): + """A context whose Fresh inputs were loaded, with or without a dataset in them. + + `dataset=False` is a feed that had none as of `now` — an empty load, which is not the + same thing as a load that never ran (see `test_unloaded_inputs_say_so`). + """ + defaults = { + "inputs": { + SealCriterionName.FRESH_COVERAGE: ( + self._inputs(coverage_end) if dataset else FreshCoverageInputs({}) + ) + } + } defaults.update(overrides) return _ctx(**defaults) @@ -299,7 +385,7 @@ def test_a_seasonal_feed_is_not_applicable_even_with_no_dataset(self): """Applicability is a property of the feed, so it is settled before the inputs.""" self.assertIs( FreshCoverageEvaluator() - .evaluate(self._fresh_ctx(seasonal=True, latest_dataset=None)) + .evaluate(self._fresh_ctx(seasonal=True, dataset=False)) .observed_status, CriterionStatus.NOT_APPLICABLE, ) @@ -316,7 +402,7 @@ def test_a_non_seasonal_feed_is_evaluated(self): def test_no_latest_dataset_is_unknown(self): """Not a failure: a feed we have never fetched says nothing about its freshness.""" - result = FreshCoverageEvaluator().evaluate(self._fresh_ctx(latest_dataset=None)) + result = FreshCoverageEvaluator().evaluate(self._fresh_ctx(dataset=False)) self.assertIs(result.observed_status, CriterionStatus.UNKNOWN) self.assertIn("no latest dataset", result.reason) @@ -331,6 +417,56 @@ def test_has_a_grace_period_and_serves_probation(self): self.assertEqual(FreshCoverageEvaluator().grace_period, timedelta(days=14)) self.assertEqual(FreshCoverageEvaluator().probation_period, PROBATION_PERIOD) + def test_unloaded_inputs_say_so(self): + """A context built without running the loader is a bug, not a missing dataset. + + Both end as UNKNOWN, because a raise would take down a whole nightly run over the + catalogue, but the reason has to name the real cause: a silent "no latest dataset" + across every feed of a backfill is exactly what going unnoticed looks like. + """ + result = FreshCoverageEvaluator().evaluate(_ctx()) + self.assertIs(result.observed_status, CriterionStatus.UNKNOWN) + self.assertIn("never loaded", result.reason) + + def test_the_latest_dataset_is_resolved_as_of_the_day_being_evaluated(self): + """The point of loading a range: each day of a march sees its own latest dataset. + + Three datasets, published a week apart, each covering less of the future than the + last. Evaluated at three different `now`s from one loaded input set, the criterion + reads a different dataset each time. + """ + published = [ + (NOW - timedelta(days=14), NOW + timedelta(days=90)), + (NOW - timedelta(days=7), NOW + timedelta(days=30)), + (NOW - timedelta(days=1), NOW + timedelta(days=2)), + ] + inputs = FreshCoverageInputs( + { + "feed-1": [ + LatestDataset( + dataset_id=f"mdb-1-{index}", + downloaded_at=downloaded_at, + service_date_range_end=coverage_end, + ) + for index, (downloaded_at, coverage_end) in enumerate(published) + ] + } + ) + payload = {SealCriterionName.FRESH_COVERAGE: inputs} + + for offset, expected in ( + (-20, CriterionStatus.UNKNOWN), # before the feed had any dataset + (-10, CriterionStatus.PASS), # the first one, covering 90 days out + (-3, CriterionStatus.PASS), # the second, still beyond the horizon + (0, CriterionStatus.FAIL), # the third, covering only 2 more days + ): + moment = NOW + timedelta(days=offset) + with self.subTest(days_from_now=offset): + result = FreshCoverageEvaluator().evaluate( + _ctx(now=moment, inputs=payload) + ) + self.assertIs(result.observed_status, expected) + if __name__ == "__main__": unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py index e70838347..08204ce93 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py @@ -450,5 +450,86 @@ def test_is_verdict_only_covers_pass_and_fail(self): self.assertFalse(status.is_verdict, status) +class TestGraceExpiresOnAnUnknownDay(unittest.TestCase): + """A streak whose grace runs out on a day that produced no reading. + + Found by marching a feed locally: fourteen failures, UNKNOWN across the day grace + expired, then a pass — and nothing was confirmed, so the outage left no trace. + """ + + def _grace_expired_under_unknowns(self): + # Grace runs 14 days from the streak start on day 1, so day 15 both expires it and + # is the first day with no reading. + return _run( + [(0, PASS)] + + _failing(1, 15) + + [ + (15, CriterionStatus.UNKNOWN), + (16, CriterionStatus.UNKNOWN), + (17, CriterionStatus.UNKNOWN), + ] + ) + + def test_the_streak_start_survives_the_unknown_days(self): + """The unknowns neither reset nor forget the streak.""" + state = self._grace_expired_under_unknowns() + self.assertEqual(state.first_observed_failure_at, _day(1)) + self.assertEqual( + state.last_verdict_at, + _day(14), + "no verdict since the last observed failure", + ) + + def test_a_streak_past_its_grace_confirms_without_a_fresh_verdict(self): + state = self._grace_expired_under_unknowns() + self.assertIs( + state.confirmed_status, + FAIL, + "17 days into a streak whose grace expired on day 15", + ) + + def test_a_pass_cannot_forgive_a_streak_that_outlived_its_grace(self): + """Otherwise one pass after the unknowns erases the whole outage.""" + recovered = _run([(18, PASS)], state=self._grace_expired_under_unknowns()) + self.assertIsNotNone( + recovered.last_confirmed_failure_at, + "a fortnight of failure left no record at all", + ) + + +class TestProbationAcrossAnUnknownDay(unittest.TestCase): + """`probation_start` when a confirmed failure is followed by no reading. + + A confirmed streak re-stamps it to the following day. An UNKNOWN day leaves it alone, + which makes that day the first of probation and counts it toward the term. + """ + + def _confirmed_then_unknown(self): + streaking = _run([(0, PASS)] + _failing(1, 17)) + return streaking, _run([(17, CriterionStatus.UNKNOWN)], state=streaking) + + def test_the_streak_leaves_probation_stamped_for_the_following_day(self): + streaking, _ = self._confirmed_then_unknown() + self.assertIs(streaking.confirmed_status, FAIL) + self.assertEqual(streaking.probation_start, _day(17)) + + def test_an_unknown_day_does_not_push_probation_forward(self): + streaking, after = self._confirmed_then_unknown() + self.assertEqual( + after.probation_start, + streaking.probation_start, + "no verdict re-stamps it, so probation begins on the unknown day", + ) + self.assertIs(after.confirmed_status, FAIL, "the last verdict still stands") + self.assertIs(phase(after), CriterionPhase.ON_PROBATION) + + def test_the_unknown_day_counts_toward_serving_probation(self): + """A day with no reading still serves the term.""" + _, after = self._confirmed_then_unknown() + served = _run([(17 + PROBATION_PERIOD.days, PASS)], state=after) + self.assertIsNone(served.probation_start, "the term was served") + self.assertIs(phase(served), CriterionPhase.STEADY) + + if __name__ == "__main__": unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py index 70af32f04..479c57873 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py @@ -34,8 +34,10 @@ from tasks.seal_of_reliability.evaluators import ( EVALUATORS, CriterionEvaluator, + FreshCoverageEvaluator, OfficialEvaluator, ) +from tasks.seal_of_reliability.evaluators.fresh_coverage import FreshCoverageInputs from tasks.seal_of_reliability.seal_updater import update_seals from tasks.seal_of_reliability.state_machine import SealCriterionState from sqlalchemy import delete, select @@ -401,7 +403,7 @@ class TestBuildContexts(SealDbTestCase): @with_db_session(db_url=default_db_url) def test_loads_the_fields_the_evaluators_need(self, db_session): feeds = list(_feeds_by_stable_id(db_session, OFFICIAL).values()) - ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + ctx = build_contexts(db_session, feeds, NOW, EVALUATORS)[feeds[0].id] self.assertEqual(ctx.stable_id, OFFICIAL) self.assertTrue(ctx.official) self.assertEqual(ctx.now, NOW) @@ -411,44 +413,103 @@ def test_loads_the_fields_the_evaluators_need(self, db_session): @with_db_session(db_url=default_db_url) def test_stables_clock_is_the_feed_row_and_needs_no_query(self, db_session): feeds = list(_feeds_by_stable_id(db_session, OFFICIAL).values()) - ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + ctx = build_contexts(db_session, feeds, NOW, EVALUATORS)[feeds[0].id] self.assertEqual(ctx.feed_created_at, NOW - timedelta(days=400)) + @with_db_session(db_url=default_db_url) + def test_freshs_inputs_reach_the_context_through_its_own_loader(self, db_session): + """The seam: `build_contexts` asks each criterion, and stashes what it returns. + + Fresh's inputs are not a field on the context — its latest dataset is a different row + on each day a march evaluates — so they arrive under the criterion's own key and it + reads them back with `inputs_for`. + """ + _seed_dataset(db_session, TRACKED, coverage_end=NOW + timedelta(days=90)) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW, EVALUATORS)[feeds[0].id] + + inputs = ctx.inputs_for(SealCriterionName.FRESH_COVERAGE) + self.assertIsInstance(inputs, FreshCoverageInputs) + latest = inputs.as_of(feeds[0].id, NOW) + self.assertEqual(latest.dataset_id, f"{TRACKED}_dataset") + self.assertEqual(latest.service_date_range_end, NOW + timedelta(days=90)) + + @with_db_session(db_url=default_db_url) + def test_builds_one_context_per_feed(self, db_session): + feeds = list(_feeds_by_stable_id(db_session, OFFICIAL, NOT_OFFICIAL).values()) + contexts = build_contexts(db_session, feeds, NOW, EVALUATORS) + self.assertEqual(len(contexts), 2) + self.assertEqual({ctx.official for ctx in contexts.values()}, {True, False}) + + @with_db_session(db_url=default_db_url) + def test_a_criterion_inputs_reach_every_context_in_the_batch(self, db_session): + """The nightly run is the one-day case: the loader is asked for `now`'s day only. + + The payload is shared by reference across the batch's contexts, so a criterion + indexes it by feed itself rather than the builder slicing it per feed. + """ + seen_days = [] + + class Loading(CriterionEvaluator): + name = SealCriterionName.AVAILABLE + + def load_inputs(self, db_session, feeds, days): + seen_days.append(list(days)) + return {feed.id: feed.stable_id for feed in feeds} + + def _evaluate(self, ctx): + return CriterionStatus.PASS, "loaded" + + feeds = list(_feeds_by_stable_id(db_session, OFFICIAL, NOT_OFFICIAL).values()) + contexts = build_contexts(db_session, feeds, NOW, [Loading()]) + + self.assertEqual(seen_days, [[NOW.date()]]) + for feed in feeds: + inputs = contexts[feed.id].inputs_for(SealCriterionName.AVAILABLE) + self.assertEqual(inputs[feed.id], feed.stable_id) + + +class TestFreshCoverageLoadInputs(SealDbTestCase): + """`FreshCoverageEvaluator.load_inputs` against real dataset rows.""" + + @staticmethod + def _load(db_session, stable_id, days=None): + feeds = list(_feeds_by_stable_id(db_session, stable_id).values()) + return feeds[0], FreshCoverageEvaluator().load_inputs( + db_session, feeds, days or [NOW.date()] + ) + @with_db_session(db_url=default_db_url) def test_a_feed_with_no_dataset_says_so(self, db_session): - """The bulk load misses, and the context says so rather than guessing a value.""" - feeds = list(_feeds_by_stable_id(db_session, OFFICIAL).values()) - ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] - self.assertIsNone(ctx.latest_dataset) + """The load misses, and `as_of` says so rather than guessing a value.""" + feed, inputs = self._load(db_session, OFFICIAL) + self.assertIsNone(inputs.as_of(feed.id, NOW)) @with_db_session(db_url=default_db_url) def test_the_latest_dataset_coverage_is_loaded(self, db_session): _seed_dataset(db_session, TRACKED, coverage_end=NOW + timedelta(days=90)) - feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) - ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] - self.assertEqual(ctx.latest_dataset.dataset_id, f"{TRACKED}_dataset") - self.assertEqual( - ctx.latest_dataset.service_date_range_end, NOW + timedelta(days=90) - ) + feed, inputs = self._load(db_session, TRACKED) + latest = inputs.as_of(feed.id, NOW) + self.assertEqual(latest.dataset_id, f"{TRACKED}_dataset") + self.assertEqual(latest.service_date_range_end, NOW + timedelta(days=90)) @with_db_session(db_url=default_db_url) def test_a_dataset_with_no_coverage_end_is_not_a_missing_dataset(self, db_session): - """The two UNKNOWN cases must stay distinguishable at the context layer.""" + """The two UNKNOWN cases must stay distinguishable at the loading layer.""" _seed_dataset(db_session, TRACKED, coverage_end=None) - feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) - ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] - self.assertIsNotNone(ctx.latest_dataset, "the dataset is there ...") - self.assertIsNone( - ctx.latest_dataset.service_date_range_end, "... its coverage end is not" - ) + feed, inputs = self._load(db_session, TRACKED) + latest = inputs.as_of(feed.id, NOW) + self.assertIsNotNone(latest, "the dataset is there ...") + self.assertIsNone(latest.service_date_range_end, "... its coverage end is not") @with_db_session(db_url=default_db_url) - def test_the_latest_dataset_is_resolved_as_of_now(self, db_session): + def test_the_latest_dataset_is_resolved_as_of_the_day_asked_for(self, db_session): """A replay must not see a dataset published after the day it is evaluating. `gtfsfeed.latest_dataset_id` points at the newest dataset that exists today, so reading it would report the feed as fresh on a day when the data covering that day - had not been published yet. + had not been published yet. One load covering the whole range answers both days, + which is the property a march depends on. """ _seed_dataset( db_session, @@ -464,23 +525,38 @@ def test_the_latest_dataset_is_resolved_as_of_now(self, db_session): downloaded_at=NOW + timedelta(days=10), suffix="_new", ) - feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + later = NOW + timedelta(days=20) + feed, inputs = self._load(db_session, TRACKED, days=[NOW.date(), later.date()]) - as_of_now = build_contexts(db_session, feeds, NOW)[feeds[0].id] self.assertEqual( - as_of_now.latest_dataset.service_date_range_end, + inputs.as_of(feed.id, NOW).service_date_range_end, NOW + timedelta(days=5), "the newer dataset had not been downloaded yet", ) - - later = NOW + timedelta(days=20) - as_of_later = build_contexts(db_session, feeds, later)[feeds[0].id] self.assertEqual( - as_of_later.latest_dataset.service_date_range_end, + inputs.as_of(feed.id, later).service_date_range_end, NOW + timedelta(days=400), "by then it had", ) + @with_db_session(db_url=default_db_url) + def test_the_dataset_carried_into_the_range_is_loaded_too(self, db_session): + """A march's first day sees the dataset the feed already had when it opened. + + Without the carry-in query the range would hold no dataset for that feed at all, and + Fresh would read a healthy feed as UNKNOWN for every day until it next published. + """ + _seed_dataset( + db_session, + TRACKED, + coverage_end=NOW + timedelta(days=400), + downloaded_at=NOW - timedelta(days=200), + ) + feed, inputs = self._load(db_session, TRACKED) + self.assertIsNotNone( + inputs.as_of(feed.id, NOW), "downloaded long before the range opened" + ) + @with_db_session(db_url=default_db_url) def test_a_dataset_with_no_downloaded_at_cannot_be_placed_in_time(self, db_session): """It is excluded rather than guessed at: we cannot say whether it existed yet.""" @@ -491,16 +567,8 @@ def test_a_dataset_with_no_downloaded_at_cannot_be_placed_in_time(self, db_sessi .values(downloaded_at=None) ) db_session.commit() - feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) - ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] - self.assertIsNone(ctx.latest_dataset) - - @with_db_session(db_url=default_db_url) - def test_builds_one_context_per_feed(self, db_session): - feeds = list(_feeds_by_stable_id(db_session, OFFICIAL, NOT_OFFICIAL).values()) - contexts = build_contexts(db_session, feeds, NOW) - self.assertEqual(len(contexts), 2) - self.assertEqual({ctx.official for ctx in contexts.values()}, {True, False}) + feed, inputs = self._load(db_session, TRACKED) + self.assertIsNone(inputs.as_of(feed.id, NOW)) @patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", ONLY_OFFICIAL) diff --git a/scripts/api-tests.sh b/scripts/api-tests.sh index 621e3efc4..06ee85ca1 100755 --- a/scripts/api-tests.sh +++ b/scripts/api-tests.sh @@ -100,7 +100,7 @@ execute_tests() { # Run tests with coverage. Add the path to the main file and the shared packages that were linked. PT="src:tests:$PYTHONPATH" - PYTHONPATH="$PT" venv/bin/coverage run --branch -m pytest -s -W 'ignore::DeprecationWarning' tests + PYTHONPATH="$PT" venv/bin/coverage run --branch -m pytest -W 'ignore::DeprecationWarning' tests # Fail if tests fail if [ $? -ne 0 ]; then printf "\n${RED}Tests failed in $1${NC}\n"