diff --git a/api/.openapi-generator/FILES b/api/.openapi-generator/FILES index 74129a6b5..afcea0a9d 100644 --- a/api/.openapi-generator/FILES +++ b/api/.openapi-generator/FILES @@ -52,25 +52,3 @@ src/feeds_gen/models/search_feeds200_response.py src/feeds_gen/models/source_info.py src/feeds_gen/models/validation_report.py src/feeds_gen/security_api.py -src/user_service/impl/__init__.py -src/user_service_gen/apis/__init__.py -src/user_service_gen/apis/notifications_api.py -src/user_service_gen/apis/notifications_api_base.py -src/user_service_gen/apis/subscriptions_api.py -src/user_service_gen/apis/subscriptions_api_base.py -src/user_service_gen/apis/users_api.py -src/user_service_gen/apis/users_api_base.py -src/user_service_gen/main.py -src/user_service_gen/models/__init__.py -src/user_service_gen/models/create_notification_subscription_request.py -src/user_service_gen/models/extra_models.py -src/user_service_gen/models/feature_flag.py -src/user_service_gen/models/feed_subscription_summary.py -src/user_service_gen/models/notification_subscription.py -src/user_service_gen/models/notification_type.py -src/user_service_gen/models/subscription_feed.py -src/user_service_gen/models/subscription_feed_group.py -src/user_service_gen/models/update_notification_subscription_request.py -src/user_service_gen/models/update_user_request.py -src/user_service_gen/models/user_profile.py -src/user_service_gen/security_api.py diff --git a/api/src/shared/common/seal_criteria.py b/api/src/shared/common/seal_criteria.py index 9c8a14844..220b934eb 100644 --- a/api/src/shared/common/seal_criteria.py +++ b/api/src/shared/common/seal_criteria.py @@ -14,7 +14,7 @@ from datetime import datetime, timedelta from enum import Enum -from typing import Dict, Final, Optional +from typing import Dict, Final, Iterable, Optional, Tuple from shared.common.error_handling import raise_internal_http_error, unknown_seal_criterion @@ -61,6 +61,20 @@ def is_verdict(self) -> bool: return self in (CriterionStatus.PASS, CriterionStatus.FAIL) +class SealStatus(str, Enum): + """The feed-level seal outcome.""" + + GRANTED = "granted" + NOT_GRANTED = "not_granted" + UNKNOWN = "unknown" + NEVER_EVALUATED = "never_evaluated" + + @property + def is_answer(self) -> bool: + """True for GRANTED and NOT_GRANTED - the two values that actually decide the seal.""" + return self in (SealStatus.GRANTED, SealStatus.NOT_GRANTED) + + class CriterionPhase(str, Enum): """Which of the two debouncing mechanisms is currently acting on a criterion. @@ -111,10 +125,49 @@ class CriterionPhase(str, Enum): ) +def roll_up_seal_status( + criteria: Iterable[Tuple[CriterionStatus, bool]], +) -> SealStatus: + """The feed-level seal outcome from its criteria. + * every criterion NEVER_EVALUATED -> seal NEVER_EVALUATED. + * any criterion failing or on probation -> seal NOT_GRANTED, whatever the rest say. + * any remaining criterion NEVER_EVALUATED -> seal UNKNOWN. + * otherwise every criterion is a confirmed pass and none is on probation -> seal GRANTED. + """ + in_scope = [ + (status, on_probation) for status, on_probation in criteria if status is not CriterionStatus.NOT_APPLICABLE + ] + if not in_scope: + # Every criterion is NOT_APPLICABLE, so there is nothing left to judge the feed by. + return SealStatus.NEVER_EVALUATED + + unjudged = sum(1 for status, _ in in_scope if status is CriterionStatus.NEVER_EVALUATED) + if unjudged == len(in_scope): + # Every criterion that isn't NOT_APPLICABLE is NEVER_EVALUATED + return SealStatus.NEVER_EVALUATED + + # One criterion is enough to deny the seal, so this is decidable even with the rest unjudged. + # A pass on probation denies it too: probation withholds the criterion whatever its status. + denied = any( + status is CriterionStatus.FAIL or (status is CriterionStatus.PASS and on_probation) + for status, on_probation in in_scope + ) + if denied: + return SealStatus.NOT_GRANTED + if unjudged: + # Nothing denies the seal, but not everything has been judged: it cannot be granted yet. + return SealStatus.UNKNOWN + + return SealStatus.GRANTED + + # Stable: how long we must have been tracking a feed - measured from its # `feed.created_at` - before it can be called stable. TRACKING_PERIOD: Final[timedelta] = timedelta(days=180) +# Available: how far back to look for an availability check +AVAILABILITY_LOOKBACK: Final[timedelta] = timedelta(hours=24) + # Fresh / future coverage: how far ahead the latest dataset's service coverage must reach FUTURE_COVERAGE_HORIZON: Final[timedelta] = timedelta(days=7) diff --git a/api/src/shared/db_models/feed_reliability_report_impl.py b/api/src/shared/db_models/feed_reliability_report_impl.py index 9b568e307..3a6c77e7b 100644 --- a/api/src/shared/db_models/feed_reliability_report_impl.py +++ b/api/src/shared/db_models/feed_reliability_report_impl.py @@ -1,9 +1,27 @@ from feeds_gen.models.feed_reliability_report import FeedReliabilityReport -from shared.common.seal_criteria import SealCriterionName, resolve_criterion +from shared.common.seal_criteria import ( + PROBATION_EXEMPT_CRITERIA, + CriterionStatus, + SealCriterionName, + resolve_criterion, + roll_up_seal_status, +) from shared.database_gen.sqlacodegen_models import Gtfsfeed as GtfsfeedOrm +from shared.database_gen.sqlacodegen_models import SealCriterion as SealCriterionOrm from shared.db_models.reliability_criterion_impl import ReliabilityCriterionImpl +def _seal_status_of(criterion_rows: list[SealCriterionOrm]) -> str: + """The feed-level seal status derived from a feed's `seal_criterion` rows.""" + return roll_up_seal_status( + ( + CriterionStatus(row.confirmed_status), + row.probation_start is not None and row.criterion not in PROBATION_EXEMPT_CRITERIA, + ) + for row in criterion_rows + ).value + + class FeedReliabilityReportImpl(FeedReliabilityReport): """Implementation of the `FeedReliabilityReport` model. @@ -58,6 +76,7 @@ def from_orm(cls, feed: GtfsfeedOrm | None) -> FeedReliabilityReport | None: return cls( feed_id=feed.stable_id, has_seal=bool(seal.has_seal) if seal is not None else False, + seal_status=_seal_status_of(criterion_rows), earned_at=seal.seal_earned_at if seal is not None else None, lost_at=seal.seal_lost_at if seal is not None else None, evaluated_at=max(evaluated_ats) if evaluated_ats else None, diff --git a/api/tests/unittest/test_feeds.py b/api/tests/unittest/test_feeds.py index 1ae240bbc..8ff15e50c 100644 --- a/api/tests/unittest/test_feeds.py +++ b/api/tests/unittest/test_feeds.py @@ -466,6 +466,7 @@ def test_gtfs_feed_reliability_never_evaluated(client: TestClient): body = response.json() assert body["feed_id"] == TEST_GTFS_FEED_STABLE_IDS[0] assert body["has_seal"] is False + assert body["seal_status"] == "never_evaluated", "no criterion row at all, so nothing was ever decided" assert body["on_probation"] is False assert len(body["criteria"]) == 6 assert {criterion["status"] for criterion in body["criteria"]} == {"never_evaluated"} @@ -566,6 +567,41 @@ def test_gtfs_feed_get_embeds_reliability_seal(client: TestClient): assert seal["evaluated_at"] is not None +def test_gtfs_feed_reliability_reports_an_undecided_seal_as_unknown(client: TestClient): + """`has_seal: false` covers three different things; `seal_status` is what separates them. + + A feed whose criteria have not all been judged is not a feed that was judged and failed, and a + client has to be able to tell those apart. The status is derived from the criterion rows rather + than stored, so one passing criterion beside one never-evaluated one is all it takes. + """ + feed_stable_id = TEST_GTFS_FEED_STABLE_IDS[2] + criteria = { + "official": {"observed_status": "pass", "confirmed_status": "pass", "evaluated_at": SEAL_NOW}, + "available": {"observed_status": "unknown", "confirmed_status": "never_evaluated", "evaluated_at": SEAL_NOW}, + } + with _seal_rows(feed_stable_id, has_seal=False, criteria=criteria): + response = client.request("GET", f"/v1/gtfs_feeds/{feed_stable_id}/reliability", headers=authHeaders) + + assert response.status_code == 200, f"Response status code was {response.status_code} instead of 200" + body = response.json() + assert body["has_seal"] is False + assert body["seal_status"] == "unknown" + + +def test_gtfs_feed_reliability_reports_a_judged_seal_as_granted(client: TestClient): + """The other side of the same coin: every criterion in scope judged, and all passing.""" + feed_stable_id = TEST_GTFS_FEED_STABLE_IDS[3] + criteria = { + criterion: {"observed_status": "pass", "confirmed_status": "pass", "evaluated_at": SEAL_NOW} + for criterion in ("official", "stable", "available", "compliant", "fresh_coverage", "fresh_continuous") + } + with _seal_rows(feed_stable_id, has_seal=True, criteria=criteria): + response = client.request("GET", f"/v1/gtfs_feeds/{feed_stable_id}/reliability", headers=authHeaders) + + assert response.status_code == 200, f"Response status code was {response.status_code} instead of 200" + assert response.json()["seal_status"] == "granted" + + def test_gtfs_feed_get_without_seal_reports_null(client: TestClient): """A feed that has never been evaluated reports a null summary rather than an empty object.""" response = client.request( diff --git a/docs/DatabaseCatalogAPI.yaml b/docs/DatabaseCatalogAPI.yaml index 98868bb6d..bd261877b 100644 --- a/docs/DatabaseCatalogAPI.yaml +++ b/docs/DatabaseCatalogAPI.yaml @@ -1119,6 +1119,19 @@ components: description: Whether the feed currently holds the Seal of Reliability. type: boolean example: false + seal_status: + description: > + Descriptive status of the feed's seal. `has_seal` is true only when this is `granted`. + `not_granted`: at least one criterion is failing. `unknown`: none is failing, but not + every criterion has been evaluated yet. `never_evaluated`: none of the criteria has + been evaluated yet. + type: string + enum: + - granted + - not_granted + - unknown + - never_evaluated + example: granted earned_at: description: When the feed most recently earned the seal, in ISO 8601 date-time format. type: string diff --git a/docs/OperationsAPI.yaml b/docs/OperationsAPI.yaml index 5febae2de..cf6d6bc1d 100644 --- a/docs/OperationsAPI.yaml +++ b/docs/OperationsAPI.yaml @@ -1211,6 +1211,17 @@ components: description: Whether the feed currently holds the Seal of Reliability. type: boolean example: false + seal_status: + description: > + Descriptive status of the feed's seal. `has_seal` is true only when this is `granted`. `not_granted`: at least one criterion is failing. `unknown`: none is failing, but not every criterion has been evaluated yet. `never_evaluated`: none of the criteria has been evaluated yet. + + type: string + enum: + - granted + - not_granted + - unknown + - never_evaluated + example: granted earned_at: description: When the feed most recently earned the seal, in ISO 8601 date-time format. type: string @@ -2270,6 +2281,11 @@ components: The type of realtime entry: + + + + + * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -2396,6 +2412,11 @@ components: The type of realtime entry: + + + + + * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -2497,6 +2518,11 @@ components: Describes status of the Feed. Should be one of + + + + + * `active` Feed should be used in public trip planners. * `deprecated` Feed is explicitly deprecated and should not be used in public trip planners. * `inactive` Feed hasn't been recently updated and should be used at risk of providing outdated information. @@ -2516,6 +2542,11 @@ components: Describes data type of a feed. Should be one of + + + + + * `gtfs` GTFS feed. * `gtfs_rt` GTFS-RT feed. * `gbfs` GBFS feed. diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py index 71380816d..0e4a074c9 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py @@ -34,7 +34,33 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from shared.database_gen.sqlacodegen_models import Feed, Gtfsdataset, Gtfsfeed +from shared.common.seal_criteria import AVAILABILITY_LOOKBACK +from shared.database_gen.sqlacodegen_models import ( + Feed, + GtfsFeedAvailabilityCheck, + Gtfsdataset, + Gtfsfeed, + Validationreport, + t_validationreportgtfsdataset, +) + + +@dataclass(frozen=True) +class AvailabilityCheck: + """The latest availability check for a feed within the run's window.""" + + checked_at: datetime + success: bool + + +@dataclass(frozen=True) +class ValidationReport: + """The latest validation report of the feed's latest dataset, as of the run's `now`.""" + + report_id: str + dataset_id: str + validated_at: datetime + total_error: Optional[int] = None @dataclass(frozen=True) @@ -72,6 +98,12 @@ class FeedSealContext: # The feed's latest dataset as of `now` - resolved by `downloaded_at` vs `now` latest_dataset: Optional[LatestDataset] = None + # Available: the latest availability check in the window this run covers. + availability_check: Optional[AvailabilityCheck] = None + + # Compliant: the latest validation report of the dataset in `latest_dataset`. + latest_validation_report: Optional[ValidationReport] = None + # Feeds in these statuses, or not published, are never eligible for the seal. # `inactive` and `future` feeds are deliberately kept eligible. @@ -195,6 +227,94 @@ def _load_latest_datasets( } +def _load_validation_reports( + db_session: Session, + latest_datasets: Dict[str, LatestDataset], + now: datetime, +) -> Dict[str, ValidationReport]: + """feed_id -> the latest validation report of that feed's latest dataset, as of `now`. + + Scoped to the latest dataset, not the feed: a verdict on a superseded dataset does not describe + what is being served. A feed whose latest dataset is not validated yet is left out, which the + evaluator reads as UNKNOWN. One dataset can have several reports (a re-validation); the most + recently validated wins, and ones with no `validated_at` are excluded. + """ + if not latest_datasets: + return {} + feed_id_by_dataset = { + dataset.dataset_id: feed_id for feed_id, dataset in latest_datasets.items() + } + join_table = t_validationreportgtfsdataset + rows = db_session.execute( + select( + join_table.c.dataset_id, + Validationreport.id, + Validationreport.validated_at, + Validationreport.total_error, + ) + .select_from(join_table) + .join( + Validationreport, + Validationreport.id == join_table.c.validation_report_id, + ) + .where( + join_table.c.dataset_id.in_(list(feed_id_by_dataset)), + Validationreport.validated_at.is_not(None), + Validationreport.validated_at <= now, + ) + .distinct(join_table.c.dataset_id) + .order_by( + join_table.c.dataset_id, + Validationreport.validated_at.desc(), + Validationreport.id.desc(), + ) + ).all() + return { + feed_id_by_dataset[row.dataset_id]: ValidationReport( + report_id=row.id, + dataset_id=row.dataset_id, + validated_at=row.validated_at, + total_error=row.total_error, + ) + for row in rows + } + + +def _load_availability( + db_session: Session, feed_ids: Sequence[str], now: datetime +) -> Dict[str, AvailabilityCheck]: + """feed_id -> its latest availability check in the 24 hours up to `now`. + + A rolling window rather than the UTC day of `now`, so a check still counts when the + availability job (02:00 UTC) and the seal run (04:00 UTC) drift apart or one of them runs + late. + """ + if not feed_ids: + return {} + rows = db_session.execute( + select( + GtfsFeedAvailabilityCheck.feed_id, + GtfsFeedAvailabilityCheck.checked_at, + GtfsFeedAvailabilityCheck.success, + ) + .where( + GtfsFeedAvailabilityCheck.feed_id.in_(list(feed_ids)), + GtfsFeedAvailabilityCheck.checked_at > now - AVAILABILITY_LOOKBACK, + GtfsFeedAvailabilityCheck.checked_at <= now, + ) + .distinct(GtfsFeedAvailabilityCheck.feed_id) + .order_by( + GtfsFeedAvailabilityCheck.feed_id, + GtfsFeedAvailabilityCheck.checked_at.desc(), + GtfsFeedAvailabilityCheck.id.desc(), + ) + ).all() + return { + row.feed_id: AvailabilityCheck(checked_at=row.checked_at, success=row.success) + for row in rows + } + + def build_contexts( db_session: Session, feeds: Sequence[Gtfsfeed], now: datetime ) -> Dict[str, FeedSealContext]: @@ -229,9 +349,10 @@ def _load_availability_today(db_session, feed_ids, day_start) -> Dict[str, bool] called once as `availability = _load_availability_today(...)` and consumed per feed as `availability_success_today=availability.get(feed.id, False)`. """ - latest_datasets = _load_latest_datasets( - db_session, [feed.id for feed in feeds], now - ) + feed_ids = [feed.id for feed in feeds] + latest_datasets = _load_latest_datasets(db_session, feed_ids, now) + availability = _load_availability(db_session, feed_ids, now) + validation_reports = _load_validation_reports(db_session, latest_datasets, now) return { feed.id: FeedSealContext( @@ -243,6 +364,8 @@ def _load_availability_today(db_session, feed_ids, day_start) -> Dict[str, bool] seasonal=feed.seasonal, feed_created_at=feed.created_at, latest_dataset=latest_datasets.get(feed.id), + availability_check=availability.get(feed.id), + latest_validation_report=validation_reports.get(feed.id), ) for feed in feeds } diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py index 349f33311..535c3e3cc 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py @@ -15,9 +15,9 @@ # """The seal criterion evaluators. -`EVALUATORS` is the registry the job iterates. Official (issue #1783), Stable and Fresh / -future coverage (issue #1784) are implemented; Available and Compliant are the rest of -#1784, and Fresh / continuous coverage is tracked by #1782. Adding one means a new subclass, +`EVALUATORS` is the registry the job iterates. Official (issue #1783), Stable, Available, +Compliant and Fresh / future coverage (issue #1784) are implemented; Fresh / continuous +coverage is tracked by #1782. Adding one means a new subclass, an entry here, and whatever fields it needs on `FeedSealContext`. Its windows are not declared on the subclass: they come from the policy maps in `shared.common.seal_criteria`, which the read API reads too. @@ -32,6 +32,8 @@ CriterionEvaluator, CriterionObservation, ) +from tasks.seal_of_reliability.evaluators.available import AvailableEvaluator +from tasks.seal_of_reliability.evaluators.compliant import CompliantEvaluator from tasks.seal_of_reliability.evaluators.fresh_coverage import FreshCoverageEvaluator from tasks.seal_of_reliability.evaluators.official import OfficialEvaluator from tasks.seal_of_reliability.evaluators.stable import StableEvaluator @@ -39,11 +41,15 @@ EVALUATORS: Final[List[CriterionEvaluator]] = [ OfficialEvaluator(), StableEvaluator(), + AvailableEvaluator(), + CompliantEvaluator(), FreshCoverageEvaluator(), ] __all__ = [ "EVALUATORS", + "AvailableEvaluator", + "CompliantEvaluator", "CriterionEvaluator", "CriterionObservation", "FreshCoverageEvaluator", diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/available.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/available.py new file mode 100644 index 000000000..3edb8e121 --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/available.py @@ -0,0 +1,47 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Available criterion: the feed's producer URL answered today.""" + +from typing import Tuple + +from shared.common.seal_criteria import ( + AVAILABILITY_LOOKBACK, + CriterionStatus, + SealCriterionName, +) +from tasks.seal_of_reliability.context import FeedSealContext +from tasks.seal_of_reliability.evaluators.base import CriterionEvaluator + + +class AvailableEvaluator(CriterionEvaluator): + """The latest `gtfs_feed_availability_check` since the previous run has `success = TRUE`. + + The window runs from the last time this criterion was evaluated for the feed up to + `now`. A window with no check at all is UNKNOWN, not a failure: it means we + did not look, not that the feed was down. + """ + + name = SealCriterionName.AVAILABLE + + def _evaluate(self, ctx: FeedSealContext) -> Tuple[CriterionStatus, str]: + check = ctx.availability_check + if check is None: + since = (ctx.now - AVAILABILITY_LOOKBACK).isoformat() + return CriterionStatus.UNKNOWN, f"no availability check since {since}" + + status = CriterionStatus.PASS if check.success else CriterionStatus.FAIL + outcome = "succeeded" if check.success else "failed" + return status, f"the check at {check.checked_at.isoformat()} {outcome}" diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/compliant.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/compliant.py new file mode 100644 index 000000000..74307599a --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/compliant.py @@ -0,0 +1,57 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Compliant criterion: the latest dataset validates with no errors.""" + +from typing import Tuple + +from shared.common.seal_criteria import CriterionStatus, SealCriterionName +from tasks.seal_of_reliability.context import FeedSealContext +from tasks.seal_of_reliability.evaluators.base import CriterionEvaluator + + +class CompliantEvaluator(CriterionEvaluator): + """`total_error = 0` on the latest validation report of the feed's latest dataset. + + A dataset with no report yet - unvalidated, or validation lagging publication - is UNKNOWN, + which freezes the criterion at its last confirmed verdict rather than failing it. So is a feed + with no dataset: a missing report is not a clean bill of health, nor evidence of one. + """ + + name = SealCriterionName.COMPLIANT + + def _evaluate(self, ctx: FeedSealContext) -> Tuple[CriterionStatus, str]: + if ctx.latest_dataset is None: + return CriterionStatus.UNKNOWN, "the feed has no dataset" + + report = ctx.latest_validation_report + if report is None: + return ( + CriterionStatus.UNKNOWN, + f"dataset {ctx.latest_dataset.dataset_id} has no validation report", + ) + + if report.total_error is None: + return ( + CriterionStatus.UNKNOWN, + f"validation report {report.report_id} has no total_error", + ) + + validated = ( + f"dataset {report.dataset_id}, validated {report.validated_at.isoformat()}" + ) + if report.total_error == 0: + return CriterionStatus.PASS, f"no errors ({validated})" + return CriterionStatus.FAIL, f"{report.total_error} error(s) ({validated})" diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py index 37807c1d2..e942239db 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py @@ -41,6 +41,8 @@ CriterionPhase, CriterionStatus, SealCriterionName, + SealStatus, + roll_up_seal_status, ) from shared.database.database import with_db_session from shared.database_gen.sqlacodegen_models import ( @@ -169,36 +171,15 @@ def _load_previous_seals( return {row.feed_id: bool(row.has_seal) for row in rows} -def _roll_up_has_seal(states: Dict[str, SealCriterionState]) -> bool: - """True when every criterion in service is a confirmed pass and not on probation. +def _roll_up_seal_status(states: Dict[str, SealCriterionState]) -> SealStatus: + """The feed-level seal outcome from its criteria. - A criterion is *in service* when its `confirmed_status` is a verdict. `confirmed_status` - is only ever PASS, FAIL, NEVER_EVALUATED or NOT_APPLICABLE — never UNKNOWN, since an - unevaluable run leaves the stored value alone rather than writing UNKNOWN into it (see - `transition`). So the roll-up only has to skip the two non-verdict values it can see: - - * NEVER_EVALUATED — never produced a verdict, so it is skipped rather than counted as a - failure. This is what lets the seal be computed before every criterion has a data - source: one whose source starts collecting later simply is not part of the roll-up. - * NOT_APPLICABLE — deliberately excluded for this feed, so it is skipped too. This is why - a seasonal feed is not denied the seal by a criterion that is meaningless for it. - - An unevaluable run (UNKNOWN) does not appear here at all: `transition` has already frozen the - criterion at its last verdict, so it stays in the roll-up with that verdict if it had - one, or stays NEVER_EVALUATED and skipped if it never did. - - A criterion IN_GRACE_PERIOD is a confirmed pass and holds the seal. - One ON_PROBATION denies it. + A thin adapter over `roll_up_seal_status`, which owns the rule and is shared with the read API. + All this adds is reading `on_probation` off the state the way the job derives it, via `phase`. """ - in_service = [ - state for state in states.values() if state.confirmed_status.is_verdict - ] - if not in_service: - return False - return all( - state.confirmed_status is CriterionStatus.PASS - and phase(state) is not CriterionPhase.ON_PROBATION - for state in in_service + return roll_up_seal_status( + (state.confirmed_status, phase(state) is CriterionPhase.ON_PROBATION) + for state in states.values() ) @@ -452,6 +433,18 @@ def update_seals( ): first_evaluations += 1 + logging.info( + "Seal criterion evaluated: feed=%s criterion=%s observed=%s " + "confirmed=%s (was %s) phase=%s reason=%s", + ctx.stable_id, + evaluator.name.value, + observation.observed_status.value, + state.confirmed_status.value, + previous.confirmed_status.value if previous is not None else None, + phase(state).value, + observation.reason, + ) + criteria_report.append( { "criterion": evaluator.name.value, @@ -486,12 +479,20 @@ def update_seals( # A feed with no seal row yet is treated as not holding one, so a first run can # grant the seal but can never withdraw one: nothing was held to lose. had_seal = previous_seals.get(feed.id) - has_seal = _roll_up_has_seal(merged) + # Not stored: `seal_status` is derived from the same seal_criterion rows the read + # API derives it from (see `roll_up_seal_status`), so a column would be a second copy + # to keep in step. The job computes it to report and log the outcome, and to answer + # the one question feed_reliability_seal does store. + seal_status = _roll_up_seal_status(merged) + # The boolean stays the narrow question it always was: only GRANTED holds the seal, + # so unknown and never-evaluated read as `false` to everything already consuming it. + has_seal = seal_status is SealStatus.GRANTED outcome = { "feed_id": feed.id, "stable_id": ctx.stable_id, "had_seal": bool(had_seal), "has_seal": has_seal, + "seal_status": seal_status, # A first evaluation is a grant if it passes, but it is not a loss if # it fails: nothing was held, so nothing was lost. Only these two # flags stamp seal_earned_at / seal_lost_at. @@ -501,11 +502,20 @@ def update_seals( outcomes.append(outcome) # Every requested feed is reported, capped at max_reported_feeds below. + logging.info( + "Seal rolled up: feed=%s status=%s has_seal=%s (had_seal=%s)", + ctx.stable_id, + seal_status.value, + has_seal, + outcome["had_seal"], + ) + feed_reports.append( { "stable_id": ctx.stable_id, "had_seal": outcome["had_seal"], "has_seal": has_seal, + "seal_status": seal_status.value, "criteria": criteria_report, } ) @@ -548,6 +558,14 @@ def update_seals( "seals_after_run": sum(1 for outcome in outcomes if outcome["has_seal"]), "seals_granted": len(granted), "seals_revoked": len(revoked), + # The four-way outcome behind `seals_after_run`: which feeds were judged and did not + # qualify, and which could not be judged because a criterion has never had a verdict. + "seal_status_counts": { + status.value: sum( + 1 for outcome in outcomes if outcome["seal_status"] is status + ) + for status in SealStatus + }, # The two transitions in feed_reliability_seal, by feed. Counts alone cannot say # which feed moved, and that is the first thing anyone asks of a run. "granted_stable_ids": [outcome["stable_id"] for outcome in granted], diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py index 1b49964c3..eab7e224d 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py @@ -52,8 +52,11 @@ Feed, FeedReliabilitySeal, Gtfsdataset, + GtfsFeedAvailabilityCheck, Gtfsfeed, SealCriterion, + Validationreport, + t_validationreportgtfsdataset, ) from test_shared.test_utils.database_utils import default_db_url @@ -69,8 +72,11 @@ # The task always runs against an explicit feed list. These are the eligible seeded feeds. REQUESTED = [OFFICIAL, NOT_OFFICIAL, UNKNOWN_OFFICIAL] -# Every seeded feed gets a dataset covering the next 400 days, so Fresh passes and `official` -# stays the only criterion that separates the feeds. Stable needs nothing seeded: it reads +# Every seeded feed gets a dataset covering the next 400 days, a successful availability check +# and a clean validation report of that dataset, so Fresh, Available and Compliant all pass and +# `official` stays the only criterion that separates the feeds. A criterion with no verdict at +# all makes the whole seal `unknown`, so a feed missing one of these inputs would never be +# granted or revoked. Stable needs nothing seeded: it reads # `feed.created_at`, which `_seed` already backdates by 400 days. COVERAGE_END = NOW + timedelta(days=400) @@ -112,9 +118,53 @@ def _seed(db_session, feed_id, official=True, status="active", operational="publ ) db_session.flush() + # Available's input. + db_session.add( + GtfsFeedAvailabilityCheck( + feed_id=feed_id, + checked_at=NOW - timedelta(hours=2), + request_url=f"https://example.com/{feed_id}.zip", + request_type="http_head", + status_code=200, + success=True, + ) + ) + db_session.flush() + + # Compliant's input: a clean report of the dataset seeded just above. + report_id = f"{dataset_id}_report" + db_session.add( + Validationreport( + id=report_id, + validator_version="1.0.0", + validated_at=NOW - timedelta(hours=1), + total_error=0, + ) + ) + db_session.flush() + db_session.execute( + t_validationreportgtfsdataset.insert().values( + dataset_id=dataset_id, validation_report_id=report_id + ) + ) + db_session.flush() + def _cleanup(db_session): - """Deleting the parent Feed cascades to gtfsfeed and both seal tables.""" + """Deleting the parent Feed cascades to gtfsfeed, its datasets and both seal tables. + + Validation reports are not owned by the feed - they hang off the dataset through + validationreportgtfsdataset - so they and their link rows are removed by hand, dependency + first, before the ids are reused by the next test. + """ + db_session.execute( + t_validationreportgtfsdataset.delete().where( + t_validationreportgtfsdataset.c.validation_report_id.like(f"{PREFIX}%") + ) + ) + db_session.execute( + delete(Validationreport).where(Validationreport.id.like(f"{PREFIX}%")) + ) db_session.execute(delete(Feed).where(Feed.stable_id.like(f"{PREFIX}%"))) db_session.commit() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py index a37940102..2e8bbc5fc 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py @@ -25,9 +25,16 @@ CriterionStatus, SealCriterionName, ) -from tasks.seal_of_reliability.context import FeedSealContext, LatestDataset +from tasks.seal_of_reliability.context import ( + AvailabilityCheck, + FeedSealContext, + LatestDataset, + ValidationReport, +) from tasks.seal_of_reliability.evaluators import ( EVALUATORS, + AvailableEvaluator, + CompliantEvaluator, CriterionEvaluator, FreshCoverageEvaluator, OfficialEvaluator, @@ -332,5 +339,141 @@ def test_has_a_grace_period_and_serves_probation(self): self.assertEqual(FreshCoverageEvaluator().probation_period, PROBATION_PERIOD) +class TestAvailable(unittest.TestCase): + """The latest availability check in the window since the previous evaluation.""" + + @staticmethod + def _check(success, checked_at=None): + return AvailabilityCheck(checked_at=checked_at or NOW, success=success) + + def test_a_successful_check_passes(self): + self.assertIs( + AvailableEvaluator() + .evaluate(_ctx(availability_check=self._check(True))) + .observed_status, + CriterionStatus.PASS, + ) + + def test_a_failed_check_fails(self): + result = AvailableEvaluator().evaluate( + _ctx(availability_check=self._check(False)) + ) + self.assertIs(result.observed_status, CriterionStatus.FAIL) + self.assertIn("failed", result.reason) + + def test_no_check_in_the_window_is_unknown_not_a_failure(self): + """A window the availability job did not cover says nothing about the producer.""" + result = AvailableEvaluator().evaluate(_ctx(availability_check=None)) + self.assertIs(result.observed_status, CriterionStatus.UNKNOWN) + self.assertIn("no availability check since", result.reason) + + def test_the_reason_names_the_check_it_read(self): + """The window makes "which check decided this" a real question, so answer it.""" + checked_at = NOW - timedelta(hours=3) + result = AvailableEvaluator().evaluate( + _ctx(availability_check=self._check(False, checked_at)) + ) + self.assertIn(checked_at.isoformat(), result.reason) + + def test_is_never_not_applicable(self): + """Availability applies to every feed, seasonal ones included.""" + for check in (self._check(True), self._check(False), None): + with self.subTest(check=check): + self.assertIsNot( + AvailableEvaluator() + .evaluate(_ctx(availability_check=check, seasonal=True)) + .observed_status, + CriterionStatus.NOT_APPLICABLE, + ) + + def test_has_a_grace_period_and_serves_probation(self): + self.assertEqual(AvailableEvaluator().grace_period, timedelta(days=14)) + self.assertEqual(AvailableEvaluator().probation_period, PROBATION_PERIOD) + + +class TestCompliant(unittest.TestCase): + """`total_error = 0` on the latest validation report of the feed's latest dataset.""" + + DATASET_ID = "mdb-1-202605280000" + + def _compliant_ctx( + self, total_error=0, with_report=True, with_dataset=True, **overrides + ): + report = ( + ValidationReport( + report_id="report-1", + dataset_id=self.DATASET_ID, + validated_at=NOW - timedelta(hours=1), + total_error=total_error, + ) + if with_report + else None + ) + dataset = ( + LatestDataset( + dataset_id=self.DATASET_ID, + downloaded_at=NOW - timedelta(hours=2), + ) + if with_dataset + else None + ) + defaults = {"latest_validation_report": report, "latest_dataset": dataset} + defaults.update(overrides) + return _ctx(**defaults) + + def test_a_clean_report_passes(self): + self.assertIs( + CompliantEvaluator().evaluate(self._compliant_ctx(0)).observed_status, + CriterionStatus.PASS, + ) + + def test_any_error_fails(self): + result = CompliantEvaluator().evaluate(self._compliant_ctx(1)) + self.assertIs(result.observed_status, CriterionStatus.FAIL) + self.assertIn("1 error(s)", result.reason) + + def test_a_feed_with_no_dataset_is_unknown(self): + """Nothing published means nothing to validate, not a failure.""" + result = CompliantEvaluator().evaluate( + self._compliant_ctx(with_report=False, with_dataset=False) + ) + self.assertIs(result.observed_status, CriterionStatus.UNKNOWN) + self.assertIn("no dataset", result.reason) + + def test_an_unvalidated_latest_dataset_is_unknown(self): + """The case that keeps a never-validated feed off a confirmed failure. + + Validation lags publication, so a feed publishing faster than the validator sits at + UNKNOWN and keeps whatever verdict it last earned. + """ + result = CompliantEvaluator().evaluate(self._compliant_ctx(with_report=False)) + self.assertIs(result.observed_status, CriterionStatus.UNKNOWN) + self.assertIn("no validation report", result.reason) + self.assertIn(self.DATASET_ID, result.reason) + + def test_the_reason_names_the_dataset_that_was_validated(self): + result = CompliantEvaluator().evaluate(self._compliant_ctx(3)) + self.assertIn(self.DATASET_ID, result.reason) + + def test_a_report_with_no_error_count_is_unknown_not_a_pass(self): + """total_error is nullable, and a missing count must not read as zero errors.""" + result = CompliantEvaluator().evaluate(self._compliant_ctx(None)) + self.assertIs(result.observed_status, CriterionStatus.UNKNOWN) + self.assertIn("no total_error", result.reason) + + def test_is_never_not_applicable(self): + """Compliance applies to every feed, seasonal ones included.""" + self.assertIsNot( + CompliantEvaluator() + .evaluate(self._compliant_ctx(0, seasonal=True)) + .observed_status, + CriterionStatus.NOT_APPLICABLE, + ) + + def test_has_a_grace_period_and_serves_probation(self): + self.assertEqual(CompliantEvaluator().grace_period, timedelta(days=30)) + self.assertEqual(CompliantEvaluator().probation_period, PROBATION_PERIOD) + + if __name__ == "__main__": unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py index 70af32f04..d875c4b38 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py @@ -21,9 +21,13 @@ from unittest.mock import MagicMock, patch from shared.common.seal_criteria import ( + AVAILABILITY_LOOKBACK, PROBATION_PERIOD, + PROBATION_EXEMPT_CRITERIA, CriterionStatus, SealCriterionName, + SealStatus, + roll_up_seal_status, ) from tasks.seal_of_reliability.context import ( build_contexts, @@ -44,10 +48,13 @@ from shared.database_gen.sqlacodegen_models import ( Feed, FeedReliabilitySeal, + GtfsFeedAvailabilityCheck, Gtfsdataset, Gtfsfeed, SealCriterion, SealCriterionSnapshot, + Validationreport, + t_validationreportgtfsdataset, ) from test_shared.test_utils.database_utils import default_db_url @@ -79,7 +86,8 @@ class _StandInEvaluator(CriterionEvaluator): It reads `official` so a test can drive it with the existing `set_official` helper, but unlike Official it debounces failures and serves probation afterwards. It borrows the - `available` enum value, which has no evaluator of its own yet (#1784). + `available` enum value and is patched over the whole registry, so the real + AvailableEvaluator never runs alongside it. """ name = SealCriterionName.AVAILABLE @@ -105,7 +113,7 @@ def _evaluate(self, ctx): class _GoesDarkEvaluator(CriterionEvaluator): - """A criterion that loses its upstream input partway through, standing in for #1784. + """A criterion that loses its upstream input partway through. It returns no verdict from `DARK_FROM` onwards, keyed on the clock rather than on `official` so that a test can drive it and Official in opposite directions at the same @@ -158,6 +166,25 @@ def _evaluate(self, ctx): STOPS_APPLYING = [OfficialEvaluator(), _StopsApplyingEvaluator()] +class _NeverAnswersEvaluator(CriterionEvaluator): + """A criterion whose input never arrives, so it never produces a verdict. + + Stands in for a criterion whose data source has not started collecting yet. Its + `confirmed_status` therefore stays NEVER_EVALUATED for good, which is the input the + feed-level UNKNOWN roll-up is about. + """ + + name = SealCriterionName.AVAILABLE + grace_period = None + + def _evaluate(self, ctx): + return CriterionStatus.UNKNOWN, "stand-in never has an input" + + +NEVER_ANSWERS = [_NeverAnswersEvaluator()] +OFFICIAL_AND_NEVER_ANSWERS = [OfficialEvaluator(), _NeverAnswersEvaluator()] + + def _seed_feed( db_session, feed_id: str, @@ -215,6 +242,43 @@ def _seed_dataset( db_session.commit() +def _seed_availability_check(db_session, feed_id: str, success, checked_at=None): + """One `gtfs_feed_availability_check` row for the feed.""" + db_session.add( + GtfsFeedAvailabilityCheck( + feed_id=feed_id, + checked_at=checked_at or NOW, + request_url=f"https://example.com/{feed_id}.zip", + request_type="http_head", + status_code=200 if success else 503, + success=success, + ) + ) + db_session.commit() + + +def _seed_validation_report( + db_session, dataset_id: str, total_error, validated_at=None, suffix="" +): + """A validation report for the dataset, linked through validationreportgtfsdataset.""" + report_id = f"{dataset_id}_report{suffix}" + db_session.add( + Validationreport( + id=report_id, + validator_version=f"1.0.0{suffix}", + validated_at=validated_at or NOW - timedelta(hours=1), + total_error=total_error, + ) + ) + db_session.flush() + db_session.execute( + t_validationreportgtfsdataset.insert().values( + dataset_id=dataset_id, validation_report_id=report_id + ) + ) + db_session.commit() + + def _set_seasonal(db_session, feed_id: str, seasonal): db_session.execute( Feed.__table__.update() @@ -242,6 +306,12 @@ def _cleanup(db_session): tables, all of which are ON DELETE CASCADE. """ db_session.execute(delete(Feed).where(Feed.stable_id.like(f"{PREFIX}%"))) + # validationreport is not reachable by cascade from feed: deleting the feed cascades to + # gtfsdataset and to the validationreportgtfsdataset join row, but leaves the report + # itself orphaned, and the next test collides on its primary key. + db_session.execute( + delete(Validationreport).where(Validationreport.id.like(f"{PREFIX}%")) + ) db_session.commit() @@ -294,6 +364,21 @@ def seal_row(feed_id, db_session): select(table).where(table.c.feed_id == feed_id) ).first() + def derived_seal_status(self, feed_id): + """The feed's seal status, derived from the persisted rows the way the read API does. + + Nothing stores it, so an assertion has to re-derive it - and doing so from the rows the run + wrote checks what the API will actually see. + """ + return roll_up_seal_status( + ( + CriterionStatus(row.confirmed_status), + row.probation_start is not None + and row.criterion not in PROBATION_EXEMPT_CRITERIA, + ) + for row in self.criterion_rows(feed_id).values() + ).value + @staticmethod @with_db_session(db_url=default_db_url) def set_official(feed_id, official, db_session): @@ -321,6 +406,20 @@ def seed_dataset( ): _seed_dataset(db_session, feed_id, coverage_end, downloaded_at, suffix) + @staticmethod + @with_db_session(db_url=default_db_url) + def seed_availability_check(feed_id, success, checked_at=None, db_session=None): + _seed_availability_check(db_session, feed_id, success, checked_at) + + @staticmethod + @with_db_session(db_url=default_db_url) + def seed_validation_report( + dataset_id, total_error, validated_at=None, suffix="", db_session=None + ): + _seed_validation_report( + db_session, dataset_id, total_error, validated_at, suffix + ) + @staticmethod @with_db_session(db_url=default_db_url) def set_feed_created_at(feed_id, created_at, db_session): @@ -481,6 +580,162 @@ def test_the_latest_dataset_is_resolved_as_of_now(self, db_session): "by then it had", ) + @with_db_session(db_url=default_db_url) + def test_a_check_inside_the_rolling_window_counts(self, db_session): + _seed_availability_check(db_session, TRACKED, True, NOW - timedelta(hours=6)) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertTrue(ctx.availability_check.success) + + @with_db_session(db_url=default_db_url) + def test_the_window_is_exactly_the_lookback(self, db_session): + """A check just inside the window counts; the same check an hour older does not.""" + _seed_availability_check( + db_session, + TRACKED, + True, + NOW - AVAILABILITY_LOOKBACK + timedelta(minutes=1), + ) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + self.assertIsNotNone( + build_contexts(db_session, feeds, NOW)[feeds[0].id].availability_check + ) + self.assertIsNone( + build_contexts(db_session, feeds, NOW + timedelta(hours=1))[ + feeds[0].id + ].availability_check + ) + + @with_db_session(db_url=default_db_url) + def test_a_check_older_than_the_fallback_window_is_ignored(self, db_session): + _seed_availability_check(db_session, TRACKED, True, NOW - timedelta(days=3)) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertIsNone(ctx.availability_check) + + @with_db_session(db_url=default_db_url) + def test_a_check_after_now_is_not_visible_yet(self, db_session): + """Same replay rule as everywhere else: a run never reads its own future.""" + _seed_availability_check(db_session, TRACKED, True, NOW + timedelta(hours=1)) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + self.assertIsNone( + build_contexts(db_session, feeds, NOW)[feeds[0].id].availability_check + ) + + @with_db_session(db_url=default_db_url) + def test_the_latest_check_in_the_window_decides(self, db_session): + """Not "any success": the most recent answer describes the feed now.""" + _seed_availability_check(db_session, TRACKED, True, NOW - timedelta(hours=5)) + _seed_availability_check(db_session, TRACKED, False, NOW - timedelta(hours=1)) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertFalse(ctx.availability_check.success) + self.assertEqual(ctx.availability_check.checked_at, NOW - timedelta(hours=1)) + + @with_db_session(db_url=default_db_url) + def test_a_failed_check_is_a_verdict_not_a_missing_one(self, db_session): + """A check that ran and failed and no check at all are different answers.""" + _seed_availability_check(db_session, TRACKED, False, NOW) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertIsNotNone(ctx.availability_check) + self.assertFalse(ctx.availability_check.success) + + @with_db_session(db_url=default_db_url) + def test_the_latest_validation_report_is_loaded(self, db_session): + _seed_dataset(db_session, TRACKED, coverage_end=NOW + timedelta(days=90)) + _seed_validation_report(db_session, f"{TRACKED}_dataset", total_error=3) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertEqual(ctx.latest_validation_report.total_error, 3) + + @with_db_session(db_url=default_db_url) + def test_a_feed_with_no_report_carries_none(self, db_session): + _seed_dataset(db_session, TRACKED, coverage_end=NOW + timedelta(days=90)) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertIsNotNone(ctx.latest_dataset, "the dataset is there ...") + self.assertIsNone(ctx.latest_validation_report, "... a report is not") + + @with_db_session(db_url=default_db_url) + def test_a_report_on_a_superseded_dataset_is_not_loaded(self, db_session): + """The report is the latest dataset's, so an older dataset's report is not it. + + Validation lags publication, so the newest dataset may have no report yet. That is + left as "no report" rather than backfilled from the dataset before it: the criterion + is about the data being served now. + """ + _seed_dataset( + db_session, + TRACKED, + coverage_end=NOW + timedelta(days=90), + downloaded_at=NOW - timedelta(days=5), + suffix="_older", + ) + _seed_validation_report( + db_session, f"{TRACKED}_dataset_older", total_error=0, suffix="_older" + ) + _seed_dataset( + db_session, + TRACKED, + coverage_end=NOW + timedelta(days=90), + downloaded_at=NOW - timedelta(hours=2), + suffix="_newest", + ) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + + self.assertEqual(ctx.latest_dataset.dataset_id, f"{TRACKED}_dataset_newest") + self.assertIsNone( + ctx.latest_validation_report, + "the older dataset's report does not describe what is being served", + ) + + @with_db_session(db_url=default_db_url) + def test_the_validation_report_is_resolved_as_of_now(self, db_session): + """Same replay rule as the dataset: a later re-validation must not leak backwards.""" + _seed_dataset(db_session, TRACKED, coverage_end=NOW + timedelta(days=90)) + dataset_id = f"{TRACKED}_dataset" + _seed_validation_report( + db_session, + dataset_id, + 5, + validated_at=NOW - timedelta(days=2), + suffix="_old", + ) + _seed_validation_report( + db_session, + dataset_id, + 0, + validated_at=NOW + timedelta(days=2), + suffix="_new", + ) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + + as_of_now = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertEqual( + as_of_now.latest_validation_report.total_error, + 5, + "the re-validation had not happened yet", + ) + + later = NOW + timedelta(days=3) + as_of_later = build_contexts(db_session, feeds, later)[feeds[0].id] + self.assertEqual(as_of_later.latest_validation_report.total_error, 0) + + @with_db_session(db_url=default_db_url) + def test_a_report_with_no_validated_at_is_excluded(self, db_session): + """It cannot be placed in time, so it is dropped rather than guessed at.""" + _seed_dataset(db_session, TRACKED, coverage_end=NOW + timedelta(days=90)) + _seed_validation_report(db_session, f"{TRACKED}_dataset", total_error=0) + db_session.execute( + Validationreport.__table__.update().values(validated_at=None) + ) + db_session.commit() + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertIsNone(ctx.latest_validation_report) + @with_db_session(db_url=default_db_url) def test_a_dataset_with_no_downloaded_at_cannot_be_placed_in_time(self, db_session): """It is excluded rather than guessed at: we cannot say whether it existed yet.""" @@ -767,6 +1022,146 @@ def test_feeds_omitted_is_zero_when_nothing_was_dropped(self): self.assertEqual(report["feeds_omitted"], 0) +class TestSealStatusRollUp(SealDbTestCase): + """The four-way feed-level outcome, and the boolean that hangs off it. + + `has_seal` answers only "does the feed hold the seal", so all three non-granting values + read as false through it. `seal_status` is what tells them apart, and the distinction that + matters is between a feed that was judged and did not qualify and one that could not be + judged at all. + """ + + @patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", ONLY_OFFICIAL) + def test_every_criterion_passing_grants_the_seal(self): + update_seals(dry_run=False, stable_feed_ids=[OFFICIAL], now=NOW) + + seal = self.seal_row(OFFICIAL) + self.assertEqual(self.derived_seal_status(OFFICIAL), SealStatus.GRANTED.value) + self.assertTrue(seal.has_seal) + + @patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", ONLY_OFFICIAL) + def test_a_judged_feed_that_does_not_qualify_is_not_granted(self): + """Not the same as unknown: every criterion answered, and the answer was no.""" + update_seals(dry_run=False, stable_feed_ids=[NOT_OFFICIAL], now=NOW) + + seal = self.seal_row(NOT_OFFICIAL) + self.assertEqual( + self.derived_seal_status(NOT_OFFICIAL), SealStatus.NOT_GRANTED.value + ) + self.assertFalse(seal.has_seal) + + @patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", NEVER_ANSWERS) + def test_no_criterion_ever_judged_is_never_evaluated(self): + """A row is written - the attempt happened - but nothing has been decided.""" + update_seals(dry_run=False, stable_feed_ids=[OFFICIAL], now=NOW) + + seal = self.seal_row(OFFICIAL) + self.assertEqual( + self.derived_seal_status(OFFICIAL), SealStatus.NEVER_EVALUATED.value + ) + self.assertFalse(seal.has_seal) + + @patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", OFFICIAL_AND_NEVER_ANSWERS + ) + def test_one_criterion_without_a_verdict_makes_the_whole_seal_unknown(self): + """Official passes, but the other criterion has never been judged. + + The feed may well qualify - which is exactly why this is not NOT_GRANTED - but it + cannot be granted the seal on evidence covering only half its criteria. + """ + update_seals(dry_run=False, stable_feed_ids=[OFFICIAL], now=NOW) + + rows = self.criterion_rows(OFFICIAL) + self.assertEqual( + rows[SealCriterionName.OFFICIAL.value].confirmed_status, + CriterionStatus.PASS.value, + ) + seal = self.seal_row(OFFICIAL) + self.assertEqual(self.derived_seal_status(OFFICIAL), SealStatus.UNKNOWN.value) + self.assertFalse(seal.has_seal) + + @patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", OFFICIAL_AND_NEVER_ANSWERS + ) + def test_a_transient_outage_does_not_make_a_judged_seal_unknown(self): + """The distinction the roll-up rests on: no verdict *ever*, not no verdict today. + + The first run is driven by the real registry, so both criteria reach a verdict. The + second patches one of them dark; it keeps its stored pass, and the seal stays granted. + """ + with patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", + [OfficialEvaluator(), _StandInEvaluator()], + ): + update_seals(dry_run=False, stable_feed_ids=[OFFICIAL], now=NOW) + self.assertEqual(self.derived_seal_status(OFFICIAL), SealStatus.GRANTED.value) + + later = NOW + timedelta(days=1) + update_seals(dry_run=False, stable_feed_ids=[OFFICIAL], now=later) + + row = self.criterion_rows(OFFICIAL)[SealCriterionName.AVAILABLE.value] + self.assertEqual(row.observed_status, CriterionStatus.UNKNOWN.value) + self.assertEqual(row.confirmed_status, CriterionStatus.PASS.value) + seal = self.seal_row(OFFICIAL) + self.assertEqual(self.derived_seal_status(OFFICIAL), SealStatus.GRANTED.value) + self.assertTrue(seal.has_seal) + + @patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", ONLY_OFFICIAL) + def test_the_report_counts_and_names_the_outcomes(self): + """The two decided values, from `official` alone across the seeded feeds.""" + report = update_seals(dry_run=True, stable_feed_ids=OURS, now=NOW) + + counts = report["seal_status_counts"] + self.assertEqual(set(counts), {status.value for status in SealStatus}) + self.assertEqual( + sum(counts.values()), + report["total_feeds"], + "every feed lands in exactly one", + ) + self.assertEqual( + {row["stable_id"]: row["seal_status"] for row in report["feeds"]}, + { + OFFICIAL: SealStatus.GRANTED.value, + INACTIVE: SealStatus.GRANTED.value, + NOT_OFFICIAL: SealStatus.NOT_GRANTED.value, + # `official IS NULL` is a verdict for Official, not an absent one. + UNKNOWN_OFFICIAL: SealStatus.NOT_GRANTED.value, + }, + ) + self.assertEqual(counts[SealStatus.GRANTED.value], 2) + + @patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", OFFICIAL_AND_NEVER_ANSWERS + ) + def test_a_failing_criterion_decides_the_seal_even_with_another_unjudged(self): + """A feed failing Official is NOT_GRANTED, not UNKNOWN, though a criterion has never + been judged. + + One failure is enough to deny the seal, so the outcome is decidable however little we + know about the rest: no verdict the missing criterion could produce would grant it. + UNKNOWN is reserved for the case where nothing is failing and the evaluation is simply + incomplete. + """ + update_seals(dry_run=False, stable_feed_ids=[NOT_OFFICIAL], now=NOW) + + rows = self.criterion_rows(NOT_OFFICIAL) + self.assertEqual( + rows[SealCriterionName.OFFICIAL.value].confirmed_status, + CriterionStatus.FAIL.value, + ) + self.assertEqual( + rows[SealCriterionName.AVAILABLE.value].confirmed_status, + CriterionStatus.NEVER_EVALUATED.value, + "the other criterion has no verdict at all", + ) + seal = self.seal_row(NOT_OFFICIAL) + self.assertEqual( + self.derived_seal_status(NOT_OFFICIAL), SealStatus.NOT_GRANTED.value + ) + self.assertFalse(seal.has_seal) + + @patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", ONLY_OFFICIAL) class TestCriterionSnapshot(SealDbTestCase): """seal_criterion_snapshot, the per-day record of each criterion (issue #1809). @@ -896,21 +1291,243 @@ def test_every_criterion_is_written_for_every_feed(self): {evaluator.name.value for evaluator in EVALUATORS}, ) - def test_a_feed_meeting_all_three_earns_the_seal(self): + def satisfy_everything(self): + """Seed the inputs every implemented criterion needs, all passing.""" self.seed_dataset(TRACKED, self.FAR_FUTURE) + self.seed_availability_check(TRACKED, success=True) + self.seed_validation_report(f"{TRACKED}_dataset", total_error=0) + + def test_a_feed_meeting_every_criterion_earns_the_seal(self): + self.satisfy_everything() update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) self.assertEqual( self.criteria_of(), { - SealCriterionName.OFFICIAL.value: CriterionStatus.PASS.value, - SealCriterionName.STABLE.value: CriterionStatus.PASS.value, - SealCriterionName.FRESH_COVERAGE.value: CriterionStatus.PASS.value, + evaluator.name.value: CriterionStatus.PASS.value + for evaluator in EVALUATORS }, ) self.assertTrue(self.seal_row(TRACKED).has_seal) + def test_criteria_with_no_data_leave_the_seal_unknown(self): + """Available and Compliant have never had a verdict, so the seal cannot be decided. + + They do not *deny* the seal - the feed may well qualify - but with two of five + criteria unjudged, saying it does not qualify would be as wrong as saying it does. + """ + self.seed_dataset(TRACKED, self.FAR_FUTURE) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + rows = self.criterion_rows(TRACKED) + for criterion in ( + SealCriterionName.AVAILABLE.value, + SealCriterionName.COMPLIANT.value, + ): + with self.subTest(criterion=criterion): + self.assertEqual( + rows[criterion].observed_status, CriterionStatus.UNKNOWN.value + ) + self.assertEqual( + rows[criterion].confirmed_status, + CriterionStatus.NEVER_EVALUATED.value, + ) + seal = self.seal_row(TRACKED) + self.assertEqual(self.derived_seal_status(TRACKED), SealStatus.UNKNOWN.value) + self.assertFalse(seal.has_seal, "unknown is not a grant") + + def test_a_failed_availability_check_denies_the_seal(self): + self.satisfy_everything() + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + # Next day: the only check that ran failed. + later = NOW + timedelta(days=1) + self.seed_availability_check(TRACKED, success=False, checked_at=later) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=later) + + row = self.criterion_rows(TRACKED)[SealCriterionName.AVAILABLE.value] + self.assertEqual(row.observed_status, CriterionStatus.FAIL.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.PASS.value, + "Available has a 14-day grace period and had already passed", + ) + self.assertTrue(self.seal_row(TRACKED).has_seal, "still held, under grace") + + def test_a_recovery_later_in_the_window_wins(self): + """Several checks in one window: the most recent one is the verdict.""" + self.satisfy_everything() + later = NOW + timedelta(days=1) + self.seed_availability_check(TRACKED, success=False, checked_at=later) + self.seed_availability_check( + TRACKED, success=True, checked_at=later + timedelta(hours=2) + ) + + update_seals( + dry_run=False, + stable_feed_ids=[TRACKED], + now=later + timedelta(hours=3), + ) + + row = self.criterion_rows(TRACKED)[SealCriterionName.AVAILABLE.value] + self.assertEqual(row.observed_status, CriterionStatus.PASS.value) + + def test_a_check_still_answers_a_second_run_inside_the_window(self): + """The window is a rolling 24h, not "checks this run has not seen yet".""" + self.satisfy_everything() + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + # Six hours later, no new check: the earlier one is still inside the window. + later = NOW + timedelta(hours=6) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=later) + + row = self.criterion_rows(TRACKED)[SealCriterionName.AVAILABLE.value] + self.assertEqual(row.observed_status, CriterionStatus.PASS.value) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_the_criterion_goes_quiet_once_the_check_ages_out(self): + """A day with no check at all is UNKNOWN, which freezes the last verdict.""" + self.satisfy_everything() + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + stale = NOW + AVAILABILITY_LOOKBACK + timedelta(hours=1) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=stale) + + row = self.criterion_rows(TRACKED)[SealCriterionName.AVAILABLE.value] + self.assertEqual(row.observed_status, CriterionStatus.UNKNOWN.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.PASS.value, + "UNKNOWN freezes the criterion at its last verdict rather than failing it", + ) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_a_late_availability_job_is_picked_up_by_the_next_run(self): + """The reason for the window: a check the seal run missed is not lost. + + The seal runs, sees nothing; the availability job lands afterwards; the next seal run + still reads that check instead of it falling into a closed calendar day. + """ + self.seed_dataset(TRACKED, self.FAR_FUTURE) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertEqual( + self.criterion_rows(TRACKED)[ + SealCriterionName.AVAILABLE.value + ].observed_status, + CriterionStatus.UNKNOWN.value, + ) + + self.seed_availability_check( + TRACKED, success=True, checked_at=NOW + timedelta(hours=1) + ) + later = NOW + timedelta(hours=2) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=later) + + row = self.criterion_rows(TRACKED)[SealCriterionName.AVAILABLE.value] + self.assertEqual(row.observed_status, CriterionStatus.PASS.value) + self.assertEqual(row.confirmed_status, CriterionStatus.PASS.value) + + def test_validation_errors_deny_the_seal(self): + self.seed_dataset(TRACKED, self.FAR_FUTURE) + self.seed_availability_check(TRACKED, success=True) + self.seed_validation_report(f"{TRACKED}_dataset", total_error=7) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + row = self.criterion_rows(TRACKED)[SealCriterionName.COMPLIANT.value] + self.assertEqual(row.observed_status, CriterionStatus.FAIL.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.FAIL.value, + "a first verdict gets no grace period", + ) + self.assertFalse(self.seal_row(TRACKED).has_seal) + + def test_a_dataset_with_no_report_is_unknown_not_compliant(self): + """A missing report is not a clean bill of health.""" + self.seed_dataset(TRACKED, self.FAR_FUTURE) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + row = self.criterion_rows(TRACKED)[SealCriterionName.COMPLIANT.value] + self.assertEqual(row.observed_status, CriterionStatus.UNKNOWN.value) + self.assertIsNone(row.last_verdict_at) + + def test_a_feed_publishing_faster_than_validation_keeps_its_last_verdict(self): + """Daily publishing plus lagging validation, which is the common case. + + Each new dataset arrives unvalidated, so Compliant observes UNKNOWN on the days in + between. That freezes the criterion at the verdict it last earned instead of failing + it, so the feed keeps the seal while the validator catches up. + """ + self.seed_availability_check(TRACKED, success=True) + self.seed_dataset( + TRACKED, + self.FAR_FUTURE, + downloaded_at=NOW - timedelta(days=2), + suffix="_validated", + ) + self.seed_validation_report( + f"{TRACKED}_dataset_validated", total_error=0, suffix="_validated" + ) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertEqual( + self.criterion_rows(TRACKED)[ + SealCriterionName.COMPLIANT.value + ].observed_status, + CriterionStatus.PASS.value, + ) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + # Published since, and not validated yet. + later = NOW + timedelta(hours=6) + self.seed_dataset( + TRACKED, + self.FAR_FUTURE, + downloaded_at=NOW + timedelta(hours=1), + suffix="_fresh", + ) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=later) + + row = self.criterion_rows(TRACKED)[SealCriterionName.COMPLIANT.value] + self.assertEqual(row.observed_status, CriterionStatus.UNKNOWN.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.PASS.value, + "the verdict on the dataset we did validate still stands", + ) + seal = self.seal_row(TRACKED) + self.assertEqual(self.derived_seal_status(TRACKED), SealStatus.GRANTED.value) + self.assertTrue(seal.has_seal) + + def test_compliant_reads_the_latest_report_of_the_latest_dataset(self): + """Several validator versions run against one dataset; the newest one decides.""" + self.seed_dataset(TRACKED, self.FAR_FUTURE) + self.seed_availability_check(TRACKED, success=True) + dataset_id = f"{TRACKED}_dataset" + self.seed_validation_report( + dataset_id, + total_error=9, + validated_at=NOW - timedelta(days=3), + suffix="_old", + ) + self.seed_validation_report( + dataset_id, + total_error=0, + validated_at=NOW - timedelta(hours=1), + suffix="_new", + ) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + row = self.criterion_rows(TRACKED)[SealCriterionName.COMPLIANT.value] + self.assertEqual(row.observed_status, CriterionStatus.PASS.value) + self.assertTrue(self.seal_row(TRACKED).has_seal) + def test_a_feed_new_to_the_database_cannot_hold_the_seal_yet(self): """Stable reads the feed's own age, so a freshly added feed fails it.""" self.set_feed_created_at(TRACKED, NOW - timedelta(days=30)) @@ -933,6 +1550,8 @@ def test_the_seal_arrives_once_the_feed_is_old_enough(self): """ self.set_feed_created_at(TRACKED, NOW) self.seed_dataset(TRACKED, NOW + timedelta(days=400)) + self.seed_availability_check(TRACKED, success=True) + self.seed_validation_report(f"{TRACKED}_dataset", total_error=0) update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) self.assertFalse( self.seal_row(TRACKED).has_seal, "in the database for zero days" @@ -949,7 +1568,7 @@ def test_the_seal_arrives_once_the_feed_is_old_enough(self): def test_an_old_feed_qualifies_on_its_very_first_run(self): """The point of reading `feed.created_at`: no six-month wait after deployment for a feed that has already been in the catalog for years.""" - self.seed_dataset(TRACKED, self.FAR_FUTURE) + self.satisfy_everything() update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) @@ -961,7 +1580,7 @@ def test_an_old_feed_qualifies_on_its_very_first_run(self): def test_an_unstable_producer_url_denies_the_seal_immediately(self): """Stable has no grace period, so the flag costs the seal the day it is set.""" - self.seed_dataset(TRACKED, self.FAR_FUTURE) + self.satisfy_everything() update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) self.assertTrue(self.seal_row(TRACKED).has_seal) @@ -975,8 +1594,8 @@ def test_an_unstable_producer_url_denies_the_seal_immediately(self): ) self.assertFalse(self.seal_row(TRACKED).has_seal) - def test_a_feed_with_no_dataset_leaves_fresh_out_of_the_roll_up(self): - """UNKNOWN is not a failure: the other two criteria still decide the seal.""" + def test_a_feed_with_no_dataset_leaves_the_seal_unknown(self): + """UNKNOWN is not a failure - but it is not a pass to be skipped over either.""" update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) @@ -988,10 +1607,13 @@ def test_a_feed_with_no_dataset_leaves_fresh_out_of_the_roll_up(self): "no verdict was ever produced, so the criterion is out of service", ) self.assertIsNone(row.last_verdict_at) - self.assertTrue( - self.seal_row(TRACKED).has_seal, - "Official and Stable carry it while Fresh has nothing to say", + seal = self.seal_row(TRACKED) + self.assertEqual( + self.derived_seal_status(TRACKED), + SealStatus.UNKNOWN.value, + "Official and Stable pass, but Fresh has never been judged", ) + self.assertFalse(seal.has_seal) def test_lapsed_coverage_is_confirmed_at_once_on_a_first_evaluation(self): """Fresh has a 14-day grace period, but a criterion that has never passed has not @@ -1007,6 +1629,8 @@ def test_lapsed_coverage_is_confirmed_at_once_on_a_first_evaluation(self): def test_the_grace_period_absorbs_a_lapse_on_a_feed_that_was_passing(self): self.seed_dataset(TRACKED, NOW + timedelta(days=10)) + self.seed_availability_check(TRACKED, success=True) + self.seed_validation_report(f"{TRACKED}_dataset", total_error=0) update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) self.assertTrue(self.seal_row(TRACKED).has_seal) @@ -1046,6 +1670,7 @@ def test_a_seasonal_feed_is_not_denied_by_fresh(self): """NOT_APPLICABLE withdraws the criterion instead of failing it, which is the whole point of the value: a seasonal feed keeps the seal on the criteria that do apply. """ + self.satisfy_everything() self.set_seasonal(TRACKED, True) update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) @@ -1058,6 +1683,8 @@ def test_a_seasonal_feed_is_not_denied_by_fresh(self): def test_becoming_seasonal_freezes_a_failing_fresh_rather_than_carrying_it(self): """A feed marked seasonal after a confirmed Fresh failure stops being judged on it.""" self.seed_dataset(TRACKED, NOW - timedelta(days=1)) + self.seed_availability_check(TRACKED, success=True) + self.seed_validation_report(f"{TRACKED}_dataset", total_error=0) update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) self.assertFalse(self.seal_row(TRACKED).has_seal) @@ -1072,7 +1699,7 @@ def test_becoming_seasonal_freezes_a_failing_fresh_rather_than_carrying_it(self) ) self.assertTrue(self.seal_row(TRACKED).has_seal) - def test_a_run_reports_all_three_criteria_with_their_reasons(self): + def test_a_run_reports_every_criterion_with_its_reason(self): self.seed_dataset(TRACKED, self.FAR_FUTURE) report = update_seals(dry_run=True, stable_feed_ids=[TRACKED], now=NOW) @@ -1398,11 +2025,12 @@ def test_going_dark_records_the_attempt_without_moving_the_verdict(self): frozen.last_verdict_at, self.FAILED_AT, "but got no new verdict" ) - def test_going_dark_before_any_verdict_leaves_the_criterion_out_of_service(self): + def test_going_dark_before_any_verdict_leaves_the_seal_unknown(self): """The other half: with no verdict ever, there is nothing to hold in service. A row is written, because the attempt is worth recording, but `confirmed_status` - stays NEVER_EVALUATED so the criterion is skipped rather than denying the seal. + stays NEVER_EVALUATED - and a criterion that has never been judged cannot be skipped + over, so the seal is UNKNOWN rather than granted on the strength of the others. """ self.run_at(DARK_FROM) @@ -1415,9 +2043,10 @@ def test_going_dark_before_any_verdict_leaves_the_criterion_out_of_service(self) "no verdict has ever been produced", ) self.assertIsNone(row.last_verdict_at) - self.assertTrue( - self.seal_row(OFFICIAL).has_seal, - "and the criterion is skipped rather than denying the seal", + seal = self.seal_row(OFFICIAL) + self.assertEqual(self.derived_seal_status(OFFICIAL), SealStatus.UNKNOWN.value) + self.assertFalse( + seal.has_seal, "unknown is not a grant, and not a denial either" )