diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 048baa8..8a052cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,3 +26,25 @@ jobs: - run: python -m cas_evals.cli benchmarks/v0.2/golden.json - run: python -m cas_evals.cli benchmarks/v0.2/adversarial.json - run: python -m cas_evals.release --check + + registry-smoke: + # Separate from the offline `verify` matrix job because this job needs + # network egress and should not block the offline unit-test matrix on + # transient network failures. As of this job's introduction, the + # cas-contracts PR from plan 32-01 (rewriting schema $id to this same + # Pages registry URL) has not yet merged. This job checks paths that + # already resolve today regardless of that PR's merge status -- the + # v0.1/v1.0/v1.1 registry lines were published by prior phases' pages.yml + # work. Merging 32-01 only changes the $id field text embedded inside + # those already-resolving files, not which URLs exist. + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.11" + - run: python -m pip install -e . + - run: python -m cas_evals.registry_check diff --git a/releases/v0.2.0/manifest.json b/releases/v0.2.0/manifest.json index 6c23db1..c5ac4fa 100644 --- a/releases/v0.2.0/manifest.json +++ b/releases/v0.2.0/manifest.json @@ -3,7 +3,7 @@ "releaseVersion": "v0.2.0", "releasedAt": "2026-06-11T12:00:00Z", "sharedContract": { - "provenanceDigest": "sha256:3d82b533691c779e9cf2361f0491a2d8873ef71e22f279c5cfdfc784515be9a5", + "provenanceDigest": "sha256:d206cb050c42c80d10d37b418f6a6bbb9da08aba0ae76e77d3344108613829a2", "release": "https://github.com/Coding-Autopilot-System/cas-contracts/releases/tag/v0.1.0", "repository": "https://github.com/Coding-Autopilot-System/cas-contracts", "tag": "v0.1.0" diff --git a/src/cas_evals/contracts.py b/src/cas_evals/contracts.py index 8e71bce..2837a18 100644 --- a/src/cas_evals/contracts.py +++ b/src/cas_evals/contracts.py @@ -61,9 +61,9 @@ def verify_vendored_contract() -> dict[str, Any]: common = _load_json(VENDOR_DIR / "common.schema.json") evaluation = _load_json(VENDOR_DIR / "evaluation-result.schema.json") - if common.get("$id") != "https://schemas.coding-autopilot.dev/v0.1/common.schema.json": + if common.get("$id") != "https://coding-autopilot-system.github.io/cas-contracts/registry/v0.1/common.schema.json": raise ContractValidationError("unexpected common schema identity") - if evaluation.get("$id") != "https://schemas.coding-autopilot.dev/v0.1/evaluation-result.schema.json": + if evaluation.get("$id") != "https://coding-autopilot-system.github.io/cas-contracts/registry/v0.1/evaluation-result.schema.json": raise ContractValidationError("unexpected evaluation schema identity") if evaluation["allOf"][0].get("$ref") != "common.schema.json#/$defs/lifecycleMetadata": raise ContractValidationError("evaluation schema does not reference the vendored common schema") diff --git a/src/cas_evals/registry_check.py b/src/cas_evals/registry_check.py new file mode 100644 index 0000000..09f87b1 --- /dev/null +++ b/src/cas_evals/registry_check.py @@ -0,0 +1,80 @@ +"""Registry-fetch smoke check for the live cas-contracts GitHub Pages registry. + +Performs live HTTP GETs against known-published registry paths and asserts +each responds with HTTP 200. This is deliberately network-dependent and +distinct from cas_evals.contracts.verify_vendored_contract(), which only +checks bytes on disk and never touches the network. The two mechanisms are +additive: verify_vendored_contract() proves the pinned offline copy is intact; +this module proves the live registry actually resolves today. +""" + +from __future__ import annotations + +import sys +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +DEFAULT_BASE_URL = "https://coding-autopilot-system.github.io/cas-contracts/registry" +DEFAULT_PATHS = ( + "index.json", + "v0.1/manifest.json", + "v0.1/common.schema.json", + "v0.1/evaluation-result.schema.json", +) +MAX_RESPONSE_BYTES = 2_000_000 + + +class RegistryCheckError(RuntimeError): + """Raised when a registry-fetch smoke check fails.""" + + +def check_registry_urls( + base_url: str, + paths: list[str] | tuple[str, ...], + timeout_seconds: float = 10, +) -> list[tuple[str, int]]: + """Fetch each path joined onto base_url and assert HTTP 200. + + Returns a list of (path, status_code) tuples when every URL responds + with 200. Raises RegistryCheckError naming the first failing path if any + URL returns non-200, times out, or the network is unavailable. + """ + results: list[tuple[str, int]] = [] + for path in paths: + url = f"{base_url.rstrip('/')}/{path.lstrip('/')}" + request = Request(url, method="GET") + try: + with urlopen(request, timeout=timeout_seconds) as response: + status = response.status + response.read(MAX_RESPONSE_BYTES + 1) + except HTTPError as error: + raise RegistryCheckError(f"{path} returned HTTP {error.code}") from error + except (URLError, TimeoutError, OSError) as error: + raise RegistryCheckError(f"{path} is unavailable (network error)") from error + if status != 200: + raise RegistryCheckError(f"{path} returned HTTP {status}") + results.append((path, status)) + return results + + +def main(argv: list[str] | None = None) -> int: + """CLI entry point: python -m cas_evals.registry_check [--base-url URL].""" + args = list(sys.argv[1:] if argv is None else argv) + base_url = DEFAULT_BASE_URL + if "--base-url" in args: + index = args.index("--base-url") + base_url = args[index + 1] + + try: + results = check_registry_urls(base_url, DEFAULT_PATHS) + except RegistryCheckError as error: + print(f"FAIL: {error}") + return 1 + + for path, status in results: + print(f"{path}: {status}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_registry_check.py b/tests/test_registry_check.py new file mode 100644 index 0000000..68d31bb --- /dev/null +++ b/tests/test_registry_check.py @@ -0,0 +1,95 @@ +import io +import unittest +from contextlib import redirect_stdout +from unittest.mock import MagicMock, patch +from urllib.error import HTTPError, URLError + +from cas_evals.registry_check import RegistryCheckError, check_registry_urls, main + + +def _fake_response(status_code): + response = MagicMock() + response.status = status_code + response.read.return_value = b"" + response.__enter__.return_value = response + response.__exit__.return_value = False + return response + + +class RegistryCheckTests(unittest.TestCase): + def test_all_paths_return_200(self): + with patch("cas_evals.registry_check.urlopen", return_value=_fake_response(200)) as urlopen: + results = check_registry_urls( + "https://coding-autopilot-system.github.io/cas-contracts/registry", + ["index.json", "v0.1/manifest.json"], + ) + self.assertEqual( + results, + [("index.json", 200), ("v0.1/manifest.json", 200)], + ) + self.assertEqual(urlopen.call_count, 2) + + def test_one_path_returns_404(self): + responses = [_fake_response(200), _fake_response(404)] + + def side_effect(*args, **kwargs): + return responses.pop(0) + + with patch("cas_evals.registry_check.urlopen", side_effect=side_effect): + with self.assertRaises(RegistryCheckError) as ctx: + check_registry_urls( + "https://coding-autopilot-system.github.io/cas-contracts/registry", + ["index.json", "v0.1/manifest.json"], + ) + self.assertIn("v0.1/manifest.json", str(ctx.exception)) + self.assertIn("404", str(ctx.exception)) + + def test_network_unavailable_raises_distinguishable_error(self): + with patch("cas_evals.registry_check.urlopen", side_effect=URLError("no route to host")): + with self.assertRaises(RegistryCheckError) as ctx: + check_registry_urls( + "https://coding-autopilot-system.github.io/cas-contracts/registry", + ["index.json"], + ) + self.assertIn("unavailable", str(ctx.exception).lower()) + self.assertNotIn("404", str(ctx.exception)) + + def test_http_error_raises_registry_check_error(self): + error = HTTPError("https://example.com/index.json", 500, "Internal Server Error", {}, None) + with patch("cas_evals.registry_check.urlopen", side_effect=error): + with self.assertRaises(RegistryCheckError) as ctx: + check_registry_urls( + "https://coding-autopilot-system.github.io/cas-contracts/registry", + ["index.json"], + ) + self.assertIn("500", str(ctx.exception)) + + def test_main_exits_zero_on_success(self): + with patch("cas_evals.registry_check.check_registry_urls") as mocked: + mocked.return_value = [ + ("index.json", 200), + ("v0.1/manifest.json", 200), + ("v0.1/common.schema.json", 200), + ("v0.1/evaluation-result.schema.json", 200), + ] + buffer = io.StringIO() + with redirect_stdout(buffer): + exit_code = main([]) + self.assertEqual(exit_code, 0) + output = buffer.getvalue() + self.assertIn("index.json", output) + self.assertIn("200", output) + + def test_main_exits_nonzero_on_failure(self): + with patch( + "cas_evals.registry_check.check_registry_urls", + side_effect=RegistryCheckError("v0.1/manifest.json returned HTTP 404"), + ): + buffer = io.StringIO() + with redirect_stdout(buffer): + exit_code = main([]) + self.assertEqual(exit_code, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/vendor/cas-contracts/v0.1.0/common.schema.json b/vendor/cas-contracts/v0.1.0/common.schema.json index 0eec265..410c751 100644 --- a/vendor/cas-contracts/v0.1.0/common.schema.json +++ b/vendor/cas-contracts/v0.1.0/common.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://schemas.coding-autopilot.dev/v0.1/common.schema.json", + "$id": "https://coding-autopilot-system.github.io/cas-contracts/registry/v0.1/common.schema.json", "title": "CAS Common Definitions", "$defs": { "actor": { diff --git a/vendor/cas-contracts/v0.1.0/evaluation-result.schema.json b/vendor/cas-contracts/v0.1.0/evaluation-result.schema.json index 719f46a..9e753b2 100644 --- a/vendor/cas-contracts/v0.1.0/evaluation-result.schema.json +++ b/vendor/cas-contracts/v0.1.0/evaluation-result.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://schemas.coding-autopilot.dev/v0.1/evaluation-result.schema.json", + "$id": "https://coding-autopilot-system.github.io/cas-contracts/registry/v0.1/evaluation-result.schema.json", "title": "EvaluationResult", "type": "object", "allOf": [ diff --git a/vendor/cas-contracts/v0.1.0/provenance.json b/vendor/cas-contracts/v0.1.0/provenance.json index 7688280..54cf656 100644 --- a/vendor/cas-contracts/v0.1.0/provenance.json +++ b/vendor/cas-contracts/v0.1.0/provenance.json @@ -5,12 +5,12 @@ "schemas": { "common.schema.json": { "blobSha": "0eec265131a301f924a5ca7fb61718f5bdb14012", - "sha256": "c7ce72a6f5da8394e48f2421820588a8142546962e05152997bd1e6ced994928", + "sha256": "6c86075df043ec924f6b7aa004c6a694916a6c3d7a822dda4235f1ee8368f92a", "source": "https://raw.githubusercontent.com/Coding-Autopilot-System/cas-contracts/v0.1.0/schemas/v0.1/common.schema.json" }, "evaluation-result.schema.json": { "blobSha": "719f46a6ee9024fa4462094c3d0c21d838c20f17", - "sha256": "be6d3216c95cfa6d2ccda908ff089010765b1c70223a920bfe3cb70a0cd24df5", + "sha256": "ecb222d70b389e2d47de5bb1344dd4e5ae1043c0e76f8608aa99f42e491ce384", "source": "https://raw.githubusercontent.com/Coding-Autopilot-System/cas-contracts/v0.1.0/schemas/v0.1/evaluation-result.schema.json" } }