From fed7be6464f768a6bceebbf8ee94fda6c79fd557 Mon Sep 17 00:00:00 2001 From: Matthew Wesney Date: Tue, 28 Jul 2026 21:56:12 -0400 Subject: [PATCH] Add deterministic recipe parser fuzz gate --- backend/cortex_backend/execution/recipes.py | 8 +- ...bility-tiered-agentic-execution-harness.md | 5 +- docs/adr/0001-phase2-evidence.md | 30 +- tests/test_phase2_recipe_contract.py | 12 + tests/test_phase2_recipe_fuzz.py | 25 ++ tools/execution_spikes/README.md | 24 +- tools/execution_spikes/recipe_parser_fuzz.py | 263 ++++++++++++++++++ 7 files changed, 355 insertions(+), 12 deletions(-) create mode 100644 tests/test_phase2_recipe_fuzz.py create mode 100644 tools/execution_spikes/recipe_parser_fuzz.py diff --git a/backend/cortex_backend/execution/recipes.py b/backend/cortex_backend/execution/recipes.py index 1952d01..aab541a 100644 --- a/backend/cortex_backend/execution/recipes.py +++ b/backend/cortex_backend/execution/recipes.py @@ -70,6 +70,12 @@ def _bounded_decimal(value: Decimal) -> Decimal: return value +def _bounded_optional_decimal(value: Decimal | None) -> Decimal | None: + if value is None: + return None + return _bounded_decimal(value) + + def _coerce_decimal(value: Any) -> Decimal: if isinstance(value, bool) or isinstance(value, float): raise ValueError("decimal must be an integer, string, or Decimal") @@ -217,7 +223,7 @@ class CheckPlan(_StrictModel): ) _left = field_validator("left")(_bounded_decimal) _right = field_validator("right")(_bounded_decimal) - _tolerance = field_validator("tolerance")(_bounded_decimal) + _tolerance = field_validator("tolerance")(_bounded_optional_decimal) @model_validator(mode="after") def _validate_tolerance(self) -> "CheckPlan": diff --git a/docs/adr/0001-capability-tiered-agentic-execution-harness.md b/docs/adr/0001-capability-tiered-agentic-execution-harness.md index 2072633..c1d3a3e 100644 --- a/docs/adr/0001-capability-tiered-agentic-execution-harness.md +++ b/docs/adr/0001-capability-tiered-agentic-execution-harness.md @@ -968,8 +968,9 @@ lifecycle health must pass before any provider is enabled. The 2026-07-28 packaged qualification evidence closes the hostile decoder corpus, signed provenance, broker identity, and in-flight cancellation portions of this -gate. Parser fuzzing, artifact security review, resource/watchdog accounting, -external review, and production lifecycle health remain open. +gate. The 2026-07-28 parser qualification also closes the deterministic parser- +fuzzing portion. Artifact security review, resource/watchdog accounting, external +review, and production lifecycle health remain open. ### Phase 3 — `scratch.auto.v1` arbitrary WebAssembly code diff --git a/docs/adr/0001-phase2-evidence.md b/docs/adr/0001-phase2-evidence.md index ec57dbc..306e527 100644 --- a/docs/adr/0001-phase2-evidence.md +++ b/docs/adr/0001-phase2-evidence.md @@ -1,7 +1,7 @@ # ADR-0001 Phase 2 evidence log - **Phase:** 2 — signed image recipes and calculator/check primitives -- **Status:** Typed contract, signed-manifest verification, native broker transport, authenticated worker loop, signed bundle installation, trusted artifact boundary, signed worker launch/broker qualification, and packaged transform/hostile-decoder/cancellation qualification complete; parser fuzzing, resource/watchdog accounting, artifact security review, external review, and lifecycle release gates remain open +- **Status:** Typed contract, deterministic parser-fuzz qualification, signed-manifest verification, native broker transport, authenticated worker loop, signed bundle installation, trusted artifact boundary, signed worker launch/broker qualification, and packaged transform/hostile-decoder/cancellation qualification complete; resource/watchdog accounting, artifact security review, external review, and lifecycle release gates remain open - **Scope:** Provider-independent contracts plus a qualification-only fixed-function core - **Source decision:** [Capability-tiered agentic execution harness](0001-capability-tiered-agentic-execution-harness.md) - **Contract ADR:** [Phase 2 typed recipe and primitive contract](0001-phase2-recipe-contract.md) @@ -12,6 +12,7 @@ | Deliverable | Status | Evidence | | --- | --- | --- | | Versioned image transform schema | **Complete (contract only)** | `artifact.transform.v1` allows only bounded grayscale, contrast, brightness, crop, resize, and rotate steps with PNG/JPEG/WebP output. | +| Deterministic parser-fuzz qualification | **Complete (qualification-only)** | `recipe_parser_fuzz.py --iterations 2000 --seed 20260728 --json --strict` generated bounded mutations across image, calculator, and check parsers: 158 accepted, 1,842 rejected, zero unexpected exceptions; error text remained stable and redacted. | | Opaque artifact binding | **Complete (validation only)** | Artifact IDs are bounded opaque identifiers; paths, source text, plugin names, and model-selected output names are rejected or absent. | | Calculator/check primitives | **Complete (trusted pure helpers)** | Decimal-only calculator operations and explicit comparisons are deterministic, bounded, and have no I/O or code execution surface. | | Canonical plan identity | **Complete** | Validated plans expose stable canonical JSON and SHA-256 digests for future idempotency/signature binding. | @@ -26,7 +27,7 @@ | Fixed-function image provider core | **Complete (qualification-only)** | `RecipeImageProvider` validates allowlisted PNG/JPEG/WebP bytes, verifies/loads one frame with Pillow bomb/resource limits, applies only parsed steps, strips metadata, revalidates encoded output, checks cancellation, and remains disabled until external sandbox health passes. | | Windows recipe sandbox qualification harness | **Complete (signed launch/broker/hostile/cancellation)** | `recipe_worker_e2e_qualification.py` signs a disposable package with an in-memory key, installs/verifies one immutable generation, binds the live AppContainer identity to the broker, and exercises the fixed PNG transform, truncated-PNG decoder rejection, active-SVG decoder rejection, and in-flight cancellation corpus. The native broker uses bounded availability polling before reads so the cancellation reader remains live while the packaged provider transforms. | | Suspended native launcher/resource policy | **Complete (factory + binder + ACL cleanup + qualification evidence)** | `NativeWin32ProcessFactory` grants only inherited read/execute access to the fresh AppContainer SID on the verified package root, applies and verifies Job Object policy before resume, and removes the per-launch ACE during cleanup. `NativeBrokerIdentityBinder` pins the live server to the worker PID/AppContainer SID and launcher cleanup closes it on failure. | -| OS sandbox provider and provider-produced image outputs | **Blocked / release gate** | Signed installation, provenance, AppContainer/job identity, broker handshake, `prepare`, `input_chunk`, `input_complete`, `collect_output`, hostile decoder rejection, and in-flight cancellation acknowledgement are live-qualified inside the packaged worker. The remaining gate is parser fuzzing, artifact security review, resource/watchdog accounting, external review, and production lifecycle wiring. | +| OS sandbox provider and provider-produced image outputs | **Blocked / release gate** | Signed installation, provenance, AppContainer/job identity, broker handshake, `prepare`, `input_chunk`, `input_complete`, `collect_output`, hostile decoder rejection, and in-flight cancellation acknowledgement are live-qualified inside the packaged worker. The remaining gate is artifact security review, resource/watchdog accounting, external review, and production lifecycle wiring. | ## Security invariants @@ -99,6 +100,9 @@ 27. Hostile decoder bytes are rejected inside the signed worker, and cancellation is sent only after `input_complete` over the authenticated broker; a missing or ambiguous terminal response remains a blocked qualification result. +28. Typed parser fuzzing uses a fixed seed, bounded iteration count, bounded payload + depth, and stable redacted error categories; an unexpected exception or budget + violation is a blocked qualification result. ## Re-run target @@ -117,6 +121,7 @@ python -m pytest tests/test_recipe_sandbox_qualification.py -q python tools/execution_spikes/native_launcher_qualification.py python tools/execution_spikes/recipe_sandbox_qualification.py --json --strict python tools/execution_spikes/recipe_worker_e2e_qualification.py --json --strict +python tools/execution_spikes/recipe_parser_fuzz.py --json --strict python -m compileall -q backend\cortex_backend\execution tests python -m pytest -q python tools/generate_contracts.py @@ -150,9 +155,9 @@ policy and identity binding, authenticated broker handshake, `prepare`, `PeekNamedPipe` with a 5 ms bounded wait before synchronous reads, preventing the worker's cancellation reader from starving the provider transform. Bounded cleanup closed the broker, binder, worker, and profile; no `recipe_worker.exe` process -remained. This closes the packaged provider-transform qualification gate but does -not close the remaining watchdog/hostile-decoder, external-review, or lifecycle -release gates. +remained. At that stage this closed only the packaged provider-transform +qualification gate; hostile-decoder, watchdog, external-review, and lifecycle +release evidence was still pending. **Packaged hostile/cancellation qualification result (2026-07-28):** The full `recipe_worker_e2e_qualification.py --json --timeout-seconds 10 --strict` run @@ -164,5 +169,16 @@ disposable worker; the run reported no worker process or binding cleanup residue. The optional `--case cancellation` path reproduces the cancellation gate in isolation. These results close the packaged hostile-decoder and cancellation qualification evidence, but do not enable the provider or close -parser fuzzing, resource/watchdog accounting, artifact security review, -external review, or production lifecycle health gates. +resource/watchdog accounting, artifact security review, external review, or +production lifecycle health gates. + +**Parser fuzz qualification result (2026-07-28):** The deterministic +`recipe_parser_fuzz.py --iterations 2000 --seed 20260728 --json --strict` run +completed with 158 accepted payloads, 1,842 stable validation rejections, and +zero unexpected exceptions. It exercised bounded mutations across all three +typed parsers, including unknown fields, malformed operations, non-mapping +values, oversized payloads, control/unicode text, and invalid optional values. +The corpus exposed and the parser fixed an uncaught `None` tolerance validator +failure for `check.v1`/`is_close`; the regression suite now locks that behavior +to `invalid_check`. Resource/watchdog accounting, artifact security review, +external review, and production lifecycle health remain open. diff --git a/tests/test_phase2_recipe_contract.py b/tests/test_phase2_recipe_contract.py index bfdd12d..a7b0a93 100644 --- a/tests/test_phase2_recipe_contract.py +++ b/tests/test_phase2_recipe_contract.py @@ -180,3 +180,15 @@ def test_check_tolerance_is_only_available_for_is_close(): "tolerance": 0.1, } ) + + with pytest.raises(RecipeValidationError) as missing_tolerance: + parse_check( + { + "schema_version": "check.v1", + "operation": "is_close", + "left": "1", + "right": "1", + "tolerance": None, + } + ) + assert missing_tolerance.value.code == "invalid_check" diff --git a/tests/test_phase2_recipe_fuzz.py b/tests/test_phase2_recipe_fuzz.py new file mode 100644 index 0000000..2811beb --- /dev/null +++ b/tests/test_phase2_recipe_fuzz.py @@ -0,0 +1,25 @@ +"""Deterministic parser-fuzz qualification stays bounded and reproducible.""" + +from __future__ import annotations + +import pytest + +from tools.execution_spikes.recipe_parser_fuzz import run_fuzz + + +def test_recipe_parser_fuzz_corpus_is_reproducible_and_fail_closed(): + first = run_fuzz(iterations=900, seed=20260728) + second = run_fuzz(iterations=900, seed=20260728) + + assert first == second + assert first["status"] == "passed" + assert first["accepted"] > 0 + assert first["rejected"] > 0 + assert first["unexpected"] == 0 + assert first["failures"] == [] + + +@pytest.mark.parametrize("iterations", [0, 10_001]) +def test_recipe_parser_fuzz_budget_is_bounded(iterations): + with pytest.raises(ValueError, match="iterations"): + run_fuzz(iterations=iterations) diff --git a/tools/execution_spikes/README.md b/tools/execution_spikes/README.md index a3b2d9d..2e5c0c8 100644 --- a/tools/execution_spikes/README.md +++ b/tools/execution_spikes/README.md @@ -89,8 +89,28 @@ handshake, normal `collect_output`, hostile decoder rejection for truncated PNG and active SVG bytes, and an in-flight `cancel_ack` after `input_complete`. The native read path polls `PeekNamedPipe` with a bounded 5 ms wait before `ReadFile`, keeping the worker's cancellation reader live without starving the provider -transform. Parser fuzzing, resource/watchdog accounting, artifact security review, -external review, and production-lifecycle release gates remain open. +transform. Resource/watchdog accounting, artifact security review, external review, +and production-lifecycle release gates remain open; parser-fuzz evidence is +qualified below. + +## Run the typed parser fuzz qualification + +The parser probe is deterministic and bounded; it never executes a recipe or +accepts model/user input: + +```powershell +python tools/execution_spikes/recipe_parser_fuzz.py --json +python tools/execution_spikes/recipe_parser_fuzz.py --iterations 2000 --seed 20260728 --json --strict +``` + +The fixed corpus mutates image, calculator, and check payloads, including +unknown fields, malformed operations, non-mapping values, oversized payloads, +control/unicode text, and invalid optional values. A green result requires only +typed models or stable redacted `RecipeValidationError` categories; unexpected +exceptions, unbounded budgets, or canonicalization failures return exit code 2. +The current evidence is 158 accepted payloads, 1,842 rejections, and zero +unexpected exceptions. This closes parser-fuzz evidence only; resource/watchdog, +artifact-security, external-review, and production-lifecycle gates remain open. ## What the probes prove diff --git a/tools/execution_spikes/recipe_parser_fuzz.py b/tools/execution_spikes/recipe_parser_fuzz.py new file mode 100644 index 0000000..303ee86 --- /dev/null +++ b/tools/execution_spikes/recipe_parser_fuzz.py @@ -0,0 +1,263 @@ +"""Deterministic, bounded fuzz qualification for Phase 2 typed recipe parsers. + +This probe never executes a recipe or consumes user/model input. It generates a +small JSON-like corpus in memory, mutates valid image/calculator/check payloads, +and requires every rejection to remain a stable, redacted ``RecipeValidationError``. +The fixed seed and iteration budget make failures reproducible in CI and during +release review. +""" + +from __future__ import annotations + +import argparse +from copy import deepcopy +from dataclasses import dataclass +from hashlib import sha256 +import json +from pathlib import Path +import random +import sys +from typing import Any, Callable + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "backend")) + +from cortex_backend.execution.recipes import ( # noqa: E402 + CalculatorPlan, + CheckPlan, + ImageTransformPlan, + MAX_IMAGE_DIMENSION, + MAX_IMAGE_STEPS, + MAX_RECIPE_PAYLOAD_BYTES, + RecipeValidationError, + parse_calculator, + parse_check, + parse_image_transform, +) + + +DEFAULT_ITERATIONS = 2_000 +MAX_ITERATIONS = 10_000 +DEFAULT_SEED = 20_260_728 +_GENERATOR_DEPTH = 3 +_REDACTION_MARKER = "CORTEX_FUZZ_SECRET" +_SAFE_ERROR_TEXT = "The requested typed operation is invalid." +_ALLOWED_ERROR_CODES = frozenset( + {"invalid_image_recipe", "invalid_calculation", "invalid_check", "payload_too_large"} +) +_TEXT_ALPHABET = "abcXYZ0123 _-./\\\n\r\t\x00\u2603" + + +@dataclass(frozen=True, slots=True) +class _ParserCase: + name: str + parse: Callable[[Any], Any] + expected_type: type[Any] + + +_PARSERS = ( + _ParserCase("image", parse_image_transform, ImageTransformPlan), + _ParserCase("calculator", parse_calculator, CalculatorPlan), + _ParserCase("check", parse_check, CheckPlan), +) + + +def _random_text(rng: random.Random, *, maximum: int = 32) -> str: + length = rng.randrange(maximum + 1) + return "".join(rng.choice(_TEXT_ALPHABET) for _ in range(length)) + + +def _random_json_value(rng: random.Random, *, depth: int = 0) -> Any: + choices = [None, True, False, -1, 0, 1, 1.5, "", _random_text(rng)] + if depth < _GENERATOR_DEPTH: + choices.extend( + [ + [_random_json_value(rng, depth=depth + 1) for _ in range(rng.randrange(3))], + { + _random_text(rng, maximum=8): _random_json_value(rng, depth=depth + 1) + for _ in range(rng.randrange(3)) + }, + ] + ) + return deepcopy(rng.choice(choices)) + + +def _valid_image(rng: random.Random) -> dict[str, Any]: + steps: list[dict[str, Any]] = [] + operations = ("grayscale", "contrast", "brightness", "crop", "resize", "rotate") + for _ in range(rng.randint(1, MAX_IMAGE_STEPS)): + operation = rng.choice(operations) + if operation == "grayscale": + steps.append({"op": operation}) + elif operation in {"contrast", "brightness"}: + steps.append({"op": operation, "factor": rng.choice(["0", "0.5", "1.2", "4"])}) + elif operation == "crop": + steps.append({"op": operation, "x": 0, "y": 0, "width": 16, "height": 12}) + elif operation == "resize": + steps.append({"op": operation, "width": rng.choice([1, 16, 1024, MAX_IMAGE_DIMENSION]), "height": 12}) + else: + steps.append({"op": operation, "degrees": rng.choice([90, 180, 270])}) + return { + "schema_version": "artifact.transform.v1", + "input_artifact_id": "artifact_fuzz", + "steps": steps, + "output_format": rng.choice(["png", "jpeg", "webp"]), + } + + +def _valid_calculator(rng: random.Random) -> dict[str, Any]: + operation = rng.choice(["add", "subtract", "multiply", "divide", "min", "max"]) + operands = [str(rng.randint(-1_000_000, 1_000_000)) for _ in range(rng.randint(2, 16))] + if operation == "divide": + operands[1] = rng.choice(["-3", "1", "2", "7"]) + return {"schema_version": "calculation.v1", "operation": operation, "operands": operands} + + +def _valid_check(rng: random.Random) -> dict[str, Any]: + operation = rng.choice( + ["equals", "not_equals", "less_than", "less_or_equal", "greater_than", "greater_or_equal", "is_close"] + ) + payload: dict[str, Any] = { + "schema_version": "check.v1", + "operation": operation, + "left": str(rng.randint(-1_000_000, 1_000_000)), + "right": str(rng.randint(-1_000_000, 1_000_000)), + } + if operation == "is_close": + payload["tolerance"] = rng.choice(["0.0001", "0.01", "1"]) + return payload + + +def _base_payload(case: _ParserCase, rng: random.Random) -> dict[str, Any]: + if case.name == "image": + return _valid_image(rng) + if case.name == "calculator": + return _valid_calculator(rng) + return _valid_check(rng) + + +def _mutate(case: _ParserCase, payload: dict[str, Any], rng: random.Random, index: int) -> Any: + """Return one bounded mutation; every 17th case is intentionally valid.""" + + if index % 17 == 0: + return payload + if index % 31 == 0: + return {"oversized": "x" * (MAX_RECIPE_PAYLOAD_BYTES + 1)} + if index % 29 == 0: + return [payload, _random_json_value(rng)] + + candidate = deepcopy(payload) + mutation = rng.randrange(8) + if mutation == 0: + candidate["unknown_field"] = _REDACTION_MARKER + elif mutation == 1: + candidate["schema_version"] = _random_text(rng, maximum=16) + elif mutation == 2: + candidate.pop(rng.choice(tuple(candidate)), None) + elif mutation == 3: + candidate[rng.choice(tuple(candidate))] = _random_json_value(rng) + elif mutation == 4: + candidate["operation"] = _random_text(rng, maximum=12) + elif mutation == 5: + field = "steps" if case.name == "image" else "operands" if case.name == "calculator" else "left" + candidate[field] = _random_json_value(rng) + elif mutation == 6: + candidate["input_artifact_id" if case.name == "image" else "operation"] = "..\\private\\secret.txt" + else: + candidate["nested"] = {"values": [_random_json_value(rng) for _ in range(3)]} + return candidate + + +def _payload_fingerprint(payload: Any) -> str: + try: + encoded = json.dumps( + payload, + ensure_ascii=True, + sort_keys=True, + default=lambda value: "", + ) + except (TypeError, ValueError, OverflowError): + encoded = type(payload).__name__ + return sha256(encoded.encode("utf-8", "replace")).hexdigest()[:16] + + +def _run_case(case: _ParserCase, payload: Any) -> str | None: + try: + result = case.parse(payload) + except RecipeValidationError as error: + if error.code not in _ALLOWED_ERROR_CODES: + return f"unexpected_error_code:{case.name}:{error.code}" + if str(error) != _SAFE_ERROR_TEXT or _REDACTION_MARKER in str(error): + return f"error_redaction_failed:{case.name}" + return None + except Exception as error: # pragma: no cover - qualification failure path. + return f"unexpected_exception:{case.name}:{type(error).__name__}" + if not isinstance(result, case.expected_type): + return f"unexpected_result_type:{case.name}" + try: + canonical = result.canonical_json() + digest = result.digest() + if not canonical.isascii() or len(digest) != 64: + return f"canonical_identity_invalid:{case.name}" + except Exception as error: # pragma: no cover - qualification failure path. + return f"canonicalization_failed:{case.name}:{type(error).__name__}" + return None + + +def run_fuzz(*, iterations: int = DEFAULT_ITERATIONS, seed: int = DEFAULT_SEED) -> dict[str, Any]: + if type(iterations) is not int or not 1 <= iterations <= MAX_ITERATIONS: + raise ValueError("iterations must be between 1 and 10000") + if type(seed) is not int: + raise ValueError("seed must be an integer") + rng = random.Random(seed) + accepted = 0 + rejected = 0 + failures: list[dict[str, str]] = [] + for index in range(iterations): + case = _PARSERS[index % len(_PARSERS)] + payload = _mutate(case, _base_payload(case, rng), rng, index) + failure = _run_case(case, payload) + if failure is None: + try: + case.parse(payload) + except RecipeValidationError: + rejected += 1 + else: + accepted += 1 + elif len(failures) < 8: + failures.append( + { + "case": case.name, + "index": str(index), + "failure": failure, + "payload_fingerprint": _payload_fingerprint(payload), + } + ) + return { + "status": "passed" if not failures else "blocked", + "seed": seed, + "iterations": iterations, + "accepted": accepted, + "rejected": rejected, + "unexpected": len(failures), + "failures": failures, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--iterations", type=int, default=DEFAULT_ITERATIONS) + parser.add_argument("--seed", type=int, default=DEFAULT_SEED) + parser.add_argument("--json", action="store_true") + parser.add_argument("--strict", action="store_true") + args = parser.parse_args(argv) + try: + result = run_fuzz(iterations=args.iterations, seed=args.seed) + except ValueError as error: + parser.error(str(error)) + print(json.dumps(result, sort_keys=True, separators=(",", ":") if args.json else None)) + return 2 if args.strict and result["status"] != "passed" else 0 + + +if __name__ == "__main__": + raise SystemExit(main())