diff --git a/.env.example b/.env.example index 43e941e..29b495a 100644 --- a/.env.example +++ b/.env.example @@ -35,7 +35,7 @@ DATA_LAYER_PRELOAD_WATCHDOG_ENABLED=true BINANCE_WS_MAX_CONNS_PER_SOURCE=0 STREAM_STALE_SECONDS=180 STREAM_STRICT_FEED_HEALTH=false -BINANCE_SYMBOLS_FILE=/app/symbols.json +BINANCE_SYMBOLS_FILE=/app/data/cache/binance_usdm_symbols.json BINANCE_SPOT_SYMBOLS_FILE=/app/symbols_spot.json # ── Diagnostics ───────────────────────────────── diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8160d68..a030373 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,6 +102,8 @@ jobs: run: | docker run --rm data-layer:v0.1.0 sh -c ' test ! -e /opt/venv/bin/poetry + test "$(head -n 1 /opt/venv/bin/uvicorn)" = "#!/opt/venv/bin/python" + uvicorn --version python -c "from importlib.metadata import version; from packaging.version import Version; assert Version(version(\"msgpack\")) >= Version(\"1.2.1\"); assert Version(version(\"setuptools\")) >= Version(\"78.1.1\")" python -m pip freeze --local > /tmp/qdl-runtime-requirements.txt python -m pip install --disable-pip-version-check --no-cache-dir "pip-audit>=2.9,<3" diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index fc521d7..62ca066 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -2226,6 +2226,25 @@ Phase 8 is `COMPLETE` only when: Cleanup left zero Phase 8 containers, networks or volumes. V1 stayed HTTP 200 and its inspected topology was unchanged. See the [Phase 8 report](upgrade/evidence/PHASE8_RUST_REALTIME_CORE_REPORT.md). +- Post-merge closure on 2026-08-16 fixed the Python V1 runtime image without + changing the frozen Rust candidate or starting Phase 9. The builder now + creates `/opt/venv` at its final path, so console-script shebangs do not retain + the obsolete `/app/.venv` interpreter after the multi-stage copy. CI executes + the real `uvicorn` binary and checks its shebang. Frozen candidate verification + now verifies the signed manifest, bundled SBOM and immutable artifact metadata + without incorrectly comparing an old candidate to mutable files at repository + HEAD; newly generated bundles still verify current repository artifacts by + default. The CI Compose overlay also resets fixed container names, host ports + and bind volumes so local certification cannot replace production Redis or + mutate production data/log paths. The non-root runtime also owns an explicit + `/home/qdl` cache/config boundary for provider SDKs and plotting imports; + runtime code no longer retries against an absent home directory. PR #3 checks + passed and the Phase 8 head is contained in `dev`. +- Runtime venue discovery no longer mutates tracked source configuration. + `/app/symbols.json` is a read-only bootstrap seed; refreshed Binance USD-M + metadata is atomically replaced under writable `data/cache/`. A cache write + failure returns the valid provider result and raises one warning instead of + misclassifying it as a provider outage and retrying the REST request. ### Technical Debt / Decision Gate diff --git a/Dockerfile b/Dockerfile index 7d4b287..a1e2f61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,6 @@ FROM python:3.12-slim AS builder ENV PYTHONUNBUFFERED=1 ENV PYTHONDONTWRITEBYTECODE=1 ENV POETRY_VERSION=2.3.4 -ENV POETRY_VIRTUALENVS_IN_PROJECT=true ENV POETRY_NO_INTERACTION=1 WORKDIR /app @@ -12,9 +11,11 @@ RUN pip install --no-cache-dir poetry==$POETRY_VERSION COPY pyproject.toml poetry.lock ./ -RUN poetry config installer.max-workers 10 && \ - poetry install --no-root --only main --no-ansi && \ - /app/.venv/bin/python -m pip install --no-cache-dir --upgrade "setuptools>=78.1.1" +RUN python -m venv /opt/venv && \ + poetry config installer.max-workers 10 && \ + VIRTUAL_ENV=/opt/venv PATH="/opt/venv/bin:$PATH" \ + poetry install --no-root --only main --no-ansi && \ + /opt/venv/bin/python -m pip install --no-cache-dir --upgrade "setuptools>=78.1.1" FROM python:3.12-slim AS runtime @@ -24,14 +25,20 @@ ARG QDL_GID=10001 ENV PYTHONUNBUFFERED=1 ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONPATH=/app +ENV VIRTUAL_ENV=/opt/venv +ENV HOME=/home/qdl +ENV XDG_CACHE_HOME=/home/qdl/.cache +ENV MPLCONFIGDIR=/home/qdl/.cache/matplotlib ENV PATH=/opt/venv/bin:$PATH WORKDIR /app RUN groupadd --gid ${QDL_GID} qdl && \ - useradd --uid ${QDL_UID} --gid ${QDL_GID} --no-create-home --shell /usr/sbin/nologin qdl + useradd --uid ${QDL_UID} --gid ${QDL_GID} --create-home \ + --home-dir /home/qdl --shell /usr/sbin/nologin qdl && \ + install -d -o qdl -g qdl -m 0750 /home/qdl/.cache/matplotlib -COPY --from=builder --chown=qdl:qdl /app/.venv /opt/venv +COPY --from=builder --chown=qdl:qdl /opt/venv /opt/venv COPY --chown=qdl:qdl . /app diff --git a/app/config.py b/app/config.py index 2c94d6b..98a4cf9 100644 --- a/app/config.py +++ b/app/config.py @@ -33,7 +33,9 @@ BINANCE_WS_BATCH_SIZE = int(os.getenv("BINANCE_WS_BATCH_SIZE", 100)) BINANCE_WS_QUEUE_MAXSIZE = int(os.getenv("BINANCE_WS_QUEUE_MAXSIZE", 10000)) BINANCE_WS_MAX_CONNS_PER_SOURCE = int(os.getenv("BINANCE_WS_MAX_CONNS_PER_SOURCE", 0)) -BINANCE_SYMBOLS_FILE = os.getenv("BINANCE_SYMBOLS_FILE", "/app/symbols.json") +BINANCE_SYMBOLS_FILE = os.getenv( + "BINANCE_SYMBOLS_FILE", "/app/data/cache/binance_usdm_symbols.json" +) BINANCE_SPOT_SYMBOLS_FILE = os.getenv("BINANCE_SPOT_SYMBOLS_FILE", "/app/symbols_spot.json") STREAM_STALE_SECONDS = float(os.getenv("STREAM_STALE_SECONDS", "180")) STREAM_STRICT_FEED_HEALTH = os.getenv("STREAM_STRICT_FEED_HEALTH", "false").lower() in {"1", "true", "yes", "on"} diff --git a/app/stream/binance_ws.py b/app/stream/binance_ws.py index 4906176..e82f8d4 100644 --- a/app/stream/binance_ws.py +++ b/app/stream/binance_ws.py @@ -14,9 +14,12 @@ import threading import time import os +import json import logging import requests +import tempfile import websocket +from pathlib import Path from queue import Queue, Full, Empty from time import perf_counter @@ -41,15 +44,24 @@ def get_usdm_symbols( refresh: bool = False, ) -> list: """Load active symbols from venue metadata, with the last good cache as fallback.""" + explicit_file_path = file_path is not None file_path = file_path or BINANCE_SYMBOLS_FILE + cache_path = Path(file_path) cached_symbols = None - if os.path.exists(file_path): - with open(file_path, "r") as f: - import json - cached_symbols = json.load(f) - if not refresh: - logger.info(f"Loaded {len(cached_symbols)} symbols from {file_path}") - return cached_symbols + read_path = cache_path + seed_path = Path("/app/symbols.json") + if not read_path.exists() and not explicit_file_path and seed_path.exists(): + read_path = seed_path + if read_path.exists(): + try: + with read_path.open("r", encoding="utf-8") as f: + cached_symbols = json.load(f) + if not refresh: + logger.info("Loaded %s symbols from %s", len(cached_symbols), read_path) + return cached_symbols + except (OSError, TypeError, ValueError) as cache_error: + logger.warning("Ignoring invalid symbol cache %s: %s", read_path, cache_error) + cached_symbols = None url = "https://fapi.binance.com/fapi/v1/exchangeInfo" max_retries = 3 @@ -63,10 +75,33 @@ def get_usdm_symbols( if (contract_type is None or s["contractType"] == contract_type) and s["status"] == "TRADING" ] - with open(file_path, "w") as f: - import json - json.dump(symbols, f) - logger.info(f"Fetched and cached {len(symbols)} symbols from Binance API") + temporary_path = None + try: + cache_path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=cache_path.parent, + prefix=f".{cache_path.name}.", + suffix=".tmp", + delete=False, + ) as handle: + json.dump(symbols, handle) + temporary_path = Path(handle.name) + os.replace(temporary_path, cache_path) + except Exception as cache_error: + if temporary_path is not None: + try: + temporary_path.unlink(missing_ok=True) + except OSError: + pass + logger.warning( + "Fetched %s Binance symbols but could not persist cache %s: %s", + len(symbols), + cache_path, + cache_error, + ) + logger.info("Fetched %s symbols from Binance API", len(symbols)) return symbols except Exception as e: logger.warning(f"[get_usdm_symbols] Attempt {attempt + 1}/{max_retries} failed: {e}") diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index d51bf06..9cbb8ae 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -1,4 +1,25 @@ # CI must not depend on or mutate host-wide production networks. +services: + redis_marketdata: + container_name: !reset null + + data_layer: + container_name: !reset null + ports: !reset [] + volumes: !reset [] + + alpha_service: + container_name: !reset null + volumes: !reset [] + + test_runner: + container_name: !reset null + volumes: !reset [] + + data_source_checker: + container_name: !reset null + volumes: !reset [] + networks: bobby_network: external: false diff --git a/qdl/certification/release.py b/qdl/certification/release.py index bab0760..280c176 100644 --- a/qdl/certification/release.py +++ b/qdl/certification/release.py @@ -171,18 +171,31 @@ def verify_release_bundle( output_dir: Path, *, verification_key: Path | None = None, + verify_repository_artifacts: bool = True, ) -> dict[str, Any]: manifest_path = output_dir / "release-manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if manifest.get("schema") != "qdl.release-manifest.v1": raise ValueError("unsupported release manifest schema") for artifact in manifest["artifacts"]: + expected_hash = str(artifact.get("sha256") or "") + expected_size = artifact.get("size_bytes") + if not re.fullmatch(r"[0-9a-f]{64}", expected_hash): + raise ValueError(f"invalid release artifact hash: {artifact.get('path')}") + if not isinstance(expected_size, int) or expected_size < 0: + raise ValueError(f"invalid release artifact size: {artifact.get('path')}") + if artifact["path"] != "sbom.spdx.json" and not verify_repository_artifacts: + continue path = ( output_dir / artifact["path"] if artifact["path"] == "sbom.spdx.json" else repo / artifact["path"] ) - if not path.is_file() or sha256_file(path) != artifact["sha256"]: + if ( + not path.is_file() + or path.stat().st_size != expected_size + or sha256_file(path) != expected_hash + ): raise ValueError(f"release artifact checksum mismatch: {artifact['path']}") if verification_key is not None: subprocess.run( diff --git a/tests/test_fund_phase6_release.py b/tests/test_fund_phase6_release.py index 1a8042a..f4eb86e 100644 --- a/tests/test_fund_phase6_release.py +++ b/tests/test_fund_phase6_release.py @@ -15,6 +15,24 @@ class ReleaseBundleTests(unittest.TestCase): def test_runtime_image_is_non_root_and_trivy_waiver_is_narrow(self): dockerfile = (ROOT / "Dockerfile").read_text(encoding="utf-8") self.assertIn("USER qdl:qdl", dockerfile) + self.assertIn("python -m venv /opt/venv", dockerfile) + self.assertIn("ENV HOME=/home/qdl", dockerfile) + self.assertIn("ENV XDG_CACHE_HOME=/home/qdl/.cache", dockerfile) + self.assertIn("ENV MPLCONFIGDIR=/home/qdl/.cache/matplotlib", dockerfile) + self.assertIn("--create-home", dockerfile) + self.assertNotIn("--no-create-home", dockerfile) + config = (ROOT / "app/config.py").read_text(encoding="utf-8") + self.assertIn("/app/data/cache/binance_usdm_symbols.json", config) + env_example = (ROOT / ".env.example").read_text(encoding="utf-8") + self.assertIn( + "BINANCE_SYMBOLS_FILE=/app/data/cache/binance_usdm_symbols.json", + env_example, + ) + self.assertIn( + "COPY --from=builder --chown=qdl:qdl /opt/venv /opt/venv", + dockerfile, + ) + self.assertNotIn("COPY --from=builder --chown=qdl:qdl /app/.venv", dockerfile) preparation = (ROOT / "scripts/prepare_nonroot_runtime.sh").read_text( encoding="utf-8" ) @@ -38,6 +56,15 @@ def test_runtime_image_is_non_root_and_trivy_waiver_is_narrow(self): self.assertIn("pip-audit --cache-dir /tmp/qdl-pip-audit-cache", workflow) self.assertIn('python-version: "3.12"', workflow) self.assertIn("python -m scripts.phase6_release_bundle", workflow) + self.assertIn( + 'test "$(head -n 1 /opt/venv/bin/uvicorn)" = ' + '"#!/opt/venv/bin/python"', + workflow, + ) + ci_compose = (ROOT / "docker-compose.ci.yml").read_text(encoding="utf-8") + self.assertGreaterEqual(ci_compose.count("container_name: !reset null"), 5) + self.assertIn("ports: !reset []", ci_compose) + self.assertGreaterEqual(ci_compose.count("volumes: !reset []"), 4) ignored = { line.strip() for line in (ROOT / ".trivyignore").read_text(encoding="utf-8").splitlines() diff --git a/tests/test_fund_phase83_release.py b/tests/test_fund_phase83_release.py index 86e64de..c280b40 100644 --- a/tests/test_fund_phase83_release.py +++ b/tests/test_fund_phase83_release.py @@ -31,11 +31,14 @@ def test_frozen_candidate_evidence_passes_and_signatures_verify(self): self.assertTrue(capacity["thresholds_pass"]) self.assertEqual(capacity["replay"]["record_mismatches"], 0) self.assertFalse((release_dir / "private.pem").exists()) - verify_release_bundle( + manifest = verify_release_bundle( ROOT, release_dir, verification_key=release_dir / "attestation-public.pem", + verify_repository_artifacts=False, ) + self.assertEqual(manifest["git_sha"], "053ec76") + self.assertEqual(manifest["authority"], "SHADOW") def test_release_capacity_uses_authentic_multi_venue_inputs(self): source = (ROOT / "scripts/phase83_release_capacity.py").read_text() diff --git a/tests/test_phase2_demand_reliability.py b/tests/test_phase2_demand_reliability.py index 1fc24ec..89a980a 100644 --- a/tests/test_phase2_demand_reliability.py +++ b/tests/test_phase2_demand_reliability.py @@ -131,6 +131,31 @@ def test_perpetual_filter_remains_available_for_legacy_callers(self, get): ) self.assertEqual(symbols, ["BTCUSDT"]) + @patch("app.stream.binance_ws.os.replace", side_effect=PermissionError("readonly")) + @patch("app.stream.binance_ws.requests.get") + def test_cache_write_failure_does_not_retry_or_drop_provider_result( + self, get, _replace + ): + response = get.return_value + response.raise_for_status.return_value = None + response.json.return_value = { + "symbols": [ + { + "symbol": "BTCUSDT", + "contractType": "PERPETUAL", + "status": "TRADING", + } + ] + } + with TemporaryDirectory() as directory: + symbols = get_usdm_symbols( + str(Path(directory) / "symbols.json"), + contract_type=None, + refresh=True, + ) + self.assertEqual(symbols, ["BTCUSDT"]) + get.assert_called_once() + class TopupCoordinatorTests(unittest.IsolatedAsyncioTestCase): async def test_concurrent_topups_call_provider_once(self): diff --git a/upgrade/evidence/PHASE8_RUST_REALTIME_CORE_REPORT.md b/upgrade/evidence/PHASE8_RUST_REALTIME_CORE_REPORT.md index 9ad63f1..1005cb3 100644 --- a/upgrade/evidence/PHASE8_RUST_REALTIME_CORE_REPORT.md +++ b/upgrade/evidence/PHASE8_RUST_REALTIME_CORE_REPORT.md @@ -88,3 +88,24 @@ shadow evidence before any canary. OKX SBE, Deribit live, BBO, L2/book and BAR remain separate capability certifications. Regional failure domains, production workload identity, external secret rotation and registry admission are not claimed by this same-host shadow certification. + +## Post-Merge Runtime Closure + +On 2026-08-16, final runtime inspection found that the Python V1 image copied a +Poetry environment from `/app/.venv` to `/opt/venv`, leaving console-script +shebangs pointed at the builder path. The image now builds the environment at +`/opt/venv` directly, and CI invokes the actual `uvicorn` executable. This is a +V1 packaging correction only: the signed Phase 8 Rust candidate, its source +revision, shadow authority and evidence remain unchanged. Verification of that +historical candidate is explicitly separated from verification of artifacts at +the current repository HEAD. The CI Compose overlay now removes fixed container +names, host ports and host bind volumes, preventing a local CI rehearsal from +joining or replacing production runtime resources. The non-root Python runtime +now has an owned `/home/qdl` cache/config boundary so provider libraries cannot +enter a permission-error retry loop. No Phase 9 canary or authority transition +occurred. + +The final V1 audit also separated tracked universe seed configuration from the +runtime Binance metadata cache. Refreshes now use atomic replacement under +`data/cache/`; a cache persistence error cannot trigger redundant provider +retries or discard an otherwise valid exchange-info response.