Add baseline Grafana observability and stage timing telemetry

.github/scripts/modal-sync-secrets.sh

@@ -1,28 +1,50 @@
 #!/bin/bash
 # Sync secrets from GitHub to Modal environment
 # Usage: ./modal-sync-secrets.sh <modal-environment> <gh-environment>
-# Required env vars: LOGFIRE_TOKEN, GCP_CREDENTIALS_JSON (optional)
+# Required env vars:
+#   LOGFIRE_TOKEN
+#   GCP_CREDENTIALS_JSON (optional)
+#   OBSERVABILITY_ENABLED (optional, defaults to false)
+#   OBSERVABILITY_SHADOW_MODE (optional, defaults to true)
+#   OBSERVABILITY_OTLP_ENDPOINT (required when OBSERVABILITY_ENABLED=true)
+#   OBSERVABILITY_OTLP_HEADERS (required when OBSERVABILITY_ENABLED=true)
 
 set -euo pipefail
 
 MODAL_ENV="${1:?Modal environment required}"
 GH_ENV="${2:?GitHub environment required}"
+OBSERVABILITY_ENABLED="${OBSERVABILITY_ENABLED:-false}"
+OBSERVABILITY_SHADOW_MODE="${OBSERVABILITY_SHADOW_MODE:-true}"
 
 echo "Syncing secrets to Modal environment: $MODAL_ENV"
 
+if [ "$OBSERVABILITY_ENABLED" = "true" ]; then
+  : "${OBSERVABILITY_OTLP_ENDPOINT:?OBSERVABILITY_OTLP_ENDPOINT is required when observability is enabled}"
+  : "${OBSERVABILITY_OTLP_HEADERS:?OBSERVABILITY_OTLP_HEADERS is required when observability is enabled}"
+fi
+
 # Sync Logfire secret
 uv run modal secret create policyengine-logfire \
   "LOGFIRE_TOKEN=${LOGFIRE_TOKEN:-}" \
   "LOGFIRE_ENVIRONMENT=$GH_ENV" \
   --env="$MODAL_ENV" \
-  --force || true
+  --force
 
 # Sync GCP credentials if provided
 if [ -n "${GCP_CREDENTIALS_JSON:-}" ]; then
   uv run modal secret create gcp-credentials \
     "GOOGLE_APPLICATION_CREDENTIALS_JSON=$GCP_CREDENTIALS_JSON" \
     --env="$MODAL_ENV" \
-    --force || true
+    --force
 fi
 
+uv run modal secret create policyengine-observability \
+  "OBSERVABILITY_ENABLED=$OBSERVABILITY_ENABLED" \
+  "OBSERVABILITY_SHADOW_MODE=$OBSERVABILITY_SHADOW_MODE" \
+  "OBSERVABILITY_ENVIRONMENT=$GH_ENV" \
+  "OBSERVABILITY_OTLP_ENDPOINT=${OBSERVABILITY_OTLP_ENDPOINT:-}" \
+  "OBSERVABILITY_OTLP_HEADERS=${OBSERVABILITY_OTLP_HEADERS:-}" \
+  --env="$MODAL_ENV" \
+  --force
+
 echo "Modal secrets synced"

.github/workflows/modal-deploy.reusable.yml

@@ -55,6 +55,10 @@ jobs:
         MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
         LOGFIRE_TOKEN: ${{ secrets.LOGFIRE_TOKEN }}
         GCP_CREDENTIALS_JSON: ${{ secrets.GCP_CREDENTIALS_JSON }}
+        OBSERVABILITY_ENABLED: ${{ vars.OBSERVABILITY_ENABLED }}
+        OBSERVABILITY_SHADOW_MODE: ${{ vars.OBSERVABILITY_SHADOW_MODE }}
+        OBSERVABILITY_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }}
+        OBSERVABILITY_OTLP_HEADERS: ${{ secrets.OBSERVABILITY_OTLP_HEADERS }}
       run: ../../.github/scripts/modal-sync-secrets.sh "${{ inputs.modal_environment }}" "${{ inputs.environment }}"
 
     - name: Deploy simulation API to Modal

changelog_entry.yaml

@@ -1,4 +1,4 @@
 - bump: patch
   changes:
     changed:
-      - Bumped policyengine-core minimum version to 3.23.5 for pandas 3.0 compatibility
+      - Added baseline Grafana-compatible OTLP observability and stage timing telemetry for the simulation gateway and worker.

libs/policyengine-fastapi/pyproject.toml

@@ -11,6 +11,7 @@ dependencies = [
     "fastapi[standard] >=0.115.8,<0.116.0",
     "pyjwt >=2.10.1,<3.0.0",
     "opentelemetry-sdk >=1.30.0,<2.0.0",
+    "opentelemetry-exporter-otlp-proto-http >=1.30.0,<2.0.0",
     "sqlmodel >=0.0.22,<0.0.23",
     "python-json-logger >=3.2.1,<4.0.0",
     "opentelemetry-instrumentation-logging >=0.51b0,<0.52",

libs/policyengine-fastapi/src/policyengine_fastapi/__init__.py

@@ -8,4 +8,5 @@
     TracerCaptureMode as TracerCaptureMode,
     build_observability as build_observability,
     get_observability as get_observability,
+    reset_observability_cache as reset_observability_cache,
 )

libs/policyengine-fastapi/src/policyengine_fastapi/observability/__init__.py

@@ -9,6 +9,7 @@
 
 from .config import (
     ObservabilityConfig as ObservabilityConfig,
+    parse_bool as parse_bool,
     parse_header_value_pairs as parse_header_value_pairs,
 )
 from .contracts import (
@@ -26,12 +27,15 @@
 )
 from .emitters import (
     Observability as Observability,
+    build_traceparent as build_traceparent,
     NoOpObservability as NoOpObservability,
     NoOpSpan as NoOpSpan,
+    OtlpObservability as OtlpObservability,
 )
 from .provider import (
     build_observability as build_observability,
     get_observability as get_observability,
+    reset_observability_cache as reset_observability_cache,
 )
 from .stages import (
     SimulationStage as SimulationStage,

libs/policyengine-fastapi/src/policyengine_fastapi/observability/config.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
+import os
 
 from .stages import TracerCaptureMode
 
@@ -30,6 +31,14 @@ def parse_header_value_pairs(raw: str | None) -> dict[str, str]:
     return headers
 
 
+def parse_bool(raw: str | bool | None, default: bool = False) -> bool:
+    if raw is None:
+        return default
+    if isinstance(raw, bool):
+        return raw
+    return raw.strip().lower() in {"1", "true", "yes", "on"}
+
+
 @dataclass(frozen=True)
 class ObservabilityConfig:
     enabled: bool = False
@@ -46,3 +55,36 @@ class ObservabilityConfig:
     @classmethod
     def disabled(cls, service_name: str = "policyengine-observability"):
         return cls(enabled=False, service_name=service_name)
+
+    @classmethod
+    def from_env(
+        cls,
+        service_name: str,
+        environment: str = "production",
+        prefix: str = "OBSERVABILITY_",
+    ) -> "ObservabilityConfig":
+        return cls(
+            enabled=parse_bool(os.getenv(f"{prefix}ENABLED"), default=False),
+            shadow_mode=parse_bool(
+                os.getenv(f"{prefix}SHADOW_MODE"),
+                default=True,
+            ),
+            service_name=os.getenv(f"{prefix}SERVICE_NAME", service_name),
+            environment=os.getenv(f"{prefix}ENVIRONMENT", environment),
+            otlp_endpoint=os.getenv(f"{prefix}OTLP_ENDPOINT"),
+            otlp_headers=parse_header_value_pairs(os.getenv(f"{prefix}OTLP_HEADERS")),
+            artifact_bucket=os.getenv(f"{prefix}ARTIFACT_BUCKET"),
+            artifact_prefix=os.getenv(
+                f"{prefix}ARTIFACT_PREFIX",
+                "simulation-observability",
+            ),
+            tracer_capture_mode=TracerCaptureMode(
+                os.getenv(
+                    f"{prefix}TRACER_CAPTURE_MODE",
+                    TracerCaptureMode.DISABLED.value,
+                )
+            ),
+            slow_run_threshold_seconds=float(
+                os.getenv(f"{prefix}SLOW_RUN_THRESHOLD_SECONDS", "30.0")
+            ),
+        )