Split the simulation service into gateway/executor projects with lockfile-derived Modal images

.github/CONTRIBUTING.md

@@ -17,15 +17,15 @@ make push-pr-branch        # push current branch to origin with tracking
 For simulation-service-only checks:
 
 ```bash
-cd projects/policyengine-api-simulation
+cd projects/policyengine-simulation-executor
 uv sync --extra test
 uv run pytest tests/ -v
 ```
 
 ## Test Organisation
 
 - Service unit tests live under each service's `tests/` directory, for example
-  `projects/policyengine-api-simulation/tests/`.
+  `projects/policyengine-simulation-executor/tests/`.
 - Generated-client integration tests live under
   `projects/policyengine-apis-integ/tests/`.
 - Unit tests should mock Modal, GCP, Hugging Face, and other network seams.

.github/scripts/modal-deploy-app.sh

@@ -38,11 +38,16 @@ echo "  UK version: ${POLICYENGINE_UK_VERSION}"
 echo "  Force latest: ${FORCE_LATEST}"
 echo "========================================"
 
-# 1. Deploy the gateway app (stable URL)
+# 1. Deploy the gateway app (stable URL) from its own project
 echo ""
 echo "Step 1: Deploying gateway app..."
 echo "  App name: policyengine-simulation-gateway"
-uv run modal deploy --env="$MODAL_ENV" src/modal/gateway/app.py
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+(
+    cd "$REPO_ROOT/projects/policyengine-simulation-gateway"
+    uv run modal deploy --env="$MODAL_ENV" \
+        src/policyengine_simulation_gateway/app.py
+)
 
 # 2. Deploy the versioned simulation app
 echo ""

.github/scripts/modal-image-smoke.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Run the pre-merge image smokes against a Modal environment.
+# Usage: ./modal-image-smoke.sh <modal-environment>
+#
+# Each smoke imports the true app entrypoints inside the real (or, for
+# the executor, prefix-identical) Modal image — catching in-image
+# dependency breakage (issue #602's class) before merge.
+
+set -euo pipefail
+
+MODAL_ENV="${1:?Modal environment required}"
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+echo "=== Gateway image smoke (env: $MODAL_ENV) ==="
+(
+    cd "$REPO_ROOT/projects/policyengine-simulation-gateway"
+    uv run modal run --env="$MODAL_ENV" \
+        src/policyengine_simulation_gateway/smoke_app.py
+)
+
+echo "=== Executor image smoke (env: $MODAL_ENV) ==="
+(
+    cd "$REPO_ROOT/projects/policyengine-simulation-executor"
+    uv run modal run --env="$MODAL_ENV" src/modal/smoke_app.py
+)
+
+echo "=== Image smokes passed ==="

.github/scripts/update-policyengine-package.sh

@@ -25,7 +25,7 @@ fi
 
 PACKAGE="policyengine"
 ROOT_DIR="$(git rev-parse --show-toplevel)"
-PROJECT_DIR="${PROJECT_DIR:-projects/policyengine-api-simulation}"
+PROJECT_DIR="${PROJECT_DIR:-projects/policyengine-simulation-executor}"
 PROJECT_PATH="${ROOT_DIR}/${PROJECT_DIR}"
 PYPROJECT="${PROJECT_PATH}/pyproject.toml"
 LOCKFILE="${PROJECT_PATH}/uv.lock"

.github/workflows/modal-deploy.reusable.yml

@@ -69,15 +69,15 @@ jobs:
       run: chmod +x .github/scripts/*.sh
 
     - name: Install dependencies
-      working-directory: projects/policyengine-api-simulation
+      working-directory: projects/policyengine-simulation-executor
       run: uv sync
 
     - name: Extract package versions
       id: versions
-      run: .github/scripts/modal-extract-versions.sh projects/policyengine-api-simulation
+      run: .github/scripts/modal-extract-versions.sh projects/policyengine-simulation-executor
 
     - name: Sync Modal secrets from GitHub
-      working-directory: projects/policyengine-api-simulation
+      working-directory: projects/policyengine-simulation-executor
       env:
         MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
         MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
@@ -92,7 +92,7 @@ jobs:
       run: ../../.github/scripts/modal-sync-secrets.sh "${{ inputs.modal_environment }}" "${{ inputs.environment }}"
 
     - name: Deploy simulation API to Modal
-      working-directory: projects/policyengine-api-simulation
+      working-directory: projects/policyengine-simulation-executor
       env:
         MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
         MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
@@ -104,7 +104,7 @@ jobs:
 
     - name: Get deployed URL
       id: get-url
-      working-directory: projects/policyengine-api-simulation
+      working-directory: projects/policyengine-simulation-executor
       env:
         MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
         MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}

.github/workflows/modal-deploy.yml

@@ -40,11 +40,11 @@ jobs:
       run: chmod +x .github/scripts/*.sh
 
     - name: Install dependencies
-      working-directory: projects/policyengine-api-simulation
+      working-directory: projects/policyengine-simulation-executor
       run: uv sync
 
     - name: Ensure Modal environments exist
-      working-directory: projects/policyengine-api-simulation
+      working-directory: projects/policyengine-simulation-executor
       env:
         MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
         MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}

.github/workflows/pr-image-smoke.yml

@@ -0,0 +1,52 @@
+name: PR image smoke
+
+# Import the true app entrypoints inside the real Modal images before
+# merge (issue #602's class: image-only dependency breakage that unit
+# tests in the locked env cannot see). Path-filtered to image inputs;
+# skipped on fork PRs (no Modal secrets there) — those rely on the
+# post-merge beta integration tests as before.
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - 'projects/policyengine-simulation-gateway/**'
+      - 'projects/policyengine-simulation-executor/uv.lock'
+      - 'projects/policyengine-simulation-executor/pyproject.toml'
+      - 'projects/policyengine-simulation-executor/src/modal/**'
+      - 'projects/policyengine-simulation-executor/src/policyengine_simulation_executor/**'
+      - 'libs/policyengine-simulation-contract/**'
+      - 'libs/policyengine-simulation-observability/**'
+      - 'libs/policyengine-fastapi/**'
+      - '.github/workflows/pr-image-smoke.yml'
+      - '.github/scripts/modal-image-smoke.sh'
+
+jobs:
+  image-smoke:
+    name: Image smoke (staging)
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.head.repo.full_name == github.repository
+    # Warm cache: ~2 min. After a relock the executor smoke pays the
+    # policyengine bundle install layer (~15-20 min).
+    timeout-minutes: 45
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.13'
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v8.1.0
+      with:
+        enable-cache: true
+
+    - name: Run image smokes
+      env:
+        MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
+        MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
+      run: |
+        chmod +x .github/scripts/modal-image-smoke.sh
+        .github/scripts/modal-image-smoke.sh staging

.github/workflows/pr.yml

@@ -10,7 +10,14 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        service: [api-simulation]
+        # Each entry is a uv project tested in its own locked environment.
+        # For the gateway this is the parity guarantee: the same lock the
+        # Modal image installs with uv_sync(frozen=True).
+        project:
+          - projects/policyengine-simulation-executor
+          - projects/policyengine-simulation-gateway
+          - libs/policyengine-simulation-contract
+          - libs/policyengine-simulation-observability
 
     steps:
     - uses: actions/checkout@v6
@@ -27,12 +34,12 @@ jobs:
 
     - name: Install dependencies
       run: |
-        cd projects/policyengine-${{ matrix.service }}
+        cd ${{ matrix.project }}
         uv sync --extra test
 
     - name: Run tests
       run: |
-        cd projects/policyengine-${{ matrix.service }}
+        cd ${{ matrix.project }}
         uv run pytest tests/ -v
 
   lint:
@@ -68,7 +75,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        service: [api-simulation]
+        service: [simulation-executor]
 
     steps:
     - uses: actions/checkout@v6

.github/workflows/publish-clients.yml

@@ -33,7 +33,7 @@ jobs:
 
     - name: Build simulation API client
       run: |
-        cd projects/policyengine-api-simulation/artifacts/clients/python
+        cd projects/policyengine-simulation-gateway/artifacts/clients/python
         # Update version
         DATE=$(date +%Y%m%d)
         RUN_NUMBER="${{ github.run_number }}"
@@ -45,4 +45,4 @@ jobs:
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         password: ${{ secrets.PYPI }}
-        packages-dir: projects/policyengine-api-simulation/artifacts/clients/python/dist/
+        packages-dir: projects/policyengine-simulation-gateway/artifacts/clients/python/dist/

.gitignore

@@ -14,3 +14,6 @@ backend.tfvars
 .vscode
 deployment/terraform/*/auto.tfvars
 .claude-plan
+
+# macOS Finder metadata
+.DS_Store

AGENTS.md

@@ -52,7 +52,7 @@ same-repository draft PR.
 
 ## Repository Notes
 
-- The simulation service lives in `projects/policyengine-api-simulation`.
+- The simulation service lives in `projects/policyengine-simulation-executor`.
 - API integration tests live in `projects/policyengine-apis-integ`.
 - PR CI runs simulation unit tests, Ruff format checks, Docker build, and local
   integration tests.

Makefile

@@ -104,10 +104,16 @@ publish-clients: generate-clients
 # Testing
 test:
 	@echo "Running tests for all services..."
-	@for service in api-full api-simulation api-tagger; do \
+	@for service in api-full simulation-executor api-tagger; do \
 		echo "Testing $$service..."; \
 		docker-compose -f deployment/docker-compose.yml run --rm $$service sh -c "cd /app/projects/policyengine-$$service && uv run --extra test pytest" || exit 1; \
 	done
+	@echo "Testing simulation-gateway (no compose service)..."
+	@cd projects/policyengine-simulation-gateway && uv sync --extra test && uv run pytest
+	@for lib in policyengine-simulation-contract policyengine-simulation-observability; do \
+		echo "Testing $$lib..."; \
+		(cd libs/$$lib && uv sync --extra test && uv run pytest) || exit 1; \
+	done
 
 test-service:
 ifndef service
@@ -229,8 +235,8 @@ terraform-plan: terraform-ensure-init
 	fi
 	@echo "\n=== Planning INFRA module ==="
 	@# Auto-populate all required variables
-	@US_VERSION=$$(grep -A1 'name = "policyengine-us"' projects/policyengine-api-simulation/uv.lock | grep version | head -1 | sed 's/.*"\(.*\)".*/\1/') && \
-	UK_VERSION=$$(grep -A1 'name = "policyengine-uk"' projects/policyengine-api-simulation/uv.lock | grep version | head -1 | sed 's/.*"\(.*\)".*/\1/') && \
+	@US_VERSION=$$(grep -A1 'name = "policyengine-us"' projects/policyengine-simulation-executor/uv.lock | grep version | head -1 | sed 's/.*"\(.*\)".*/\1/') && \
+	UK_VERSION=$$(grep -A1 'name = "policyengine-uk"' projects/policyengine-simulation-executor/uv.lock | grep version | head -1 | sed 's/.*"\(.*\)".*/\1/') && \
 	COMMIT_URL="https://github.com/PolicyEngine/policyengine-api-v2/commit/$$(git rev-parse HEAD)" && \
 	echo "project_id = \"$${TF_VAR_project_id}\"" > deployment/terraform/infra/auto.tfvars && \
 	echo "commit_url = \"$$COMMIT_URL\"" >> deployment/terraform/infra/auto.tfvars && \
@@ -257,8 +263,8 @@ terraform-deploy-project: terraform-ensure-init
 terraform-deploy-infra: terraform-ensure-init
 	@echo "Deploying infrastructure (Cloud Run, etc)..."
 	@# Auto-populate all required variables
-	@US_VERSION=$$(grep -A1 'name = "policyengine-us"' projects/policyengine-api-simulation/uv.lock | grep version | head -1 | sed 's/.*"\(.*\)".*/\1/') && \
-	UK_VERSION=$$(grep -A1 'name = "policyengine-uk"' projects/policyengine-api-simulation/uv.lock | grep version | head -1 | sed 's/.*"\(.*\)".*/\1/') && \
+	@US_VERSION=$$(grep -A1 'name = "policyengine-us"' projects/policyengine-simulation-executor/uv.lock | grep version | head -1 | sed 's/.*"\(.*\)".*/\1/') && \
+	UK_VERSION=$$(grep -A1 'name = "policyengine-uk"' projects/policyengine-simulation-executor/uv.lock | grep version | head -1 | sed 's/.*"\(.*\)".*/\1/') && \
 	COMMIT_URL="https://github.com/PolicyEngine/policyengine-api-v2/commit/$$(git rev-parse HEAD)" && \
 	echo "project_id = \"$${TF_VAR_project_id}\"" > deployment/terraform/infra/auto.tfvars && \
 	echo "commit_url = \"$$COMMIT_URL\"" >> deployment/terraform/infra/auto.tfvars && \
@@ -388,7 +394,7 @@ dev-full:
 	docker-compose -f deployment/docker-compose.yml up api-full
 
 dev-sim:
-	docker-compose -f deployment/docker-compose.yml up api-simulation
+	docker-compose -f deployment/docker-compose.yml up simulation-executor
 
 dev-tagger:
 	docker-compose -f deployment/docker-compose.yml up api-tagger

README.md

@@ -33,7 +33,7 @@ make test-complete                 # Everything: unit + integration tests
 The repository contains three main API services:
 
 - **api-full** (port 8081): Main PolicyEngine API with household calculations
-- **api-simulation** (port 8082): Economic simulation engine  
+- **simulation-executor** (port 8082): Economic simulation engine  
 - **api-tagger** (port 8083): Cloud Run revision management
 
 Each service generates OpenAPI specs and Python client libraries for integration testing.
@@ -67,7 +67,7 @@ make test-integration        # Run integration tests (requires services running)
 /
 ├── projects/               # Service applications
 │   ├── policyengine-api-full/
-│   ├── policyengine-api-simulation/
+│   ├── policyengine-simulation-executor/
 │   ├── policyengine-api-tagger/
 │   └── policyengine-apis-integ/    # Integration tests
 ├── libs/                   # Shared libraries

deployment/docker-compose.yml

@@ -1,16 +1,16 @@
 services:
-  api-simulation:
+  simulation-executor:
     build:
       context: ..
-      dockerfile: projects/policyengine-api-simulation/Dockerfile
+      dockerfile: projects/policyengine-simulation-executor/Dockerfile
     environment:
       - ENVIRONMENT=desktop
     volumes:
-      - ../projects/policyengine-api-simulation/src:/app/projects/policyengine-api-simulation/src
+      - ../projects/policyengine-simulation-executor/src:/app/projects/policyengine-simulation-executor/src
       - ../libs:/app/libs
     ports:
       - "8082:8080"
-    command: sh -c "cd src && uv run uvicorn policyengine_api_simulation.main:app --reload --host 0.0.0.0 --port 8080"
+    command: sh -c "cd src && uv run --frozen --no-dev uvicorn policyengine_simulation_executor.main:app --reload --host 0.0.0.0 --port 8080"
     networks:
       - policyengine
 

docs/engineering/skills/testing.md

@@ -5,7 +5,7 @@ Use this skill whenever adding, moving, or reviewing tests.
 ## Canonical Layout
 
 - Service unit tests live under each service's `tests/` directory, for example
-  `projects/policyengine-api-simulation/tests/`.
+  `projects/policyengine-simulation-executor/tests/`.
 - Generated-client integration tests live under
   `projects/policyengine-apis-integ/tests/`.
 - Put reusable test helpers in local fixture modules or support modules near
@@ -38,7 +38,7 @@ make test-complete
 Simulation-service focused checks:
 
 ```bash
-cd projects/policyengine-api-simulation
+cd projects/policyengine-simulation-executor
 uv sync --extra test
 uv run pytest tests/ -v
 ```

libs/policyengine-simulation-contract/README.md

@@ -0,0 +1,9 @@
+# policyengine-simulation-contract
+
+The contract between the simulation gateway and executor: request/response
+models (`gateway_models`), shared budget-window job state over `modal.Dict`
+(`budget_window_state`), and dataset reference resolution
+(`dataset_uri`, `hf_dataset`).
+
+The gateway and executor never import each other — they communicate only
+through the models and state helpers in this lib.