UN-3636 [DEV] Hermetic LLM mock hook for execute-path critical tests

.github/workflows/ci-test.yaml

@@ -22,19 +22,27 @@ on:
 jobs:
   # Path filtering at job level, not trigger paths-ignore: an untriggered
   # workflow reports no check run, so a required check waits forever; a job
-  # skipped via `if:` reports conclusion `skipped`, which passes. Gates only
-  # unit/integration — e2e runs regardless (docker/** changes affect it).
+  # skipped via `if:` reports conclusion `skipped`, which passes. Two scopes:
+  # `relevant` gates unit/integration; `e2e_relevant` gates e2e and skips only
+  # docs — the e2e image build is the sole CI check that compiles the frontend,
+  # and docker/** changes must be e2e-tested.
   changes:
     runs-on: ubuntu-latest
     outputs:
-      relevant: ${{ steps.filter.outputs.relevant }}
+      # Filtering is a PR-time saving only. On workflow_dispatch there is no
+      # base to diff, so paths-filter resolves against the default branch and a
+      # manual run on main diffs main against itself — every filter false, every
+      # tier skipped, gate green with nothing run. Force both true off-PR.
+      relevant: ${{ github.event_name != 'pull_request' || steps.filter.outputs.relevant == 'true' }}
+      e2e_relevant: ${{ github.event_name != 'pull_request' || steps.filter.outputs.e2e_relevant == 'true' }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
           persist-credentials: false
 
       - name: Check for changes outside ignored paths
+        if: github.event_name == 'pull_request'
         uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
         id: filter
         with:
@@ -46,6 +54,8 @@ jobs:
               - "!docker/**"
               - "!frontend/**"
               - "!docs/**"
+            e2e_relevant:
+              - "!docs/**"
 
   # Tiers share no state (separate runners, disjoint groups), so they run as
   # parallel matrix legs; cross-tier reconciliation happens in `report`.
@@ -114,11 +124,12 @@ jobs:
           if-no-files-found: ignore
           retention-days: 14
 
-  # Own runner: builds images then runs e2e against the full compose platform.
-  # Not gated by `changes` — docker/** edits don't count as "relevant" yet must
-  # be e2e-tested.
+  # Builds images then runs e2e against the full compose platform. Gated by
+  # `e2e_relevant`, which unlike `relevant` keeps docker/** and frontend/** in
+  # scope — both are built here — and skips only docs-only changes.
   e2e:
-    if: github.event.pull_request.draft == false
+    needs: changes
+    if: needs.changes.outputs.e2e_relevant == 'true' && github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     timeout-minutes: 45
     env:
@@ -168,8 +179,39 @@ jobs:
       - name: Set up env files
         run: ./run-platform.sh -e
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4
+
+      # Bake instead of `compose build` so each ephemeral runner restores layers
+      # from the GHA cache the release workflow also writes (matching per-service
+      # scopes), instead of building cold every run. `load` exports the images
+      # into the daemon so the compose stack can boot them.
       - name: Build service images
-        run: docker compose -f docker/docker-compose.build.yaml build
+        uses: docker/bake-action@d3418bd7d0e9324001bca92fa8ba175ea7e6dc9b # v7
+        with:
+          files: ./docker/docker-compose.build.yaml
+          load: true
+          set: |
+            *.context=.
+            *.args.VERSION=${{ github.sha }}
+            frontend.cache-from=type=gha,scope=frontend
+            frontend.cache-to=type=gha,mode=max,scope=frontend
+            backend.cache-from=type=gha,scope=backend
+            backend.cache-to=type=gha,mode=max,scope=backend
+            runner.cache-from=type=gha,scope=runner
+            runner.cache-to=type=gha,mode=max,scope=runner
+            tool-sidecar.cache-from=type=gha,scope=tool-sidecar
+            tool-sidecar.cache-to=type=gha,mode=max,scope=tool-sidecar
+            platform-service.cache-from=type=gha,scope=platform-service
+            platform-service.cache-to=type=gha,mode=max,scope=platform-service
+            x2text-service.cache-from=type=gha,scope=x2text-service
+            x2text-service.cache-to=type=gha,mode=max,scope=x2text-service
+            tool-classifier.cache-from=type=gha,scope=tool-classifier
+            tool-classifier.cache-to=type=gha,mode=max,scope=tool-classifier
+            tool-text_extractor.cache-from=type=gha,scope=tool-text_extractor
+            tool-text_extractor.cache-to=type=gha,mode=max,scope=tool-text_extractor
+            worker-unified.cache-from=type=gha,scope=worker-unified
+            worker-unified.cache-to=type=gha,mode=max,scope=worker-unified
 
       - name: Run e2e tier (rig boots the platform via compose)
         env:
@@ -184,15 +226,6 @@ jobs:
             mv reports/.coverage "reports/.coverage.tier-e2e"
           fi
 
-      - name: Capture docker compose logs on failure
-        if: failure()
-        run: |
-          mkdir -p reports
-          docker compose -p unstract-test \
-            -f docker/docker-compose.yaml \
-            -f tests/compose/docker-compose.test.yaml \
-            logs --no-color > reports/docker-compose-logs.txt || true
-
       - name: Upload e2e reports artifact
         if: always()
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
@@ -204,9 +237,9 @@ jobs:
 
   report:
     needs: [changes, test, e2e]
-    # `always()` so a red tier still posts a report. e2e always runs on
-    # non-draft, so gate only on draft; the fail step below distinguishes a
-    # legitimate skip (irrelevant paths) from a real tier failure.
+    # `always()` so a red tier still posts a report. Any tier may skip on
+    # irrelevant paths; the fail step below distinguishes a legitimate skip
+    # from a real tier failure.
     if: always() && github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     steps:
@@ -295,12 +328,15 @@ jobs:
           retention-days: 14
 
       - name: Fail if any tier failed
-        # Single branch-protection check. A skipped `test` means irrelevant
-        # paths (path-filter skip == pass); e2e always runs, so it must be
-        # green. Runs after the comment so a red tier still reports.
+        # Single branch-protection check. A skipped tier means irrelevant paths
+        # (path-filter skip == pass); only an actual failure fails the gate.
+        # Runs after the comment so a red tier still reports.
         if: always()
         run: |
+          changes_result="${{ needs.changes.result }}"
           test_result="${{ needs.test.result }}"
           e2e_result="${{ needs.e2e.result }}"
+          # A broken filter job skips both tiers, which would read as pass.
+          [ "$changes_result" = "success" ] || exit 1
           [ "$test_result" = "success" ] || [ "$test_result" = "skipped" ] || exit 1
-          [ "$e2e_result" = "success" ] || exit 1
+          [ "$e2e_result" = "success" ] || [ "$e2e_result" = "skipped" ] || exit 1

tests/README.md

@@ -24,11 +24,12 @@ tests/
 │   ├── coverage.py          # Per-group coverage files + combine
 │   └── critical_paths.py    # Gap + regression detection
 ├── e2e/
-│   ├── conftest.py          # Session-scoped `platform` fixture
+│   ├── conftest.py          # `platform` fixture + `provisioned_workflow` chain
 │   ├── smoke/               # Login → /health smoke
-│   ├── workflows/           # (future) workflow execution e2e
-│   ├── api_deployment/      # (future) API deployment e2e
-│   ├── prompt_studio/       # (future) Prompt Studio e2e
+│   ├── workflows/           # Workflow execution e2e (mocked LLM)
+│   ├── api_deployment/      # API deployment e2e: run, callback delivery, fan-out (all async)
+│   ├── etl/                 # ETL pipeline e2e (MinIO source + destination)
+│   ├── prompt_studio/       # Prompt Studio fetch-response e2e
 │   └── hurl/                # (future) hurl-based HTTP suites
 ├── integration/             # Cross-service tests needing infra but not full platform
 ├── fixtures/                # Sample PDFs, JSON, adapter configs
@@ -153,6 +154,34 @@ The rig brings the platform up **once** per `run` invocation (if any selected gr
 
 The `platform` pytest fixture in `tests/e2e/conftest.py` reads those env vars; e2e tests run elsewhere (without the rig) just skip with a clear message.
 
+### Hermetic LLM (`UNSTRACT_LLM_MOCK_RESPONSE`)
+
+Execute-path e2e tests must not call a real provider, so the rig sets `UNSTRACT_LLM_MOCK_RESPONSE` (default `MOCK_LLM_OK`) before boot, for any runtime and treating an exported empty string as unset. The test overlay forwards it into the workers, and `unstract.sdk1.llm` passes it to litellm as `mock_response`: for the non-streaming completion path litellm returns the string verbatim with fixed usage (10 prompt / 20 completion / 30 total), so both the answer and the token counts are exact-assertable. Streaming (`stream_complete`) goes through a different litellm path whose usage differs, so don't assume 10/20/30 there. Sentinels like `litellm.RateLimitError` force error paths. Unset (the production default) the hook is a no-op.
+
+Mocking needs two conditions, not one: `ENVIRONMENT` must also be `test` or `development`. Production sets neither variable, so a stray mock var alone cannot fake completions and their billing — and a refusal is logged rather than silent, since setting the var at all means someone expected mocking. `development` is allowed because that is what base compose sets on the workers that run the injection; the test overlay pins `ENVIRONMENT=test` on those same two workers explicitly, so the tier can't lose its mock to a base-compose edit.
+
+A CI/dev override wins (the rig only fills an unset value). Running these tests under the rig **fails** if the var is missing; running **without** the rig just skips the execute-path tests — export it on both sides (your shell and the workers) if you boot the stack yourself.
+
+While the mock is active platform-wide, the LLM adapter's test-connection cannot pass: it regex-matches the completion for a specific city, which `MOCK_LLM_OK` won't satisfy — so `adapter-register-llm` stays an e2e gap for now.
+
+Only `LLM` completions are mocked, not embeddings: `provisioned_workflow` pins `chunk_size=0` so indexing never invokes `litellm.embedding`. A test that needs chunking will need the embedding path mocked too.
+
+### Fan-out (`MAX_PARALLEL_FILE_BATCHES`)
+
+Defaults to `1`, meaning every file of a multi-file run lands in one batch and is processed serially — so a fan-out test would pass without any fan-out happening. The overlay defaults it to `3` on `backend`, which is what normally takes effect: workers ask the backend for this value and fall back to their own env only when they can't reach it or it carries no value (the overlay sets it on `worker-api-deployment-v2` too, to keep the fallback in step). Batches are `min(MAX_PARALLEL_FILE_BATCHES, num_files)`, so N files with the same N gives one batch each.
+
+**The fan-out half is an open gap** (`workflow-execution-fan-out` in `critical_paths.yaml`), and closing it needs a product change, not a cleverer test. Nothing persists a batch or task id: the batch index is a discarded loop local, and the celery task id only ever reaches worker stdout. That leaves per-file timing as the only proxy, and timing does not work here — measured on CI, three files genuinely fanned out finished *further* apart (durations `6.6 / 18.1 / 22.9`) than the ~2s steps seen when they share a batch, because three concurrent tool containers on a loaded runner cost more in contention than they gain in overlap. No threshold separates those two distributions, and no delay value fixes it: the contention scales with the parallelism being measured.
+
+Worth stating plainly because the design is tempting: overlap of the row windows doesn't discriminate either, since a serialised batch pre-creates all its rows in one call and they overlap trivially.
+
+What would close it is a batch or task identifier on `WorkflowFileExecution` — the test then asserts N distinct ids for N files, with no timing and no stall. That is also ordinary execution observability, which is the argument for doing it in the product rather than the test.
+
+`e2e-api-deployment` still asserts the rejoin: one result per file, all files counted into `successful_files`, one row per file.
+
+### ETL (MinIO)
+
+`tests/e2e/etl` runs a pipeline from a source connector to a destination connector. MinIO is the only storage connector the compose stack both boots and registers — the local-filesystem one would need no infra but is never registered (`local_storage/` has no `__init__.py`, so `register_connectors` skips it), which is why the mounted `./workflow_data:/data` volume can't be used as an ETL endpoint. The test seeds and reads its objects over the published port (`UNSTRACT_MINIO_ENDPOINT`, default `localhost:9000`) while the workers reach the same store over the compose network (`UNSTRACT_MINIO_INTERNAL_URL`, default `http://unstract-minio:9000`). It skips when no MinIO answers, so it does not fail a runtime that publishes none.
+
 ---
 
 ## Reports

tests/compose/docker-compose.test.yaml

@@ -1,21 +1,16 @@
-# E2E test overlay for docker/docker-compose.yaml.
-#
-# Used by tests/rig/runtime.py's ComposeRuntime:
-#   docker compose -f docker/docker-compose.yaml \
-#                  -f tests/compose/docker-compose.test.yaml up -d --wait
-#
-# Keep this file minimal so e2e tests run against images as close to production
-# as possible. Override only what the test stack actually needs (test creds,
-# pinned image tags, ENVIRONMENT=test sentinel).
+# E2E overlay for docker/docker-compose.yaml — override only what the test
+# stack needs, so e2e runs against production-like images.
 
 services:
   backend:
-    # Defaults to `latest` so contributors can run without setting
-    # UNSTRACT_TEST_VERSION. CI pins it to the commit SHA of the images built
-    # in the same run (see the e2e job in .github/workflows/ci-test.yaml).
+    # CI pins the tag to the SHA built in the same run; `latest` lets a
+    # contributor run without setting it.
     image: unstract/backend:${UNSTRACT_TEST_VERSION:-latest}
     environment:
       - ENVIRONMENT=test
+      # Workers ask the backend for this, so this is the value that normally
+      # takes effect for fan-out.
+      - MAX_PARALLEL_FILE_BATCHES=${MAX_PARALLEL_FILE_BATCHES:-3}
 
   platform-service:
     image: unstract/platform-service:${UNSTRACT_TEST_VERSION:-latest}
@@ -26,3 +21,23 @@ services:
     image: unstract/runner:${UNSTRACT_TEST_VERSION:-latest}
     environment:
       - ENVIRONMENT=test
+
+  # Execute-path e2e must never reach a real provider. The delay gives each
+  # mocked completion a known cost so per-file durations stay comparable, and
+  # ENVIRONMENT is the mock's second condition — set here rather than inherited
+  # so the tier can't lose its mock to a base-compose edit.
+  worker-executor-v2:
+    environment:
+      - ENVIRONMENT=test
+      - UNSTRACT_LLM_MOCK_RESPONSE=${UNSTRACT_LLM_MOCK_RESPONSE:-}
+
+  worker-file-processing-v2:
+    environment:
+      - ENVIRONMENT=test
+      - UNSTRACT_LLM_MOCK_RESPONSE=${UNSTRACT_LLM_MOCK_RESPONSE:-}
+
+  # Only the worker's fallback if it can't reach the backend; kept in step so it
+  # can't quietly serialise the fan-out test.
+  worker-api-deployment-v2:
+    environment:
+      - MAX_PARALLEL_FILE_BATCHES=${MAX_PARALLEL_FILE_BATCHES:-3}

tests/critical_paths.yaml

@@ -49,6 +49,7 @@ paths:
   - id: workflow-create-execute
     description: "Create a workflow, configure source+destination, execute, poll, fetch result."
     covered_by: [e2e-workflow]
+    proof: marker
 
   - id: api-deployment-provision
     description: "Deploying a workflow as an API mints a usable key and a resolvable endpoint."
@@ -63,15 +64,17 @@ paths:
   - id: api-deployment-run
     description: "Deploy a workflow as an API, POST a document, receive structured JSON."
     covered_by: [e2e-api-deployment]
+    proof: marker
 
   - id: prompt-studio-author
     description: "Create a Prompt Studio project and add a prompt to it."
     covered_by: [integration-backend]
     proof: marker
 
   - id: prompt-studio-fetch-response
-    description: "Prompt Studio: create project, add prompt, run single-pass, get response."
+    description: "Prompt Studio: create project, add prompt, run a prompt, get response."
     covered_by: [e2e-prompt-studio]
+    proof: marker
 
   - id: connector-register-test
     description: "Connector credentials are validated against the live system and stored encrypted."
@@ -80,7 +83,8 @@ paths:
 
   - id: pipeline-etl-execute
     description: "Run an ETL pipeline from source connector to destination."
-    covered_by: []   # gap
+    covered_by: [e2e-etl]
+    proof: marker
 
   - id: usage-aggregate-read
     description: "Per-run token usage aggregates correctly and stays scoped to its organization."
@@ -89,12 +93,18 @@ paths:
 
   - id: usage-token-tracking
     description: "Per-execution token usage is recorded and retrievable."
-    covered_by: []   # gap
+    covered_by: [e2e-api-deployment]
+    proof: marker
 
+  # Deliberate gap. The rejoin half is covered by e2e-api-deployment, but the
+  # fan-out half has no observable: no batch or task id is persisted, and timing
+  # can't stand in for one (see the test's module docstring). Closing it needs
+  # batch identity on WorkflowFileExecution, not another test.
   - id: workflow-execution-fan-out
     description: "Multi-file workflow execution fans out to file-processing workers and rejoins."
-    covered_by: []   # gap
+    covered_by: []
 
   - id: callback-result-delivery
     description: "Async results are posted back via the callback worker."
-    covered_by: []   # gap
+    covered_by: [e2e-api-deployment]
+    proof: marker