From 63816ab94c0f37c9295723a65fd8eb7e2238639e Mon Sep 17 00:00:00 2001 From: Ryan Dombrowski Date: Wed, 12 Aug 2026 12:38:17 -0400 Subject: [PATCH] =?UTF-8?q?test:=20D13=20=E2=80=94=20end-to-end=20coverage?= =?UTF-8?q?=20for=20the=20first-adoption=20surfaces?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit found 62 of 134 composer testids in no spec, all of them on the paths a NEW team touches first. The Build/Preview path was well covered; provider configuration, surface authoring, flow decomposition, and governance authoring were not covered at all. This closes those four gaps with behaviour-level tests — state changes, persisted results, honest error text, the thing actually rendering — never "the element exists". No product source is touched. No spec makes a model call. Provider configuration (e2e/composer-settings.spec.ts, agent config) Ollama and OpenAI-compatible endpoints configured through the real agent against e2e/serve-provider.mjs — a real HTTP server speaking both discovery protocols, added as a third webServer. It serves discovery only: a build against it would be a model call. Covers discovery (including the agent's embedding-model filter), model choice, persistence and re-open, manual model entry when a server does not enumerate, and the honest failure text from an unreachable endpoint (which configures nothing). The credential invariant is proved end to end rather than asserted cosmetically: the fixture's /keyed endpoint 401s unless the exact key arrives, so the model list only appears if the key travelled browser → agent → provider — and localStorage/sessionStorage are then asserted to contain no trace of it. The agent-ABSENT half lives where it is true by construction, in composer-prod-smoke.spec.ts: with the agent probe blocked, "Agent not running" is a setup step with real instructions, and both provider forms are inert rather than dead-looking. Flows (e2e/composer-flows.spec.ts, agent-free config) Plan editing before anything is built (rename, retitle, reorder with a real swap, add, trim) and that planning creates nothing; the drive creating the flow immediately with PENDING steps, the plan freezing into per-step rebuilds, Preview's outline state and flow-lint's matching warning; a rebuilt step binding on accept; the editor's cancel creating nothing; a walk completing on the surface's own emitted action; and a step over a surface the emitter refuses showing the emitter's own reason. Surface authoring (e2e/composer-surfaces.spec.ts, agent-free config) Author → live gates → live preview → save → listed under its HUMAN title → rendered in Preview → survives reload → reported by Checks against that surface. Plus both honest-failure directions: a `must` rule blocks the save and nothing is written (before or after a reload), and fixing the violation unblocks it; a `should` rule warns without blocking. Governance (e2e/composer-governance.spec.ts, agent-free config) Intent authoring gated on a real description, then governing Build and the surface editor; the rationale gate on rules; a saved rule visibly firing in the impact panel and in Checks, and its removal undoing exactly that; the typed-rule form projections; and the session-scope honesty a browser project states about governance edits. Browser/agent parity (e2e/composer-parity.spec.ts, agent config) The product-level twin of the emit-seam unit equivalence: one repository project is EXPORTED and imported back as a browser project, so both carry byte-equal vocabulary, and the same authored surface must get the same verdict — same refusal text, same clean state, same surface-scoped findings in Checks (compared non-vacuously). Scoped to the surface on purpose: a repository additionally emits its surfacesDir, a documented corpus asymmetry that would make whole-table equality a lie. Config hygiene: the exhibit config ignored composer specs by hand-typed name — the same shape as the CI filter list of #81, where adding a spec silently opted it into the wrong suite. It now ignores them by pattern. Counts: composer-smoke 14 → 28, composer-agent 44 → 49, composer-production 14 → 15, exhibit 110 → 110 (unchanged, verified). Uncovered testids 62 → 22, and the 22 are all on surfaces outside this milestone (Catalog, Mapper, Components, Repository, hub). Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 10 +- e2e/composer-flows.spec.ts | 246 ++++++++++++++++++++++++++++ e2e/composer-governance.spec.ts | 175 ++++++++++++++++++++ e2e/composer-parity.spec.ts | 135 +++++++++++++++ e2e/composer-prod-smoke.spec.ts | 35 ++++ e2e/composer-settings.spec.ts | 164 +++++++++++++++++++ e2e/composer-surfaces.spec.ts | 143 ++++++++++++++++ e2e/serve-provider.mjs | 56 +++++++ e2e/support/composer-browser.ts | 69 ++++++++ playwright.composer-agent.config.ts | 19 ++- playwright.composer-smoke.config.ts | 15 +- playwright.config.ts | 11 +- 12 files changed, 1071 insertions(+), 7 deletions(-) create mode 100644 e2e/composer-flows.spec.ts create mode 100644 e2e/composer-governance.spec.ts create mode 100644 e2e/composer-parity.spec.ts create mode 100644 e2e/composer-settings.spec.ts create mode 100644 e2e/composer-surfaces.spec.ts create mode 100644 e2e/serve-provider.mjs create mode 100644 e2e/support/composer-browser.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f024b2f..38f109b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,9 +65,15 @@ jobs: # (propose → S1/S2/S3 → repair → emit → render → refine), validation, # preview, export, and client-bundle hygiene — the deployed acceptance # criteria, verified BEFORE deploy. - - name: Playwright (composer hosted/demo smoke — in-browser Build, repair, refine, hygiene) + # Two projects in this config: "composer-smoke" (the deploy acceptance + # criteria above) and "composer-product" — surface authoring, flows, and + # governance authoring, the agent-free surfaces a new team meets first. + - name: Playwright (composer hosted/demo — Build loop, surface authoring, flows, governance, hygiene) run: pnpm exec playwright test --config playwright.composer-smoke.config.ts - - name: Playwright (composer agent-mode decisions + accessibility) + # Also starts a fixture local-provider endpoint (e2e/serve-provider.mjs) + # so provider configuration is driven against a real socket through the + # real agent. It serves discovery only — nothing here calls a model. + - name: Playwright (composer agent-mode decisions, provider configuration, browser/agent parity, accessibility) run: pnpm exec playwright test --config playwright.composer-agent.config.ts - name: Upload Playwright report on failure if: failure() diff --git a/e2e/composer-flows.spec.ts b/e2e/composer-flows.spec.ts new file mode 100644 index 0000000..dd7ed1b --- /dev/null +++ b/e2e/composer-flows.spec.ts @@ -0,0 +1,246 @@ +import { expect, test, type Page } from "@playwright/test"; +import { buildAndAccept, newProject } from "./support/composer-browser"; + +/** + * Flows — composition and decomposition, end to end (P4 Phases B + C). + * + * The smoke suite already proves the happy path exists. What had no coverage + * is everything a person actually does around it: editing a proposed plan + * before committing to it, cancelling out of the flow editor, re-running one + * step, walking a flow to completion, and what the product says when a step's + * surface cannot be emitted at all. + * + * Every build here runs on the SCRIPTED provider — deterministic replay, zero + * model calls — and the planner is the deterministic outline, so nothing in + * this file talks to a model or a gateway. + */ + +/** Three sentences → three deterministic steps. */ +const THREE_STEP_GOAL = + "Show one order in full detail. Let people delete their account. Show a table of the remaining accounts."; +const TWO_STEP_GOAL = "Show one order in full detail. Let people delete their account."; + +/** Open flow mode and produce the deterministic outline for a goal. */ +async function planFlow(page: Page, goal: string): Promise { + await page.getByTestId("build-model").selectOption("scripted"); + await page.getByTestId("build-mode-flow").click(); + await expect(page.getByTestId("flow-composer")).toBeVisible(); + await page.getByTestId("build-flow-goal").fill(goal); + await page.getByTestId("flow-plan-run").click(); + await expect(page.getByTestId("flow-plan-editor")).toBeVisible(); +} + +/** The two-step plan both drive specs build, titled and governed explicitly. */ +async function twoStepPlan(page: Page): Promise { + await planFlow(page, TWO_STEP_GOAL); + await page.getByTestId("flow-plan-name").fill("Order journey"); + await page.getByTestId("flow-plan-title-0").fill("Review the order"); + await page.getByTestId("flow-plan-intent-0").selectOption("record-detail"); + await page.getByTestId("flow-plan-title-1").fill("Delete the account"); + await page.getByTestId("flow-plan-intent-1").selectOption("destructive-action"); + await expect(page.getByTestId("flow-drive-status")).toContainText("2 steps — builds run one at a time."); + await page.getByTestId("flow-plan-accept").click(); + await expect(page.getByTestId("build-gate-summary-1")).toContainText("Follows your design-system rules", { timeout: 60_000 }); + await expect(page.getByTestId("build-gate-summary-2")).toContainText("Follows your design-system rules", { timeout: 60_000 }); + await expect(page.getByTestId("flow-drive-status")).toContainText("steps ran — accept each turn below"); +} + +test("the proposed plan is editable — renamed, retitled, reordered, extended, trimmed — and builds nothing until it is accepted", async ({ + page, +}) => { + await page.goto("/"); + await newProject(page, "shadcn", "Flow planning"); + + // Flow mode replaces the single-surface composer; it is a mode, not an extra + // control bolted beside the ordinary prompt. + await planFlow(page, THREE_STEP_GOAL); + await expect(page.getByTestId("build-prompt")).toHaveCount(0); + await expect(page.getByTestId("flow-plan-source")).toContainText(/deterministic/i); + + // The planner names the flow from the goal; the person can overrule it. + const name = page.getByTestId("flow-plan-name"); + await expect(name).toHaveValue("Show one order in full detail"); + await name.fill("Order journey"); + await expect(name).toHaveValue("Order journey"); + + // One step per sentence, each pre-titled from its own sentence. + await expect(page.getByTestId("flow-plan-title-0")).toHaveValue("Show one order in full detail"); + await expect(page.getByTestId("flow-plan-title-2")).toHaveValue("Show a table of the remaining"); + await expect(page.getByTestId("flow-plan-title-3")).toHaveCount(0); + + // Retitle in place. + await page.getByTestId("flow-plan-title-0").fill("Review the order"); + await expect(page.getByTestId("flow-plan-title-0")).toHaveValue("Review the order"); + + // Reorder: the two rows genuinely swap, and the ends cannot move past them. + await expect(page.getByTestId("flow-plan-up-0")).toBeDisabled(); + await expect(page.getByTestId("flow-plan-down-2")).toBeDisabled(); + const second = await page.getByTestId("flow-plan-title-1").inputValue(); + await page.getByTestId("flow-plan-down-0").click(); + await expect(page.getByTestId("flow-plan-title-0")).toHaveValue(second); + await expect(page.getByTestId("flow-plan-title-1")).toHaveValue("Review the order"); + await page.getByTestId("flow-plan-up-1").click(); + await expect(page.getByTestId("flow-plan-title-0")).toHaveValue("Review the order"); + + // Extend and trim. + await page.getByTestId("flow-plan-add").click(); + await expect(page.getByTestId("flow-plan-title-3")).toHaveValue("Step 4"); + await page.getByTestId("flow-plan-remove-3").click(); + await expect(page.getByTestId("flow-plan-title-3")).toHaveCount(0); + await page.getByTestId("flow-plan-remove-2").click(); + await expect(page.getByTestId("flow-plan-title-2")).toHaveCount(0); + + // The driver states its contract up front: sequential, and (on scripted) not + // artificially spaced. + await expect(page.getByTestId("flow-drive-status")).toContainText("2 steps — builds run one at a time."); + + // Planning creates NOTHING. The flow only exists once the plan is accepted. + await page.getByTestId("nav-flows").click(); + await expect(page.getByTestId("flows-empty")).toBeVisible(); +}); + +test("accepting a plan creates the flow immediately with pending steps, and Preview and Checks say so honestly", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await newProject(page, "shadcn", "Flow drive"); + await twoStepPlan(page); + + // The plan froze when the drive started: reorder and add are gone, and each + // row now offers a rebuild instead. + await expect(page.getByTestId("flow-plan-remove-0")).toHaveCount(0); + await expect(page.getByTestId("flow-plan-add")).toHaveCount(0); + await expect(page.getByTestId("flow-build-step-0")).toBeVisible(); + await expect(page.getByTestId("flow-build-step-1")).toBeVisible(); + + // The flow exists ALREADY, with both steps pending: an outline you can walk, + // not a crash and not a dangling reference. + await page.getByTestId("nav-flows").click(); + await expect(page.getByTestId("flow-flow.flow-1")).toContainText("Order journey"); + await expect(page.getByTestId("flow-navigator")).toBeVisible(); + await expect(page.getByTestId("flow-step-step.review-the-order")).toContainText("pending"); + await expect(page.getByTestId("flow-step-step.review-the-order")).toBeDisabled(); + await expect(page.getByTestId("flow-step-pending")).toContainText("isn’t built yet"); + await expect(page.getByTestId("flow-step-pending")).toContainText("build it from Build and accept into this step"); + await expect(page.locator("[data-project-canvas]")).toHaveCount(0); + + // Flow lint agrees, in the same words, and calls it a warning — planned-but- + // unbuilt is a state of the work, not a failure of the project. + await page.getByTestId("nav-validate").click(); + await page.getByTestId("run-validate").click(); + await expect(page.getByTestId("flows-summary")).toContainText("1 flow · 2 steps · flow checks PASS"); + const pending = page.getByTestId("finding-flow-pending-step").first(); + await expect(pending).toContainText("warn"); + await expect(pending).toContainText("is not built yet"); +}); + +test("a driven step rebuilds in place, and each accepted turn binds to the step it was built for", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await newProject(page, "shadcn", "Flow rebuild"); + await twoStepPlan(page); + + // Accepting a driven turn binds its step — the driver pre-targeted it, so + // the confirmation names the step rather than just the surface. + await page.getByTestId("build-accept-1").click(); + await expect(page.getByTestId("build-accepted-1")).toContainText("Review the order"); + + // Re-run just the second step, then accept THAT turn into its step. + await page.getByTestId("flow-build-step-1").click(); + await expect(page.getByTestId("build-gate-summary-3")).toContainText("Follows your design-system rules", { timeout: 60_000 }); + await page.getByTestId("build-accept-3").click(); + await expect(page.getByTestId("build-accepted-3")).toContainText("Delete the account"); + + // Both steps now render their bound surfaces through the ordinary canvas, + // and flow lint has nothing left to report. + await page.getByTestId("nav-flows").click(); + await expect(page.getByTestId("flow-step-step.review-the-order")).toBeEnabled(); + await expect(page.getByTestId("flow-step-step.review-the-order")).not.toContainText("pending"); + await expect(page.locator("[data-project-canvas]")).toContainText(/order/i); + await page.getByTestId("flow-next").click(); + await expect(page.locator("[data-project-canvas]")).toContainText(/delete/i); + await page.getByTestId("nav-validate").click(); + await page.getByTestId("run-validate").click(); + await expect(page.getByTestId("flows-summary")).toContainText("flow checks PASS"); + await expect(page.getByTestId("finding-flow-pending-step")).toHaveCount(0); +}); + +test("the flow editor: cancelling creates nothing, and a saved flow walks to completion on its own emitted action", async ({ page }) => { + await page.goto("/"); + await newProject(page, "shadcn", "Flow editor"); + await buildAndAccept(page, "record-detail", "show one order in full detail"); + + await page.getByTestId("nav-preview").click(); + await expect(page.getByTestId("flows-empty")).toBeVisible(); + + // Cancel abandons the draft: no flow, no editor, nothing persisted. + await page.getByTestId("new-flow").click(); + await expect(page.getByTestId("flow-editor")).toBeVisible(); + await page.getByTestId("flow-name").fill("Abandoned"); + await page.getByTestId("flow-cancel").click(); + await expect(page.getByTestId("flow-editor")).toHaveCount(0); + await expect(page.getByTestId("flow-flow.flow-1")).toHaveCount(0); + await expect(page.getByTestId("flows-empty")).toBeVisible(); + + // Compose a one-step flow that advances on the surface's OWN emitted action. + await page.getByTestId("new-flow").click(); + await page.getByTestId("flow-name").fill("Order walkthrough"); + await page.getByTestId("flow-step-surface").selectOption("ex.chat-1"); + await page.getByTestId("flow-add-step").click(); + await page.getByTestId("flow-step-title-0").fill("Review the order"); + await page.getByTestId("flow-step-advance-0").fill("download_invoice"); + await page.getByTestId("flow-save").click(); + + await expect(page.getByTestId("flow-editor")).toHaveCount(0); + await expect(page.getByTestId("flow-navigator")).toBeVisible(); + await expect(page.getByTestId("flow-prev")).toBeDisabled(); + await expect(page.getByTestId("flow-next")).toBeDisabled(); + await expect(page.locator("[data-project-canvas]")).toContainText(/order/i); + + // The last step's advance action walks PAST the end: "flow complete" is a + // quiet, named state, not an empty canvas. (The record-detail surface's first + // button dispatches download_invoice; the shadcn renderer does not yet + // compose its child Text into an accessible name, so the click targets the + // first enabled canvas button and the action log proves which fired.) + await page.getByTestId("registry-shadcn").click(); + await page.locator("[data-project-canvas] button:not([disabled])").first().click(); + await expect(page.getByTestId("action-log")).toContainText("download_invoice"); + const complete = page.getByTestId("flow-complete"); + await expect(complete).toContainText("Flow complete"); + await expect(complete).toContainText("Order walkthrough"); + + // A completed flow is revisitable — picking a step re-enters the walk. + await page.getByTestId("flow-step-step.review-the-order").click(); + await expect(page.getByTestId("flow-complete")).toHaveCount(0); + await expect(page.locator("[data-project-canvas]")).toContainText(/order/i); +}); + +test("a step over a surface the emitter refuses shows the refusal, not a blank canvas", async ({ page }) => { + await page.goto("/"); + await newProject(page, "shadcn", "Unrenderable step"); + await page.getByTestId("nav-preview").click(); + + // The step picker offers refused surfaces too, labelled as such — a flow may + // legitimately name a screen the design system cannot draw yet. + await page.getByTestId("new-flow").click(); + await page.getByTestId("flow-name").fill("Wayfinding"); + const picker = page.getByTestId("flow-step-surface"); + await expect(picker.locator('option[value="ex.docs-article-trail"]')).toContainText("can't be emitted"); + await picker.selectOption("ex.docs-article-trail"); + await page.getByTestId("flow-add-step").click(); + await page.getByTestId("flow-step-title-0").fill("Docs trail"); + await page.getByTestId("flow-save").click(); + + const refusal = page.getByTestId("flow-step-unrenderable"); + await expect(refusal).toContainText("This step’s surface can’t render"); + // The EMITTER's own reason travels to the step — not a generic apology, and + // not the dangling-reference wording (the reference resolves fine). + await expect(refusal.locator(".af-empty__body")).not.toBeEmpty(); + await expect(refusal).not.toContainText("which is not in this project's surfaces"); + await expect(page.locator("[data-project-canvas]")).toHaveCount(0); + + // The flow itself is well-formed: the reference resolves, so flow-lint has + // nothing to say — the refusal belongs to the surface, not to the flow. + await page.getByTestId("nav-validate").click(); + await page.getByTestId("run-validate").click(); + await expect(page.getByTestId("flows-summary")).toContainText("1 flow · 1 step · flow checks PASS"); +}); diff --git a/e2e/composer-governance.spec.ts b/e2e/composer-governance.spec.ts new file mode 100644 index 0000000..e6a2186 --- /dev/null +++ b/e2e/composer-governance.spec.ts @@ -0,0 +1,175 @@ +import { expect, test, type Page } from "@playwright/test"; +import { newProject } from "./support/composer-browser"; + +/** + * Governance authoring — intents and the four typed rules. + * + * The spec has no expression language by design, so the forms ARE the editor: + * every rule is identifiers and value lists. These specs assert that authoring + * through those forms genuinely changes what the project is checked against — + * the rule fires, the impact panel names the surfaces it fires on, Checks + * agrees, and removing it undoes exactly that. + * + * Read-only by construction (asserted as PRESENTATION, not as editing): + * an existing intent row offers no edit or remove control — an intent can only + * be added, or replaced by re-typing its id — so the intent assertions here + * cover what the row must SHOW. + */ + +const RATIONALE = "Every collection of records must be readable as a table so the columns can be scanned."; + +/** "11 intents · 49 rules govern …" → [11, 49]. */ +async function counts(page: Page): Promise<[number, number]> { + const text = await page.getByTestId("governance-summary").innerText(); + const m = /(\d+) intents? · (\d+) rules?/.exec(text)!; + return [Number(m[1]), Number(m[2])]; +} + +test("an intent needs a description before it can join the contract, and then it governs Build", async ({ page }) => { + await page.goto("/"); + await newProject(page, "shadcn", "Intent authoring"); + await page.getByTestId("nav-governance").click(); + + const [intentsBefore, rulesBefore] = await counts(page); + + // Every listed intent shows what a person needs to choose one: the name it + // is called, the canonical id beside it, and what surfaces under it are FOR. + const existing = page.getByTestId("intent-destructive-action"); + await expect(existing).toContainText("destructive-action"); + await expect(existing).not.toBeEmpty(); + + // An id alone is not an intent: the description is structural, not optional. + await page.getByTestId("intent-id").fill("audit-log"); + await expect(page.getByTestId("add-intent")).toBeDisabled(); + await page.getByTestId("intent-description").fill("too short"); + await expect(page.getByTestId("add-intent")).toBeDisabled(); + + await page.getByTestId("intent-description").fill("Read-only records of who changed what, and when."); + await expect(page.getByTestId("add-intent")).toBeEnabled(); + await page.getByTestId("add-intent").click(); + + // It is a governed context now: listed with its description, counted, and + // offered by Build's context override and the surface editor alike. + const added = page.getByTestId("intent-audit-log"); + await expect(added).toContainText("audit-log"); + await expect(added).toContainText("Read-only records of who changed what, and when."); + expect(await counts(page)).toEqual([intentsBefore + 1, rulesBefore]); + + await page.getByTestId("nav-build").click(); + await page.getByTestId("build-intent").selectOption("audit-log"); + await expect(page.getByTestId("build-intent")).toHaveValue("audit-log"); + await page.getByTestId("nav-surfaces").click(); + await page.getByTestId("new-scenario").click(); + await page.getByTestId("scenario-intent").selectOption("audit-log"); + await expect(page.getByTestId("scenario-intent")).toHaveValue("audit-log"); +}); + +test("a rule cannot be saved without its rationale, and once saved it visibly fires on the surfaces it governs", async ({ page }) => { + await page.goto("/"); + await newProject(page, "shadcn", "Rule authoring"); + await page.getByTestId("nav-governance").click(); + const [, rulesBefore] = await counts(page); + + // A component-choice rule: identifiers only, no expression to write. + await page.getByTestId("rule-id").fill("require-a-table"); + await page.getByTestId("rule-type").selectOption("component-choice"); + await page.getByTestId("rule-require").getByText("table", { exact: true }).click(); + + // The rationale gates the save — "if you cannot say why, it is not yet a rule". + await expect(page.getByTestId("save-rule")).toBeDisabled(); + await expect(page.getByText("rationale first")).toBeVisible(); + await page.getByTestId("rule-rationale").fill(RATIONALE); + await expect(page.getByText("rationale first")).toHaveCount(0); + await expect(page.getByTestId("save-rule")).toBeEnabled(); + await page.getByTestId("save-rule").click(); + + // The rule is in the contract, presented with the two things that make it + // reviewable: what kind of rule it is, and why it exists. + const row = page.getByTestId("rule-rule.require-a-table"); + await expect(row).toContainText("component-choice · must"); + await expect(row).toContainText(RATIONALE); + expect((await counts(page))[1]).toBe(rulesBefore + 1); + + // The impact panel is the point of the whole view: every save re-lints every + // surface, so the author sees what the rule fires on immediately. + await expect(page.getByTestId("impact-ex.delete-account-confirmation")).toContainText("finding(s)"); + await expect(page.getByTestId("impact-ex.delete-account-confirmation")).toContainText("require-a-table"); + await expect(page.getByTestId("impact-ex.support-ticket-queue")).toContainText("clean"); + + // Checks runs the same gates over the same corpus and reaches the same verdict. + await page.getByTestId("nav-validate").click(); + await page.getByTestId("run-validate").click(); + await expect(page.getByTestId("validate-status")).toContainText("FAIL"); + await expect(page.getByTestId("finding-S3-rule.require-a-table").first()).toContainText(RATIONALE); + + // Removing the rule undoes exactly that, in both places. + await page.getByTestId("nav-governance").click(); + await page.getByTestId("rule-rule.require-a-table").getByRole("button", { name: "remove" }).click(); + await expect(page.getByTestId("rule-rule.require-a-table")).toHaveCount(0); + await expect(page.getByTestId("impact-ex.delete-account-confirmation")).toContainText("clean"); + await page.getByTestId("nav-validate").click(); + await page.getByTestId("run-validate").click(); + await expect(page.getByTestId("validate-status")).toContainText("PASS"); + await expect(page.getByTestId("finding-S3-rule.require-a-table")).toHaveCount(0); +}); + +test("the other rule types are form projections too: a required-props rule over a chosen component", async ({ page }) => { + await page.goto("/"); + await newProject(page, "shadcn", "Typed rules"); + await page.getByTestId("nav-governance").click(); + + await page.getByTestId("rule-id").fill("buttons-declare-a-variant"); + await page.getByTestId("rule-type").selectOption("required-props"); + + // The type switch reshapes the form: a component picker appears, and the + // prop rows only exist once a component is chosen. + await expect(page.getByTestId("rule-component")).toBeVisible(); + await expect(page.getByTestId("add-required-prop")).toHaveCount(0); + await page.getByTestId("rule-component").selectOption("button"); + await expect(page.getByTestId("add-required-prop")).toBeVisible(); + + // The dependent control appears only when the requirement is switched on. + await expect(page.getByRole("combobox").filter({ hasText: "on the node itself" })).toHaveCount(0); + await page.getByTestId("rule-required-text").check(); + await expect(page.getByRole("combobox").filter({ hasText: "on the node itself" })).toHaveCount(1); + + // The prop list is the component's OWN declared descriptors — the vocabulary + // constrains the rule, not free text. + await page.getByTestId("add-required-prop").click(); + await page.getByTestId("rule-prop-0").selectOption("variant"); + await expect(page.getByTestId("rule-prop-0")).toHaveValue("variant"); + + await page.getByTestId("rule-rationale").fill("A button's tone must be declared so destructive actions never read as ordinary ones."); + await page.getByTestId("save-rule").click(); + + const row = page.getByTestId("rule-rule.buttons-declare-a-variant"); + await expect(row).toContainText("required-props · must"); + await expect(row).toContainText("destructive actions never read as ordinary ones"); + + // And it genuinely governs: surfaces whose buttons declare no variant now + // carry a finding naming this rule. + const impacted = page.locator('[data-testid^="impact-ex."]').filter({ hasText: "buttons-declare-a-variant" }); + await expect(impacted.first()).toContainText("finding(s)"); +}); + +test("governance edits in a browser project are session-scoped, and the view says so", async ({ page }) => { + await page.goto("/"); + await newProject(page, "shadcn", "Session scope"); + await page.getByTestId("nav-governance").click(); + const [intentsBefore, rulesBefore] = await counts(page); + + // The honesty this pins is the view's own: a browser project has no + // repository to write governance into, and it does not pretend otherwise. + await expect(page.getByText("connect the local agent to save them to your repository")).toBeVisible(); + + await page.getByTestId("intent-id").fill("audit-log"); + await page.getByTestId("intent-description").fill("Read-only records of who changed what, and when."); + await page.getByTestId("add-intent").click(); + expect(await counts(page)).toEqual([intentsBefore + 1, rulesBefore]); + + await page.reload(); + await expect(page.getByTestId("build-prompt")).toBeVisible(); + await page.getByTestId("nav-governance").click(); + expect(await counts(page)).toEqual([intentsBefore, rulesBefore]); + await expect(page.getByTestId("intent-audit-log")).toHaveCount(0); +}); diff --git a/e2e/composer-parity.spec.ts b/e2e/composer-parity.spec.ts new file mode 100644 index 0000000..0f3e8ec --- /dev/null +++ b/e2e/composer-parity.spec.ts @@ -0,0 +1,135 @@ +import { readFileSync } from "node:fs"; +import { expect, test } from "@playwright/test"; +import { connect, demoProject } from "./support/agent-project"; +import { authorSurface } from "./support/composer-browser"; + +/** + * Browser/agent parity, at the PRODUCT level. + * + * The emit seam is already asserted equivalent in unit tests on both sides. + * This is the thing a person can actually feel: author one surface, and the + * gates must reach the same verdict with the same words whether the project + * lives in a repository on disk (the agent writes it) or in the browser (the + * hosted experience owns it). + * + * The two projects are made identical the honest way — the repository project + * is EXPORTED and imported back as a browser project, so both carry byte-equal + * vocabulary rather than two hand-kept copies. + * + * Scope note: the comparison is SURFACE-scoped on purpose. A connected + * repository additionally emits its `surfacesDir`, which a browser project has + * no access to; that documented corpus asymmetry (validation.ts) means the two + * findings TABLES are legitimately different sizes. What must not differ is the + * verdict on the same authored surface, and that is what is compared here. + */ + +/** Violates both of the demo contract's rules: an action-button with no label, + * and a status-report surface with no info-card. */ +const DRAFT = { id: "parity-check", intent: "status-report", title: "Parity check", component: "action-button" } as const; + +/** + * Repair the draft into something both the rules AND the emitter accept: an + * info-card (the intent requires one) wrapping a labelled action-button (the + * card needs a child, the button needs its label). Identical keystrokes in both + * projects — that is the whole point. + */ +async function repairDraft(page: import("@playwright/test").Page): Promise { + await page.getByTestId("node-component-0").selectOption("info-card"); + await page.getByTestId("add-child-0").click(); + await page.getByTestId("node-text-1").fill("Acknowledge"); + await page.getByTestId("node-prop-label").fill("Acknowledge"); +} + +/** Read the live gate panel as the person sees it: findings text, or "clean". */ +async function gateVerdict(page: import("@playwright/test").Page): Promise { + const clean = page.getByTestId("lint-clean"); + if (await clean.count()) return `CLEAN: ${(await clean.innerText()).trim()}`; + return (await page.getByTestId("lint-findings").innerText()).trim(); +} + +/** + * Every Checks row that names one surface, with the id normalized out so two + * differently-named copies of the SAME surface compare directly. Fidelity notes + * count: they are findings about how this surface projects onto A2UI, and they + * must not depend on which door the project came through. + */ +async function surfaceFindings(page: import("@playwright/test").Page, id: string): Promise { + const rows = page.locator('[data-testid^="finding-"]').filter({ hasText: id }); + const texts = await rows.allInnerTexts(); + return texts.map((t) => t.split(id).join("").replace(/\s+/g, " ").trim()).sort(); +} + +test("the same authored surface gets the same gate verdict and findings in a repository project and a browser project", async ({ + page, +}) => { + const project = demoProject(); + await connect(page, project.root); + + /* ---------- repository project (the agent writes the file) ---------- */ + + await page.getByTestId("nav-surfaces").click(); + await authorSurface(page, DRAFT); + const refusedInRepo = await gateVerdict(page); + expect(refusedInRepo).toContain("info-card"); + expect(refusedInRepo).toContain("label"); + await expect(page.getByTestId("save-scenario")).toBeDisabled(); + + // Fix it in place — the same editor, the same live gates, and the emitter + // accepting it too (the gates panel covers S1–S3; the preview panel is where + // an emit refusal shows, and a genuinely clean surface has neither). + await repairDraft(page); + const cleanInRepo = await gateVerdict(page); + expect(cleanInRepo).toContain("CLEAN"); + await expect(page.getByTestId("preview-refused")).toHaveCount(0); + await expect(page.getByTestId("scenario-preview")).toBeVisible(); + await page.getByTestId("save-scenario").click(); + await expect(page.getByTestId("scenario-ex.parity-check")).toContainText("Parity check"); + + // The agent really wrote it: the file on disk is the second witness. + const onDisk = JSON.parse(readFileSync(project.contractPath, "utf8")); + expect(onDisk.examples.map((e: { id: string }) => e.id)).toContain("ex.parity-check"); + + await page.getByTestId("nav-validate").click(); + await page.getByTestId("run-validate").click(); + const repoStatus = await page.getByTestId("validate-status").innerText(); + expect(repoStatus).toContain("PASS"); + const repoFindings = await surfaceFindings(page, "ex.parity-check"); + expect(repoFindings.length).toBeGreaterThan(0); // a vacuous [] would prove nothing + + // Take the project with us — identity plus governed vocabulary, nothing + // machine-specific — and become a BROWSER project with the same vocabulary. + const [download] = await Promise.all([page.waitForEvent("download"), page.getByTestId("project-export").click()]); + const file = (await download.path())!; + + await page.evaluate(() => localStorage.clear()); + await page.reload(); + await expect(page.getByTestId("projects-empty")).toBeVisible(); + await page.getByTestId("import-project-input").setInputFiles(file); + await expect(page.getByTestId("project-context")).toContainText("Imported"); + // The same project, now browser-backed: Build is ready here too, from the + // vocabulary alone. + await expect(page.getByTestId("build-prompt")).toBeVisible(); + + /* ---------- browser project (no agent involved in the save) ---------- */ + + await page.getByTestId("nav-surfaces").click(); + // The surface authored on disk travelled with the project… + await expect(page.getByTestId("scenario-ex.parity-check")).toContainText("Parity check"); + + // …and the SAME draft is judged identically here, word for word. + await authorSurface(page, { ...DRAFT, id: "parity-check-browser" }); + expect(await gateVerdict(page)).toBe(refusedInRepo); + await expect(page.getByTestId("save-scenario")).toBeDisabled(); + + await repairDraft(page); + expect(await gateVerdict(page)).toBe(cleanInRepo); + await expect(page.getByTestId("preview-refused")).toHaveCount(0); + await expect(page.getByTestId("scenario-preview")).toBeVisible(); + await page.getByTestId("save-scenario").click(); + await expect(page.getByTestId("scenario-ex.parity-check-browser")).toContainText("Parity check"); + + await page.getByTestId("nav-validate").click(); + await page.getByTestId("run-validate").click(); + expect(await page.getByTestId("validate-status").innerText()).toBe(repoStatus); + expect(await surfaceFindings(page, "ex.parity-check-browser")).toEqual(repoFindings); +}); diff --git a/e2e/composer-prod-smoke.spec.ts b/e2e/composer-prod-smoke.spec.ts index 5aff3f6..57f0854 100644 --- a/e2e/composer-prod-smoke.spec.ts +++ b/e2e/composer-prod-smoke.spec.ts @@ -303,6 +303,41 @@ test("Settings states provider options honestly and the appearance control appli await expect(page.locator("html")).not.toHaveAttribute("data-theme", "ember"); }); +/** + * With no agent running, "Local AI" is unreachable — and that is a setup step, + * not a product failure. This is the agent-ABSENT half of provider + * configuration; the configured-and-connected half needs a real agent and a + * real endpoint, and lives in composer-settings.spec.ts (agent config). + * + * The agent probe is blocked outright rather than assumed absent, so the + * assertion means the same thing on a developer machine that happens to have + * the agent running as it does in CI and on the deployed site. + */ +test("with no agent running, Local AI is honestly unavailable rather than a dead-looking form", async ({ page }) => { + await page.route((url) => url.hostname === "localhost" && url.port === "8787", (route) => route.abort()); + await page.goto("/"); + await page.getByTestId("nav-settings").click(); + + await expect(page.getByTestId("agent-status")).toContainText("Agent not running"); + + // Onboarding names the two real steps, in order: a runner, then the bridge. + const onboarding = page.getByTestId("local-onboarding"); + await expect(onboarding).toContainText("Ollama"); + await expect(onboarding).toContainText("pnpm --filter agent dev"); + + // Nothing pretends to be configurable: both provider forms are inert, so a + // person cannot type an endpoint that could never be reached. + await expect(page.getByTestId("provider-ollama")).toBeVisible(); + await expect(page.getByTestId("provider-openai")).toBeVisible(); + for (const id of ["ollama-url", "ollama-test", "openai-url", "openai-key", "openai-test"]) { + await expect(page.getByTestId(id)).toBeDisabled(); + } + + // And the path that DOES work here is offered plainly. + await expect(page.getByTestId("provider-model-scripted")).toBeEnabled(); + await expect(page.getByTestId("active-provider")).not.toBeEmpty(); +}); + test("flows: create, walk, advance, persist, and round-trip through export/import", async ({ page }) => { await page.goto("/"); await newProject(page, "shadcn", "Flow walkthrough"); diff --git a/e2e/composer-settings.spec.ts b/e2e/composer-settings.spec.ts new file mode 100644 index 0000000..51fab05 --- /dev/null +++ b/e2e/composer-settings.spec.ts @@ -0,0 +1,164 @@ +import { expect, test, type Page } from "@playwright/test"; + +/** + * Provider configuration — the only path a bring-your-own-inference user takes, + * and until now covered by nothing. + * + * The browser is deliberately not the client of the model: it cannot reach + * `localhost`, so the local agent is the bridge and it owns the endpoint and any + * secret. That shape is what these specs pin — configuration reaches a real + * endpoint through the real agent, discovery is real, a failure says what + * happened, and a credential typed here never lands in browser storage. + * + * The endpoint is e2e/serve-provider.mjs: a real HTTP server speaking Ollama's + * and OpenAI's discovery protocols. It generates nothing, and no spec here runs + * a build against it — configuration is the surface under test, and a model call + * would not be honest in CI. + * + * The agent-ABSENT half of this surface lives where it is true by construction: + * composer-prod-smoke.spec.ts, which runs with no agent at all. + */ + +const FIXTURE = "http://localhost:3314"; +/** Must match EXPECTED_KEY in e2e/serve-provider.mjs. */ +const FIXTURE_KEY = "sk-fixture-credential-0123456789"; +/** Nothing listens here — the honest "unreachable endpoint" case. */ +const DEAD_ENDPOINT = "http://localhost:3399"; + +async function openSettings(page: Page): Promise { + await page.goto("/"); + await page.getByTestId("nav-settings").click(); + // Local configuration is gated on the bridge being there; wait for the probe + // rather than for a timer. + await expect(page.getByTestId("agent-status")).toContainText("Agent connected"); + await expect(page.getByTestId("local-onboarding")).toHaveCount(0); +} + +test("Ollama: a real connection test discovers the models, and choosing one configures the provider for Build", async ({ page }) => { + await openSettings(page); + + await page.getByTestId("ollama-url").fill(FIXTURE); + await page.getByTestId("ollama-test").click(); + + // Discovery is the agent's, over a real socket: two usable models, with the + // embedding model filtered out on the way through. + const status = page.getByTestId("ollama-status"); + await expect(status).toContainText("Connected — 2 models"); + await expect(page.getByTestId("ollama-models")).not.toContainText("fixture-embedding"); + await expect(page.getByTestId("ollama-model-fixture-coder:7b")).toBeVisible(); + + // Choosing a model IS the configuration step. + await page.getByTestId("ollama-model-fixture-coder:7b").click(); + await expect(page.getByTestId("active-provider")).toContainText("Local · fixture-coder:7b (ollama)"); + await expect(page.getByTestId("ollama-model-fixture-coder:7b")).toHaveAttribute("aria-pressed", "true"); + + // It persists as a preference — endpoint and model, no secret — and the form + // reopens on what was configured. + const stored = await page.evaluate(() => JSON.parse(localStorage.getItem("composer.providers.v1") ?? "null")); + expect(stored).toMatchObject({ ollama: { baseUrl: FIXTURE, model: "fixture-coder:7b" }, active: "ollama:fixture-coder:7b" }); + await page.reload(); + await page.getByTestId("nav-settings").click(); + await expect(page.getByTestId("ollama-url")).toHaveValue(FIXTURE); + await expect(page.getByTestId("active-provider")).toContainText("fixture-coder:7b"); + + // And it reaches the place it matters: a project opens with that provider + // already selected, named honestly in the privacy line. + await page.getByTestId("nav-projects").click(); + await page.getByTestId("new-project-name").fill("Bring your own model"); + await page.getByTestId("new-source-shadcn").click(); + await page.getByTestId("new-project-create").click(); + await expect(page.getByTestId("build-model")).toHaveValue("ollama:fixture-coder:7b"); + await expect(page.getByTestId("build-privacy")).toContainText("ollama:fixture-coder:7b"); +}); + +test("OpenAI-compatible: a base URL discovers models, and an endpoint that lists none falls back to typing one", async ({ page }) => { + await openSettings(page); + + await page.getByTestId("openai-url").fill(`${FIXTURE}/v1`); + await page.getByTestId("openai-test").click(); + await expect(page.getByTestId("openai-status")).toContainText("Connected — 2 models"); + await page.getByTestId("openai-model-fixture-openai-b").click(); + await expect(page.getByTestId("active-provider")).toContainText("Local · fixture-openai-b (openai)"); + + // Plenty of local servers don't implement /models. Reachable-but-unlistable is + // not a failure: the UI asks for the model id instead of inventing one. + await expect(page.getByTestId("openai-manual")).toHaveCount(0); + await page.getByTestId("openai-url").fill(`${FIXTURE}/silent`); + await page.getByTestId("openai-test").click(); + await expect(page.getByTestId("openai-status")).toContainText("Connected"); + await expect(page.getByTestId("openai-status")).not.toContainText("model"); + const manual = page.getByTestId("openai-manual"); + await expect(manual).toContainText("doesn’t list models"); + await expect(page.getByTestId("openai-model-use")).toBeDisabled(); + + await page.getByTestId("openai-model-input").fill("hand-typed-model"); + await page.getByTestId("openai-model-use").click(); + await expect(page.getByTestId("active-provider")).toContainText("Local · hand-typed-model (openai)"); + const stored = await page.evaluate(() => JSON.parse(localStorage.getItem("composer.providers.v1") ?? "null")); + expect(stored).toMatchObject({ openai: { baseUrl: `${FIXTURE}/silent`, model: "hand-typed-model" } }); +}); + +test("a failed connection test says what happened and configures nothing", async ({ page }) => { + await openSettings(page); + + // First configure something real, so the failure has something to NOT undo. + await page.getByTestId("ollama-url").fill(FIXTURE); + await page.getByTestId("ollama-test").click(); + await page.getByTestId("ollama-model-fixture-small:1b").click(); + await expect(page.getByTestId("active-provider")).toContainText("fixture-small:1b"); + + await page.getByTestId("ollama-url").fill(DEAD_ENDPOINT); + await page.getByTestId("ollama-test").click(); + + // The endpoint's own failure, in plain words — not a stack trace, not silence. + const status = page.getByTestId("ollama-status"); + await expect(status).toContainText("is the server running at that address?"); + await expect(page.getByTestId("ollama-models")).toHaveCount(0); + + // Nothing was reconfigured by a failure. + await expect(page.getByTestId("active-provider")).toContainText("fixture-small:1b"); + const stored = await page.evaluate(() => JSON.parse(localStorage.getItem("composer.providers.v1") ?? "null")); + expect(stored.ollama).toEqual({ baseUrl: FIXTURE, model: "fixture-small:1b" }); +}); + +test("a credential is used by the agent and never written to browser storage", async ({ page }) => { + await openSettings(page); + + // The endpoint requires a key. Without one it refuses, and the refusal is + // reported as the endpoint's, not swallowed. + await page.getByTestId("openai-url").fill(`${FIXTURE}/keyed`); + await page.getByTestId("openai-test").click(); + await expect(page.getByTestId("openai-status")).toContainText("401"); + await expect(page.getByTestId("openai-models")).toHaveCount(0); + + // With the key, the SAME endpoint answers — which is only possible if the + // exact credential travelled browser → agent → provider. The fixture rejects + // anything else, so this is a real end-to-end proof, not a rendering check. + await page.getByTestId("openai-key").fill(FIXTURE_KEY); + await page.getByTestId("openai-test").click(); + await expect(page.getByTestId("openai-status")).toContainText("Connected — 1 model"); + await page.getByTestId("openai-model-credentialed-model").click(); + await expect(page.getByTestId("active-provider")).toContainText("Local · credentialed-model (openai)"); + + // THE INVARIANT: the browser is a client, not a vault. Configuring a provider + // persists the endpoint and the model; the credential exists only in memory + // for the session, so nothing durable in this browser holds it. + const dump = await page.evaluate(() => { + const read = (s: Storage) => Object.fromEntries(Object.keys(s).map((k) => [k, s.getItem(k) ?? ""])); + return { local: read(localStorage), session: read(sessionStorage) }; + }); + expect(JSON.stringify(dump)).not.toContain(FIXTURE_KEY); + expect(JSON.stringify(dump)).not.toContain("sk-"); + expect(JSON.parse(dump.local["composer.providers.v1"])).toMatchObject({ + openai: { baseUrl: `${FIXTURE}/keyed`, model: "credentialed-model" }, + active: "openai:credentialed-model", + }); + expect(dump.local["composer.providers.v1"]).not.toContain("apiKey"); + + // A reload keeps the configuration and drops the secret — the honest cost of + // never storing it, and the UI's stated contract. + await page.reload(); + await page.getByTestId("nav-settings").click(); + await expect(page.getByTestId("active-provider")).toContainText("credentialed-model"); + await expect(page.getByTestId("openai-key")).toHaveValue(""); +}); diff --git a/e2e/composer-surfaces.spec.ts b/e2e/composer-surfaces.spec.ts new file mode 100644 index 0000000..3127b6a --- /dev/null +++ b/e2e/composer-surfaces.spec.ts @@ -0,0 +1,143 @@ +import { expect, test } from "@playwright/test"; +import { authorSurface, newProject } from "./support/composer-browser"; + +/** + * Surface authoring (the "Surfaces" view) — the second thing a new team does + * after their first build, and until now covered by nothing. + * + * A saved surface IS a contract worked example, so authoring here writes the + * project's few-shot corpus AND its preview corpus at once. That is what these + * specs assert: not that the editor renders, but that what a person authors + * becomes project content they can then see, walk, and check — and that a + * surface which fails a gate is refused out loud rather than saved quietly. + * + * Agent-free by construction (the hosted experience). No model call anywhere: + * authoring is forms over the contract's vocabulary. + */ + +test("author a surface: it lints clean, previews live, saves, and lists under its human title", async ({ page }) => { + await page.goto("/"); + await newProject(page, "shadcn", "Surface authoring"); + await page.getByTestId("nav-surfaces").click(); + + // A fresh project owns nothing yet, and says so — the design system's own + // surfaces are a separate, clearly-labelled corpus. + await expect(page.getByTestId("scenarios-none-yet")).toBeVisible(); + await expect(page.getByTestId("scenarios-reference")).toContainText("Reference surfaces"); + + await authorSurface(page, { + id: "save-preferences", + intent: "preference-settings", + title: "Save preferences", + prompt: "a button that saves the person's notification preferences", + component: "button", + text: "Save preferences", + }); + + // The gates run on every keystroke, and the preview renders the draft — the + // two panels that make authoring honest rather than hopeful. + await expect(page.getByTestId("lint-clean")).toContainText("S1 S2 S3 clean"); + await expect(page.getByTestId("scenario-preview")).toContainText("Save preferences"); + + await expect(page.getByTestId("save-scenario")).toBeEnabled(); + await page.getByTestId("save-scenario").click(); + + // Back in the listing: the TITLE leads, the canonical id stays beside it for + // audit, and the project no longer says it owns nothing. + const row = page.getByTestId("scenario-ex.save-preferences"); + await expect(row).toContainText("Save preferences"); + await expect(row).toContainText("ex.save-preferences"); + await expect(row).toContainText("preference-settings"); + await expect(page.getByTestId("scenarios-none-yet")).toHaveCount(0); + + // It is project content now: Preview offers it among the project's OWN + // surfaces and renders it through the project's design system. + await page.getByTestId("nav-preview").click(); + const surface = page.getByTestId("surface-ex.save-preferences"); + await expect(surface).toContainText("Save preferences"); + await surface.click(); + await expect(page.locator("[data-project-canvas]")).toContainText("Save preferences"); + + // And it survives a reload — an authored surface is persisted project work, + // not session scratch. + await page.reload(); + await expect(page.getByTestId("build-prompt")).toBeVisible(); + await page.getByTestId("nav-surfaces").click(); + await expect(page.getByTestId("scenario-ex.save-preferences")).toContainText("Save preferences"); +}); + +test("a should-level rule warns without blocking, and Checks reports it against the authored surface", async ({ page }) => { + await page.goto("/"); + await newProject(page, "shadcn", "Warned surface"); + await page.getByTestId("nav-surfaces").click(); + + // shadcn's rule.spinner-names-what-is-loading is severity `should`: a spinner + // with no aria-label is a warning, and a warning must not behave like a bar. + await authorSurface(page, { + id: "orders-loading", + intent: "loading-state", + title: "Orders are loading", + component: "spinner", + }); + + const findings = page.getByTestId("lint-findings"); + await expect(findings).toContainText("spinner-names-what-is-loading"); + await expect(page.getByTestId("save-scenario")).toBeEnabled(); + await page.getByTestId("save-scenario").click(); + await expect(page.getByTestId("scenario-ex.orders-loading")).toContainText("Orders are loading"); + + // Checks sees the surface the person just authored: the run stays PASS + // (warnings do not fail the contract) and the finding names the surface. + await page.getByTestId("nav-validate").click(); + await page.getByTestId("run-validate").click(); + await expect(page.getByTestId("validate-status")).toContainText("PASS"); + const row = page.locator('[data-testid^="finding-S3-"]').filter({ hasText: "ex.orders-loading" }); + await expect(row).toContainText("spinner"); + await expect(row).toContainText("warn"); +}); + +test("a surface that fails a gate is refused out loud, saves nothing, and unblocks when it is fixed", async ({ page }) => { + await page.goto("/"); + await newProject(page, "shadcn", "Refused surface"); + await page.getByTestId("nav-surfaces").click(); + + // rule.button-carries-text is severity `must`, and applies to every intent. + await authorSurface(page, { id: "unnamed-button", intent: "preference-settings", component: "button" }); + + await expect(page.getByTestId("lint-findings")).toContainText("button-carries-text"); + await expect(page.getByTestId("save-scenario")).toBeDisabled(); + await expect(page.getByText("gates first")).toBeVisible(); + + // Leaving the editor saves NOTHING — a refusal is a refusal, not a draft + // quietly written under the person's nose. + await page.getByRole("button", { name: "← surfaces" }).click(); + await expect(page.getByTestId("scenario-ex.unnamed-button")).toHaveCount(0); + await expect(page.getByTestId("scenarios-none-yet")).toBeVisible(); + await page.reload(); + await expect(page.getByTestId("build-prompt")).toBeVisible(); + await page.getByTestId("nav-surfaces").click(); + await expect(page.getByTestId("scenario-ex.unnamed-button")).toHaveCount(0); + + // Fixing the violation clears the gate and unlocks the save — the gate is + // live guidance, not a dead end. + await authorSurface(page, { id: "unnamed-button", intent: "preference-settings", component: "button", text: "Delete workspace" }); + await expect(page.getByTestId("lint-clean")).toBeVisible(); + await expect(page.getByTestId("save-scenario")).toBeEnabled(); + await page.getByTestId("save-scenario").click(); + await expect(page.getByTestId("scenario-ex.unnamed-button")).toContainText("ex.unnamed-button"); +}); + +test("an emitter refusal is stated in the preview panel instead of a blank canvas", async ({ page }) => { + await page.goto("/"); + await newProject(page, "shadcn", "Emitter refusal"); + await page.getByTestId("nav-surfaces").click(); + + // `alert-title` is a declared SUB-component of `alert`; standing alone at the + // root it violates S2 and the emitter refuses to produce anything for it. + await authorSurface(page, { id: "stray-alert-title", intent: "preference-settings", component: "alert-title", text: "Stray title" }); + + await expect(page.getByTestId("lint-findings")).toContainText("sub-component 'alert-title'"); + await expect(page.getByTestId("preview-refused")).toContainText("emitter refusal"); + await expect(page.getByTestId("scenario-preview")).toHaveCount(0); + await expect(page.getByTestId("save-scenario")).toBeDisabled(); +}); diff --git a/e2e/serve-provider.mjs b/e2e/serve-provider.mjs new file mode 100644 index 0000000..6ff3534 --- /dev/null +++ b/e2e/serve-provider.mjs @@ -0,0 +1,56 @@ +/** + * A fixture LOCAL PROVIDER for the Settings suite. + * + * "Local AI" in Composer means *any* OpenAI-compatible server or Ollama on the + * user's own machine, reached through the local agent. This is one such server: + * a real HTTP endpoint speaking the two discovery protocols the agent actually + * probes, so `POST /provider/test` runs its real code against a real socket. + * + * It deliberately does NOT generate. Provider *configuration* — reach the + * endpoint, discover the models, choose one, carry a credential — is what + * Settings owns and what the specs drive. Generation is a model call, and no + * spec makes one. + * + * GET /api/tags Ollama discovery. Includes one embedding model, which + * the agent is expected to filter out. + * GET /v1/models OpenAI-compatible discovery, two models. + * GET /silent/models 404 — "reachable, but doesn't enumerate": the agent + * reports ok with an empty list and the UI falls back to + * manual model entry. + * GET /keyed/models 401 without the exact expected credential, one model + * with it. Proves a key typed in the browser reached the + * provider THROUGH the agent — the only path it may take. + */ +import { createServer } from "node:http"; + +const port = Number(process.env.PROVIDER_FIXTURE_PORT ?? 3314); + +/** Must match FIXTURE_KEY in e2e/composer-settings.spec.ts. */ +const EXPECTED_KEY = "sk-fixture-credential-0123456789"; + +const send = (res, status, body) => { + res.writeHead(status, { "content-type": "application/json", "access-control-allow-origin": "*" }); + res.end(JSON.stringify(body)); +}; + +createServer((req, res) => { + const path = (req.url ?? "/").split("?")[0]; + + if (path === "/api/tags") { + return send(res, 200, { + models: [{ name: "fixture-small:1b" }, { name: "fixture-coder:7b" }, { name: "fixture-embedding:latest" }], + }); + } + if (path === "/v1/models") { + return send(res, 200, { object: "list", data: [{ id: "fixture-openai-a" }, { id: "fixture-openai-b" }] }); + } + if (path === "/silent/models") { + return send(res, 404, { error: "this server does not enumerate models" }); + } + if (path === "/keyed/models") { + const auth = req.headers.authorization ?? ""; + if (auth !== `Bearer ${EXPECTED_KEY}`) return send(res, 401, { error: "missing or wrong credential" }); + return send(res, 200, { object: "list", data: [{ id: "credentialed-model" }] }); + } + return send(res, 404, { error: "not found" }); +}).listen(port, () => console.log(`provider fixture on http://localhost:${port}`)); diff --git a/e2e/support/composer-browser.ts b/e2e/support/composer-browser.ts new file mode 100644 index 0000000..714ebaa --- /dev/null +++ b/e2e/support/composer-browser.ts @@ -0,0 +1,69 @@ +/** + * Shared moves for the AGENT-FREE composer suites (playwright.composer-smoke + * config, project "composer-product"): the hosted experience a first-time + * visitor gets — the real static export, no agent, the scripted provider. + * + * Every helper drives the product the way a person does (hub → source → + * create), never by seeding storage: the point of these suites is that the + * paths a new team takes first actually work, so the setup has to be one of + * those paths too. + * + * No helper here makes a model call. `scriptedBuild` pins the provider to + * "scripted" before every run — the deterministic replay adapter — so the + * suites stay CI-safe and repeatable by construction. + * + * `authorSurface` is the one MODE-NEUTRAL helper here: the surface editor is + * the same view in a browser project and a connected repository, which is + * exactly what composer-parity.spec.ts relies on. + */ +import { expect, type Page } from "@playwright/test"; + +/** Start a fresh project from a governed reference and land in Build. */ +export async function newProject(page: Page, source: "shadcn" | "astryx", name: string): Promise { + await page.getByTestId("nav-projects").click(); + await page.getByTestId("new-project-name").fill(name); + await page.getByTestId(`new-source-${source}`).click(); + await page.getByTestId("new-project-create").click(); + await expect(page.getByTestId("build-prompt")).toBeVisible(); + await expect(page.getByTestId("project-context")).toContainText(name); +} + +/** + * One deterministic scripted build, waited on by its OUTCOME rather than a + * timer. `turn` is the 1-based turn number the run will produce (the thread + * numbers turns in order), so a caller building twice waits on the right one. + */ +export async function scriptedBuild(page: Page, intent: string, goal: string, turn = 1): Promise { + await page.getByTestId("build-model").selectOption("scripted"); + await page.getByTestId("build-intent").selectOption(intent); + await page.getByTestId("build-prompt").fill(goal); + await page.getByTestId("build-run").click(); + await expect(page.getByTestId(`build-gate-summary-${turn}`)).toContainText("Follows your design-system rules", { timeout: 30_000 }); +} + +/** Build one surface and accept it into the project, returning its minted id. */ +export async function buildAndAccept(page: Page, intent: string, goal: string, turn = 1): Promise { + await scriptedBuild(page, intent, goal, turn); + await page.getByTestId(`build-accept-${turn}`).click(); + const accepted = page.getByTestId(`build-accepted-${turn}`); + await expect(accepted).toContainText(/ex\.chat-\d+/); + return (await accepted.innerText()).match(/ex\.chat-\d+/)![0]; +} + +/** + * Open the surface editor on a NEW surface and fill it in. The root node is a + * single component — enough to exercise the vocabulary constraint, the live + * gates, and the live preview, which is what these specs are about. + */ +export async function authorSurface( + page: Page, + fields: { id: string; intent: string; title?: string; prompt?: string; component: string; text?: string }, +): Promise { + await page.getByTestId("new-scenario").click(); + await page.getByTestId("scenario-id").fill(fields.id); + await page.getByTestId("scenario-intent").selectOption(fields.intent); + if (fields.title !== undefined) await page.getByLabel("Surface title").fill(fields.title); + if (fields.prompt !== undefined) await page.getByTestId("scenario-prompt").fill(fields.prompt); + await page.getByTestId("node-component-0").selectOption(fields.component); + if (fields.text !== undefined) await page.getByTestId("node-text-0").fill(fields.text); +} diff --git a/playwright.composer-agent.config.ts b/playwright.composer-agent.config.ts index 73b2a35..35a5f1e 100644 --- a/playwright.composer-agent.config.ts +++ b/playwright.composer-agent.config.ts @@ -16,6 +16,11 @@ import { defineConfig } from "@playwright/test"; * One worker, no retries: every spec mutates real files through a * single-session agent, and a retry would replay a decision against * already-decided state — exactly the flakiness these tests exist to catch. + * + * The third webServer is a fixture LOCAL PROVIDER (e2e/serve-provider.mjs): + * a real endpoint speaking Ollama's and OpenAI's discovery protocols, so the + * agent's provider test runs against a real socket. It generates nothing — no + * spec in this repo makes a model call. */ const BASE_URL = process.env.COMPOSER_AGENT_URL ?? "http://localhost:3312"; @@ -31,7 +36,14 @@ export default defineConfig({ projects: [ { name: "composer-agent", - testMatch: ["composer-agent.spec.ts", "composer-agent-a11y.spec.ts", "composer-build.spec.ts", "composer-build-a11y.spec.ts"], + testMatch: [ + "composer-agent.spec.ts", + "composer-agent-a11y.spec.ts", + "composer-build.spec.ts", + "composer-build-a11y.spec.ts", + "composer-settings.spec.ts", + "composer-parity.spec.ts", + ], }, ], webServer: [ @@ -45,5 +57,10 @@ export default defineConfig({ port: 8787, reuseExistingServer: !process.env.CI, }, + { + command: "node e2e/serve-provider.mjs", + port: 3314, + reuseExistingServer: !process.env.CI, + }, ], }); diff --git a/playwright.composer-smoke.config.ts b/playwright.composer-smoke.config.ts index 87fe35b..145d3c3 100644 --- a/playwright.composer-smoke.config.ts +++ b/playwright.composer-smoke.config.ts @@ -11,6 +11,13 @@ import { defineConfig } from "@playwright/test"; * The public-URL post-deploy smoke stays in playwright.composer-production.config.ts * (docs/deployment.md): same spec, different target — local artifact here, the * live Worker there. + * + * A SECOND project — "composer-product" — runs the agent-free product suites + * (surface authoring, flows, governance) on the same artifact and the same + * server. They are not smoke: they are the first-adoption surfaces a new team + * touches, driven end to end with the scripted provider (no model call, ever). + * They stay out of the deployed-URL config on purpose — they author and delete + * project data, which the post-deploy smoke must not do to a live site. */ const PORT = Number(process.env.COMPOSER_SMOKE_PORT ?? 3313); @@ -22,7 +29,13 @@ export default defineConfig({ baseURL: `http://localhost:${PORT}`, contextOptions: { reducedMotion: "reduce" }, }, - projects: [{ name: "composer-smoke", testMatch: "composer-prod-smoke.spec.ts" }], + projects: [ + { name: "composer-smoke", testMatch: "composer-prod-smoke.spec.ts" }, + { + name: "composer-product", + testMatch: ["composer-surfaces.spec.ts", "composer-flows.spec.ts", "composer-governance.spec.ts"], + }, + ], webServer: [ { command: `PORT=${PORT} node e2e/serve-composer.mjs`, diff --git a/playwright.config.ts b/playwright.config.ts index 7f56222..912600e 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -40,9 +40,14 @@ export default defineConfig({ name: "studio", // prod-smoke asserts the DEPLOYED site's properties (no local agent, // injected analytics); it runs via playwright.production.config.ts only. - // composer-prod-smoke targets the DEPLOYED composer app; it runs via - // playwright.composer-production.config.ts only. - testIgnore: ["prod-smoke.spec.ts", "composer-prod-smoke.spec.ts", "composer-agent.spec.ts", "composer-agent-a11y.spec.ts", "composer-build.spec.ts", "composer-build-a11y.spec.ts", "tour-first-run.spec.ts"], + // + // EVERY composer spec is ignored by PATTERN, not by name. This config + // serves apps/web on 3311; the composer app is served by the composer + // configs on their own ports, so a composer-*.spec.ts here could only + // ever fail. The name list this replaced was the same shape as the CI + // filter list of #81: adding a composer spec silently opted it into the + // wrong suite until someone remembered to extend the list. + testIgnore: ["prod-smoke.spec.ts", "composer-*.spec.ts", "tour-first-run.spec.ts"], use: { storageState: { cookies: [],