From fa8d31c82f30db2c66f3fa5ef78389f4e6e3e322 Mon Sep 17 00:00:00 2001 From: Anthony Sayge Date: Mon, 13 Jul 2026 19:44:05 -0500 Subject: [PATCH] Added complexity tiers for speed. Added QA tooling. --- .claude/agents/qa-engineer.md | 36 ++++-- .claude/agents/technical-writer.md | 14 ++- .claude/agents/ux-designer.md | 26 ++++ .claude/commands/feature-resume.md | 7 +- .claude/commands/feature.md | 90 +++++++++++-- .claude/hooks/enforce-lanes.js | 2 +- .claude/qa.json | 11 ++ CLAUDE.md | 51 ++++++-- plugin/agents/qa-engineer.md | 36 ++++-- plugin/agents/technical-writer.md | 14 ++- plugin/agents/ux-designer.md | 26 ++++ plugin/commands/feature-resume.md | 7 +- plugin/commands/feature.md | 90 +++++++++++-- plugin/hooks/enforce-lanes.js | 2 +- plugin/templates/lanes.json | 2 +- plugin/templates/qa.json | 17 +++ plugin/tools/browser.js | 196 +++++++++++++++++++++++++++++ plugin/tools/http-check.js | 173 +++++++++++++++++++++++++ tools/browser.js | 167 ++++++++++++++++++------ tools/http-check.js | 173 +++++++++++++++++++++++++ 20 files changed, 1045 insertions(+), 95 deletions(-) create mode 100644 .claude/qa.json create mode 100644 plugin/templates/qa.json create mode 100644 plugin/tools/browser.js create mode 100644 plugin/tools/http-check.js create mode 100644 tools/http-check.js diff --git a/.claude/agents/qa-engineer.md b/.claude/agents/qa-engineer.md index cbb5d9a..74eff98 100644 --- a/.claude/agents/qa-engineer.md +++ b/.claude/agents/qa-engineer.md @@ -43,18 +43,34 @@ conventions. You own everything under `app/test/` and `docs/qa/`. ## Mode 1 — Verify a feature -Input: the spec path and the list of files the dev agents changed. +Input: the spec path, the list of files the dev agents changed, and the run's +**Tier** (1 Trivial | 2 Standard | 3 Complex) when the orchestrator provides one. + +**Run the standardized gates first, then fill gaps.** Do not re-derive an approach +from scratch each run — the declared gates are the fast path. -Process: 1. Read the spec's acceptance criteria. -2. Write/update `docs/qa/test-plans/NNN-.md` following - `docs/qa/test-plans/001-status-dashboard.md`: a table mapping every - criterion to an automated test or a manual check. -3. Author automated tests under `app/test/` for everything automatable. -4. Run `node --test app/test/*.test.js` with Bash. For behavior tests can't reach, - verify manually (start server on a spare PORT, curl, inspect; for UI, - statically check the HTML/CSS against the criteria — semantics, ARIA, - token usage). +2. **Run the declared gates from `.claude/qa.json`** (its `checks` map), in order, + and record each gate's pass/fail. If `.claude/qa.json` is absent, use the + zero-dependency defaults: `lint` = `node --check` on each changed `.js` file; + `unit` = `node --test app/test/*.test.js`; `contract` = + `node tools/http-check.js ` (drive it from the spec's + API-contract block, server on a spare PORT); `e2e` = + `node tools/browser.js check ` against the rendered DOM. + A `null` gate is skipped. **You RUN declared quality tools; you NEVER install + them** — if a criterion needs a tool that isn't available, that is a dependency + decision (backend-developer installer lane + `dependencies.allow`); raise it + under OPEN QUESTIONS rather than installing anything. +3. Author automated tests under `app/test/` **only for criteria the gates don't + already cover**. For behavior tests can't reach, verify manually (start server + on a spare PORT, curl, inspect; for UI, statically check the HTML/CSS against + the criteria — semantics, ARIA, token usage). +4. Write/update `docs/qa/test-plans/NNN-.md` following + `docs/qa/test-plans/001-status-dashboard.md`: a table mapping every criterion + to a gate, an automated test, or a manual check. **Tier depth:** on **Tier 1**, + run the gates + a targeted check of the changed behavior and **skip authoring a + new test-plan doc** (still file defects as normal); on Tier 2/3, author the full + plan and gap-filling tests. 5. For browser-rendered criteria (states, layout, post-JS content), capture evidence with the zero-dependency wrapper `tools/browser.js`: start the server on a spare PORT, then diff --git a/.claude/agents/technical-writer.md b/.claude/agents/technical-writer.md index 1b9aaea..04566b9 100644 --- a/.claude/agents/technical-writer.md +++ b/.claude/agents/technical-writer.md @@ -52,10 +52,16 @@ codebase before it goes in. ## Input The orchestrator passes you: the feature ask, the spec path -(`docs/specs/NNN-.md`), the changed-files lists from the dev handoffs, and -the design-review verdict. Ground yourself first: read the spec, skim the -changed files, and run `git diff` / `git log` to see what this feature actually -added or changed. Document from that reality, not from the ask alone. +(`docs/specs/NNN-.md`), the changed-files lists from the dev handoffs, the +design-review verdict, and the run's complexity **Tier** (see `/feature` +Phase 0.5) when set. Ground yourself first: read the spec, skim the changed +files, and run `git diff` / `git log` to see what this feature actually added or +changed. Document from that reality, not from the ask alone. + +**Tier depth.** On **Tier 1** (trivial change), do an **overview/README touch +only**: update just the README sections the change actually affects and skip the +per-feature note (Mode 2). On Tier 2/3, run the full Mode 1 + Mode 2 (README plus +`docs/project/` overview and a per-feature note). ## Mode 1 — README.md (create or update, idempotently) diff --git a/.claude/agents/ux-designer.md b/.claude/agents/ux-designer.md index f0ccd68..8b30a17 100644 --- a/.claude/agents/ux-designer.md +++ b/.claude/agents/ux-designer.md @@ -41,6 +41,22 @@ your specs describe behavior and appearance, never implementation code. - A genuine hard block is a STOP-and-raise, not a workaround: halt and raise it under OPEN QUESTIONS with the exact blocking message. +The orchestrator sets the **depth** of your work from the run's complexity Tier +(see `/feature` Phase 0.5). Match whichever mode/depth it asks for: + +## Mode 0 — Quick brief (Tier 1, trivial changes) + +Input: the ask, `docs/design-system.md`, existing specs. + +Output: a *short* `docs/specs/NNN-.md` — Overview (1–2 lines describing the +exact change) · numbered, individually testable Acceptance criteria · a one-line +API contract only if the change touches the backend. **No wireframe, no new +tokens.** A quick brief is for copy/label/config tweaks and single existing-token +value swaps only. If you find the change actually needs a new token, a layout +change, or a new flow, **do not improvise it into the brief** — that is an +escalator: raise it under OPEN QUESTIONS so the run moves up a tier and gets a +full spec (Mode 1). Return STATUS `ready-for-dev` when the brief stands on its own. + ## Mode 1 — Design (start of a feature) Input: the product owner's feature ask, `docs/design-system.md`, existing @@ -54,6 +70,10 @@ backend can build in parallel) · Design tokens used · Accessibility requirements · Acceptance criteria (numbered, individually testable — the QA engineer builds the test plan from these). +**Tier 2 (light) variant:** reuse existing layout patterns and tokens; include a +wireframe only if the layout actually changes; keep every section but keep it tight. +Full depth (all sections, new wireframes/tokens as needed) is Tier 3. + ## Mode 2 — Design verification (after QA passes) Input: the spec path, the implemented files under `app/public/`, and — when @@ -71,6 +91,12 @@ suspected file), the checklist, and a verdict. Verify at minimum: layout vs wireframe, all specified states present, token-only styling, semantics/ landmarks, ARIA requirements, focus states, text-not-color-alone. +**Tier 1/2 (delta) variant:** when the orchestrator asks for a delta review, +verify only the changed surface named in the prompt against the spec and design +system, rather than re-reviewing the whole UI. (Tier 1 design verification is +usually skipped entirely by the orchestrator unless the change touched layout or +tokens.) Full-UI verification is Tier 3. + Verdict rules: `APPROVED` only with zero blocker/major findings (minor findings may be noted and approved). Otherwise `CHANGES REQUIRED`. diff --git a/.claude/commands/feature-resume.md b/.claude/commands/feature-resume.md index 6dfb163..aacc4d9 100644 --- a/.claude/commands/feature-resume.md +++ b/.claude/commands/feature-resume.md @@ -9,8 +9,11 @@ number/slug; if empty, pick the most recently modified state file whose You are the orchestrator, resuming a `/feature` run that was interrupted (session ended, context compacted, or stopped mid-loop). -1. Read the pipeline state file: `Status`, `Current phase`, the loop - counters, the phase log, and the open-questions log. +1. Read the pipeline state file: `Status`, `Current phase`, the recorded + `Tier` and `Lightened/skipped phases`, the loop counters, the phase log, and + the open-questions log. Honor the recorded Tier's per-phase depth for the rest + of the run (do not re-triage) — the "Tier gate" lines in + `.claude/commands/feature.md` govern what each remaining phase does. 2. Re-ground yourself in the artifacts it references: the spec in `docs/specs/`, any test plan, defects (check each defect's `Status` field), design review, and evidence directory. diff --git a/.claude/commands/feature.md b/.claude/commands/feature.md index 37a4b0b..8833eaa 100644 --- a/.claude/commands/feature.md +++ b/.claude/commands/feature.md @@ -31,8 +31,8 @@ project subagents (`ux-designer`, `frontend-developer`, `backend-developer`, product owner retargeting `.claude/lanes.json`, never a workaround. - After each phase, update the pipeline state file (Phase 0) before starting the next phase — it is what makes the run resumable via `/feature-resume`. -- Do not skip phases, do not do an agent's work yourself, do not commit unless - the product owner asks. +- Do not skip phases beyond what the run's recorded Tier permits (Phase 0.5), + do not do an agent's work yourself, do not commit unless the product owner asks. ## Phase 0 — Pipeline state file @@ -45,7 +45,9 @@ otherwise NNN = highest in `docs/specs/` + 1. Create # Pipeline state — NNN- Status: in-progress -Current phase: 1 — Design +Current phase: 0.5 — Triage +Tier: (pending — set in Phase 0.5) +Lightened/skipped phases: (set after triage) QA fix-loop iteration: 0/2 Design fix-loop iteration: 0/2 @@ -80,12 +82,54 @@ product owner to add a `.claude/lanes.json` (copy the plugin's `templates/lanes.json` and adapt the paths) before continuing. Never write `.claude/lanes.json` yourself — it is the product owner's protected file. +## Phase 0.5 — Triage & tiering + +Score the ask's complexity, record it, and announce it — the tier sets how much +each later phase does (see the "Tier gate" line under each phase). **Start at +Tier 3 and earn down** — bias toward more process, not less. + +Tier-3 escalators — if ANY holds, the run is **Tier 3 (Complex)**: +- net-new UI structure, layout, or user flow; +- a new or changed design token / visual language; +- a multi-surface change (net-new UI *and* a new/changed backend contract); +- a security, data-handling, auth, or persistence tradeoff. + +Otherwise: +- **Tier 1 — Trivial**: confined to copy/text/labels, a single existing-token + value swap, or a config/static-content tweak. No new structure, no new + endpoint, no logic/state change. +- **Tier 2 — Standard**: a self-contained feature reusing existing layout + patterns and tokens; may add a small endpoint or field; no net-new design + language. + +Rules: +- **Tie-break: when torn between two tiers, pick the higher one.** +- **Ambiguity is never scored away.** If you cannot confidently place the ask, + treat that as a Tier-3 signal or raise it under OPEN QUESTIONS (the hard-stop + rule still applies) — never a silent downgrade. +- **Product-owner override wins.** If the ask carries `--tier=N`, or the owner + says "treat this as trivial/standard/full", use that tier and log the override + in the Open questions log. + +Set `Tier: N — ` and `Lightened/skipped phases: ` in the +state file, update `Current phase` to `1 — Design`, and **state the tier and what +it lightens/skips to the product owner in your first status message** before +running Phase 1. + ## Phase 1 — Design Invoke `ux-designer` (Mode 1) with the ask and instruct it to write `docs/specs/NNN-.md`. Wait, then read the spec. Proceed only when STATUS is `ready-for-dev`. +**Tier gate.** Tier 3: `ux-designer` Mode 1 (full spec). Tier 2: Mode 1 *light* — +reuse existing layout and tokens; include a wireframe only if the layout changes. +Tier 1: `ux-designer` **Mode 0** — a quick brief (numbered, testable acceptance +criteria + a one-line API contract if backend is involved; no wireframe, no new +tokens). All three still write `docs/specs/NNN-.md` and must return STATUS +`ready-for-dev`; if a Tier-0/1 change turns out to need a new token or layout, +that is an escalator — the designer raises it and the run moves up a tier. + ## Phase 2 — Implementation (parallel) In a single message, invoke both: @@ -96,12 +140,25 @@ This is safe because the spec pre-defines the API contract and the two agents own disjoint files. Exception: if the spec leaves the API contract undefined, run backend first and pass its contract block to the frontend afterwards. +**Tier gate.** Invoke only the dev agent(s) the change actually needs (still in a +single parallel message when both apply). A Tier 1 copy/style change is usually +frontend-only; a Tier 1 API tweak is usually backend-only. + ## Phase 3 — QA Invoke `qa-engineer` (Mode 1) with the spec path and the changed-files lists -from both dev handoffs. Remind it to capture rendered evidence with -`tools/browser.js` into `docs/qa/evidence/NNN-/` for browser-behavior -criteria — the design verification in Phase 5 consumes those screenshots. +from both dev handoffs. The qa-engineer first runs the declared gates from +`.claude/qa.json` (or the zero-dep defaults) and reports their results, then +authors tests for whatever the gates don't cover. Remind it to capture rendered +evidence with `tools/browser.js` into `docs/qa/evidence/NNN-/` for +browser-behavior criteria — the design verification in Phase 5 consumes those +screenshots. + +**Tier gate.** Every tier runs the declared gates — QA is never zeroed. Tier 1: +gates + a targeted check of the changed behavior, and **no new test-plan doc**. +Tier 2: gates + authored tests for the gaps + a test plan. Tier 3: the above + +full evidence capture. Pass the run's tier to the qa-engineer so it picks the +right depth. ## Phase 4 — QA fix loop (max 2 iterations) @@ -125,6 +182,12 @@ evidence directory `docs/qa/evidence/NNN-/` (if QA produced one). It writes `docs/design-reviews/NNN-.md` and returns APPROVED or CHANGES REQUIRED. +**Tier gate.** Tier 3: `ux-designer` Mode 2 (full). Tier 2: Mode 2 *delta* — +verify only the changed surface. Tier 1: **skip design verification entirely +unless the change touched layout or tokens**; if it did, run Mode 2 delta. When +skipped, record it in the state file's `Lightened/skipped phases` and go straight +to Phase 7. + ## Phase 6 — Design fix loop (max 2 iterations) If CHANGES REQUIRED — batch findings by owning area exactly as in Phase 4 @@ -133,6 +196,9 @@ If CHANGES REQUIRED — batch findings by owning area exactly as in Phase 4 re-review. If still not APPROVED after 2 iterations, stop and report the open findings to the product owner. +**Tier gate.** This loop only runs when Phase 5 ran and returned CHANGES +REQUIRED. If Phase 5 was skipped (Tier 1, no layout/token change), skip Phase 6. + ## Phase 7 — Documentation Once the design is APPROVED, the feature is at its final shipped state — the @@ -142,6 +208,10 @@ both dev handoffs, and the design-review verdict. It creates or idempotently updates the root `README.md` and the project docs under `docs/project/` (overview + a per-feature note) to match what actually shipped. +**Tier gate.** Tier 3/2: `technical-writer` Mode 1 + Mode 2 (README + overview + +a per-feature note). Tier 1: **overview/README touch only** — update just the +sections the change affects; skip the per-feature note. + The writer's lane covers `README.md` and `docs/project/` only — do not author these docs yourself. Treat its handoff footer like any other agent's: if its OPEN QUESTIONS is anything other than "none" (e.g. a missing `.env.example`, a @@ -151,8 +221,10 @@ answers. ## Phase 8 — Report to the product owner -Present a summary table: phase · agent · artifacts written · status. State -the final QA verdict, design verdict, and the docs updated in Phase 7, list any -open items, and ask the product owner for acceptance. On acceptance, set the +Present a summary table: phase · agent · artifacts written · status. Lead with +the run's **Tier and the phases it lightened or skipped** (so the owner sees what +was traded for speed), then state the QA gate results and final QA verdict, the +design verdict (or that verification was skipped and why), and the docs updated in +Phase 7. List any open items and ask the product owner for acceptance. On acceptance, set the state file's `Status: complete` (and the backlog row to `done`, if one exists). Do not commit unless they ask. diff --git a/.claude/hooks/enforce-lanes.js b/.claude/hooks/enforce-lanes.js index d5b969d..dcc4644 100644 --- a/.claude/hooks/enforce-lanes.js +++ b/.claude/hooks/enforce-lanes.js @@ -57,7 +57,7 @@ const DEFAULT_LANES = { }; // Subagents may not touch the enforcement layer itself. -const DEFAULT_PROTECTED = ['.claude/settings.json', '.claude/hooks/', '.claude/lanes.json']; +const DEFAULT_PROTECTED = ['.claude/settings.json', '.claude/hooks/', '.claude/lanes.json', '.claude/qa.json']; // Bash heuristics, subagents only. (Package managers are handled separately by // the dependency-policy logic, not by this static list.) diff --git a/.claude/qa.json b/.claude/qa.json new file mode 100644 index 0000000..b76c687 --- /dev/null +++ b/.claude/qa.json @@ -0,0 +1,11 @@ +{ + "_note": "Standardized QA gates, product-owner-owned (protected, like lanes.json). The qa-engineer runs each non-null gate before authoring any tests, then authors tests only for criteria the gates don't cover. null = skip that gate. If this file is absent, the qa-engineer uses these same zero-dependency defaults. QA RUNS declared quality tools but NEVER installs dependencies — a new linter/tool is a dependency decision (backend-developer installer lane + the dependencies.allow allowlist), raised under OPEN QUESTIONS.", + "checks": { + "lint": "node --check", + "typecheck": null, + "unit": "node --test app/test/*.test.js", + "contract": "node tools/http-check.js", + "e2e": "node tools/browser.js check", + "smoke": null + } +} diff --git a/CLAUDE.md b/CLAUDE.md index 8e94a48..5bc6fd3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -69,14 +69,22 @@ Enforcement note). - Run the tests: `node --test app/test/*.test.js` - Capture rendered evidence (QA): `node tools/browser.js dom ` (post-JS DOM) and `node tools/browser.js shot [WxH]` — zero-dep - headless-Chromium wrapper; exits 2 with a clear message when no browser is - installed (fall back to static checks). + headless-Chromium wrapper (discovers Chrome/Chromium on PATH, via `CHROME_BIN`, + or in the standard macOS app bundle); exits 2 with a clear message when no + browser is installed (fall back to static checks). +- Assert against the rendered DOM (QA): `node tools/browser.js check + ` — evaluates `{contains}` / `{matches}` / `{absent}` + assertions against the post-JS DOM; non-zero exit on any failure. +- Check the API contract (QA): `node tools/http-check.js ` + — zero-dep `node:http` runner asserting status + JSON-subset shape per endpoint; + non-zero exit on any failure. ## Workflow rules 1. Features flow: UX spec → frontend + backend (parallel) → QA → QA fix loop → UX design verification → design fix loop → documentation → product-owner - acceptance. + acceptance. Each phase's **depth scales with the run's complexity Tier** + (rule 8) — the flow is constant, but how much each phase does is not. 2. No agent leaves its lane: UX never writes app code; devs never edit `docs/specs/` or `docs/design-system.md`; QA never fixes product code; devs never write tests; the technical-writer writes only `README.md` and @@ -99,6 +107,27 @@ Enforcement note). restarting. 7. Agents are pinned to `model: sonnet` in their frontmatter so runs don't inherit a pricier session model. Change deliberately, not per-run. +8. **Complexity tiering.** At `/feature` Phase 0.5 the orchestrator scores the + ask into a Tier — **1 Trivial | 2 Standard | 3 Complex** — recorded in the + pipeline state file and announced to the product owner. The tier scales each + phase's depth (per the "Tier gate" lines in `/feature`): Tier 1 gets a UX quick + brief, gates-only QA, no design verification unless layout/tokens changed, and + an overview/README-only doc touch; Tier 2 runs everything in a lighter "delta" + mode; Tier 3 is the full pipeline. Guardrails that make this safe and are not + negotiable: **start at Tier 3 and earn down** (bias toward more process); + **on a tie, pick the higher tier**; **OPEN QUESTIONS stays tier-independent** + (ambiguity always escalates, never gets scored away); and **QA is never + zeroed** — every tier runs the declared gates. The product owner may override + the tier (`/feature --tier=N` or in words); the override is logged. +9. **Standardized QA gates.** Quality gates are declared in `.claude/qa.json` + (product-owner-owned, protected) as a `checks` map — `lint`, `typecheck`, + `unit`, `contract`, `e2e`, `smoke` (`null` = skip). The qa-engineer runs the + declared gates first, then authors tests only for the gaps. When the file is + absent it falls back to zero-dep defaults (`node --check`, `node --test`, + `tools/http-check.js`, `tools/browser.js check`). **QA runs declared quality + tools but never installs dependencies** — adding a linter/tool is a dependency + decision (backend installer lane + `dependencies.allow`), raised under OPEN + QUESTIONS. ## Artifact conventions (the handoff contract) @@ -122,8 +151,14 @@ explicit file paths. ux-designer during design verification. - Pipeline state: `docs/pipeline/NNN-.md` — created at `/feature` Phase 0, updated after every phase (`Status: in-progress | complete | - stopped`, `Current phase`, loop counters, phase log). Read by - `/feature-resume` and surfaced at session start by a hook. + stopped`, `Current phase`, `Tier` and `Lightened/skipped phases` (set at + Phase 0.5), loop counters, phase log). Read by `/feature-resume` (which honors + the recorded Tier rather than re-triaging) and surfaced at session start by a + hook. +- QA gate config: `.claude/qa.json` — the product-owner-owned `checks` map the + qa-engineer runs before authoring tests (protected; see the Enforcement note). + Absent ⇒ zero-dep defaults. A copyable template ships at + `plugin/templates/qa.json`. - Backlog / spec-number registry: `docs/backlog.md`, managed via `/backlog`. When features run concurrently (e.g. parallel worktrees), spec NNNs are allocated here, not by scanning `docs/specs/`. @@ -143,8 +178,10 @@ Lane boundaries are mechanically enforced, not just prompted: (e.g. the ux-designer has no Bash). - `.claude/hooks/enforce-lanes.js` (a PreToolUse hook registered in `.claude/settings.json`) blocks `Write`/`Edit` outside each agent's lane and - blocks subagents from touching the enforcement layer itself. Unknown agents - fail closed — register new agents in its lane table. + blocks subagents from touching the enforcement layer itself — the protected set + is `.claude/settings.json`, `.claude/hooks/`, `.claude/lanes.json`, and + `.claude/qa.json` (all product-owner-only). Unknown agents fail closed — + register new agents in its lane table. - The same hook enforces the **Dependency policy**: it reads `dependencies.allow` / `dependencies.installers` from `.claude/lanes.json` and, per package-manager Bash command and per `package.json` write, permits diff --git a/plugin/agents/qa-engineer.md b/plugin/agents/qa-engineer.md index cbb5d9a..74eff98 100644 --- a/plugin/agents/qa-engineer.md +++ b/plugin/agents/qa-engineer.md @@ -43,18 +43,34 @@ conventions. You own everything under `app/test/` and `docs/qa/`. ## Mode 1 — Verify a feature -Input: the spec path and the list of files the dev agents changed. +Input: the spec path, the list of files the dev agents changed, and the run's +**Tier** (1 Trivial | 2 Standard | 3 Complex) when the orchestrator provides one. + +**Run the standardized gates first, then fill gaps.** Do not re-derive an approach +from scratch each run — the declared gates are the fast path. -Process: 1. Read the spec's acceptance criteria. -2. Write/update `docs/qa/test-plans/NNN-.md` following - `docs/qa/test-plans/001-status-dashboard.md`: a table mapping every - criterion to an automated test or a manual check. -3. Author automated tests under `app/test/` for everything automatable. -4. Run `node --test app/test/*.test.js` with Bash. For behavior tests can't reach, - verify manually (start server on a spare PORT, curl, inspect; for UI, - statically check the HTML/CSS against the criteria — semantics, ARIA, - token usage). +2. **Run the declared gates from `.claude/qa.json`** (its `checks` map), in order, + and record each gate's pass/fail. If `.claude/qa.json` is absent, use the + zero-dependency defaults: `lint` = `node --check` on each changed `.js` file; + `unit` = `node --test app/test/*.test.js`; `contract` = + `node tools/http-check.js ` (drive it from the spec's + API-contract block, server on a spare PORT); `e2e` = + `node tools/browser.js check ` against the rendered DOM. + A `null` gate is skipped. **You RUN declared quality tools; you NEVER install + them** — if a criterion needs a tool that isn't available, that is a dependency + decision (backend-developer installer lane + `dependencies.allow`); raise it + under OPEN QUESTIONS rather than installing anything. +3. Author automated tests under `app/test/` **only for criteria the gates don't + already cover**. For behavior tests can't reach, verify manually (start server + on a spare PORT, curl, inspect; for UI, statically check the HTML/CSS against + the criteria — semantics, ARIA, token usage). +4. Write/update `docs/qa/test-plans/NNN-.md` following + `docs/qa/test-plans/001-status-dashboard.md`: a table mapping every criterion + to a gate, an automated test, or a manual check. **Tier depth:** on **Tier 1**, + run the gates + a targeted check of the changed behavior and **skip authoring a + new test-plan doc** (still file defects as normal); on Tier 2/3, author the full + plan and gap-filling tests. 5. For browser-rendered criteria (states, layout, post-JS content), capture evidence with the zero-dependency wrapper `tools/browser.js`: start the server on a spare PORT, then diff --git a/plugin/agents/technical-writer.md b/plugin/agents/technical-writer.md index 1b9aaea..04566b9 100644 --- a/plugin/agents/technical-writer.md +++ b/plugin/agents/technical-writer.md @@ -52,10 +52,16 @@ codebase before it goes in. ## Input The orchestrator passes you: the feature ask, the spec path -(`docs/specs/NNN-.md`), the changed-files lists from the dev handoffs, and -the design-review verdict. Ground yourself first: read the spec, skim the -changed files, and run `git diff` / `git log` to see what this feature actually -added or changed. Document from that reality, not from the ask alone. +(`docs/specs/NNN-.md`), the changed-files lists from the dev handoffs, the +design-review verdict, and the run's complexity **Tier** (see `/feature` +Phase 0.5) when set. Ground yourself first: read the spec, skim the changed +files, and run `git diff` / `git log` to see what this feature actually added or +changed. Document from that reality, not from the ask alone. + +**Tier depth.** On **Tier 1** (trivial change), do an **overview/README touch +only**: update just the README sections the change actually affects and skip the +per-feature note (Mode 2). On Tier 2/3, run the full Mode 1 + Mode 2 (README plus +`docs/project/` overview and a per-feature note). ## Mode 1 — README.md (create or update, idempotently) diff --git a/plugin/agents/ux-designer.md b/plugin/agents/ux-designer.md index f0ccd68..8b30a17 100644 --- a/plugin/agents/ux-designer.md +++ b/plugin/agents/ux-designer.md @@ -41,6 +41,22 @@ your specs describe behavior and appearance, never implementation code. - A genuine hard block is a STOP-and-raise, not a workaround: halt and raise it under OPEN QUESTIONS with the exact blocking message. +The orchestrator sets the **depth** of your work from the run's complexity Tier +(see `/feature` Phase 0.5). Match whichever mode/depth it asks for: + +## Mode 0 — Quick brief (Tier 1, trivial changes) + +Input: the ask, `docs/design-system.md`, existing specs. + +Output: a *short* `docs/specs/NNN-.md` — Overview (1–2 lines describing the +exact change) · numbered, individually testable Acceptance criteria · a one-line +API contract only if the change touches the backend. **No wireframe, no new +tokens.** A quick brief is for copy/label/config tweaks and single existing-token +value swaps only. If you find the change actually needs a new token, a layout +change, or a new flow, **do not improvise it into the brief** — that is an +escalator: raise it under OPEN QUESTIONS so the run moves up a tier and gets a +full spec (Mode 1). Return STATUS `ready-for-dev` when the brief stands on its own. + ## Mode 1 — Design (start of a feature) Input: the product owner's feature ask, `docs/design-system.md`, existing @@ -54,6 +70,10 @@ backend can build in parallel) · Design tokens used · Accessibility requirements · Acceptance criteria (numbered, individually testable — the QA engineer builds the test plan from these). +**Tier 2 (light) variant:** reuse existing layout patterns and tokens; include a +wireframe only if the layout actually changes; keep every section but keep it tight. +Full depth (all sections, new wireframes/tokens as needed) is Tier 3. + ## Mode 2 — Design verification (after QA passes) Input: the spec path, the implemented files under `app/public/`, and — when @@ -71,6 +91,12 @@ suspected file), the checklist, and a verdict. Verify at minimum: layout vs wireframe, all specified states present, token-only styling, semantics/ landmarks, ARIA requirements, focus states, text-not-color-alone. +**Tier 1/2 (delta) variant:** when the orchestrator asks for a delta review, +verify only the changed surface named in the prompt against the spec and design +system, rather than re-reviewing the whole UI. (Tier 1 design verification is +usually skipped entirely by the orchestrator unless the change touched layout or +tokens.) Full-UI verification is Tier 3. + Verdict rules: `APPROVED` only with zero blocker/major findings (minor findings may be noted and approved). Otherwise `CHANGES REQUIRED`. diff --git a/plugin/commands/feature-resume.md b/plugin/commands/feature-resume.md index 6dfb163..aacc4d9 100644 --- a/plugin/commands/feature-resume.md +++ b/plugin/commands/feature-resume.md @@ -9,8 +9,11 @@ number/slug; if empty, pick the most recently modified state file whose You are the orchestrator, resuming a `/feature` run that was interrupted (session ended, context compacted, or stopped mid-loop). -1. Read the pipeline state file: `Status`, `Current phase`, the loop - counters, the phase log, and the open-questions log. +1. Read the pipeline state file: `Status`, `Current phase`, the recorded + `Tier` and `Lightened/skipped phases`, the loop counters, the phase log, and + the open-questions log. Honor the recorded Tier's per-phase depth for the rest + of the run (do not re-triage) — the "Tier gate" lines in + `.claude/commands/feature.md` govern what each remaining phase does. 2. Re-ground yourself in the artifacts it references: the spec in `docs/specs/`, any test plan, defects (check each defect's `Status` field), design review, and evidence directory. diff --git a/plugin/commands/feature.md b/plugin/commands/feature.md index 37a4b0b..8833eaa 100644 --- a/plugin/commands/feature.md +++ b/plugin/commands/feature.md @@ -31,8 +31,8 @@ project subagents (`ux-designer`, `frontend-developer`, `backend-developer`, product owner retargeting `.claude/lanes.json`, never a workaround. - After each phase, update the pipeline state file (Phase 0) before starting the next phase — it is what makes the run resumable via `/feature-resume`. -- Do not skip phases, do not do an agent's work yourself, do not commit unless - the product owner asks. +- Do not skip phases beyond what the run's recorded Tier permits (Phase 0.5), + do not do an agent's work yourself, do not commit unless the product owner asks. ## Phase 0 — Pipeline state file @@ -45,7 +45,9 @@ otherwise NNN = highest in `docs/specs/` + 1. Create # Pipeline state — NNN- Status: in-progress -Current phase: 1 — Design +Current phase: 0.5 — Triage +Tier: (pending — set in Phase 0.5) +Lightened/skipped phases: (set after triage) QA fix-loop iteration: 0/2 Design fix-loop iteration: 0/2 @@ -80,12 +82,54 @@ product owner to add a `.claude/lanes.json` (copy the plugin's `templates/lanes.json` and adapt the paths) before continuing. Never write `.claude/lanes.json` yourself — it is the product owner's protected file. +## Phase 0.5 — Triage & tiering + +Score the ask's complexity, record it, and announce it — the tier sets how much +each later phase does (see the "Tier gate" line under each phase). **Start at +Tier 3 and earn down** — bias toward more process, not less. + +Tier-3 escalators — if ANY holds, the run is **Tier 3 (Complex)**: +- net-new UI structure, layout, or user flow; +- a new or changed design token / visual language; +- a multi-surface change (net-new UI *and* a new/changed backend contract); +- a security, data-handling, auth, or persistence tradeoff. + +Otherwise: +- **Tier 1 — Trivial**: confined to copy/text/labels, a single existing-token + value swap, or a config/static-content tweak. No new structure, no new + endpoint, no logic/state change. +- **Tier 2 — Standard**: a self-contained feature reusing existing layout + patterns and tokens; may add a small endpoint or field; no net-new design + language. + +Rules: +- **Tie-break: when torn between two tiers, pick the higher one.** +- **Ambiguity is never scored away.** If you cannot confidently place the ask, + treat that as a Tier-3 signal or raise it under OPEN QUESTIONS (the hard-stop + rule still applies) — never a silent downgrade. +- **Product-owner override wins.** If the ask carries `--tier=N`, or the owner + says "treat this as trivial/standard/full", use that tier and log the override + in the Open questions log. + +Set `Tier: N — ` and `Lightened/skipped phases: ` in the +state file, update `Current phase` to `1 — Design`, and **state the tier and what +it lightens/skips to the product owner in your first status message** before +running Phase 1. + ## Phase 1 — Design Invoke `ux-designer` (Mode 1) with the ask and instruct it to write `docs/specs/NNN-.md`. Wait, then read the spec. Proceed only when STATUS is `ready-for-dev`. +**Tier gate.** Tier 3: `ux-designer` Mode 1 (full spec). Tier 2: Mode 1 *light* — +reuse existing layout and tokens; include a wireframe only if the layout changes. +Tier 1: `ux-designer` **Mode 0** — a quick brief (numbered, testable acceptance +criteria + a one-line API contract if backend is involved; no wireframe, no new +tokens). All three still write `docs/specs/NNN-.md` and must return STATUS +`ready-for-dev`; if a Tier-0/1 change turns out to need a new token or layout, +that is an escalator — the designer raises it and the run moves up a tier. + ## Phase 2 — Implementation (parallel) In a single message, invoke both: @@ -96,12 +140,25 @@ This is safe because the spec pre-defines the API contract and the two agents own disjoint files. Exception: if the spec leaves the API contract undefined, run backend first and pass its contract block to the frontend afterwards. +**Tier gate.** Invoke only the dev agent(s) the change actually needs (still in a +single parallel message when both apply). A Tier 1 copy/style change is usually +frontend-only; a Tier 1 API tweak is usually backend-only. + ## Phase 3 — QA Invoke `qa-engineer` (Mode 1) with the spec path and the changed-files lists -from both dev handoffs. Remind it to capture rendered evidence with -`tools/browser.js` into `docs/qa/evidence/NNN-/` for browser-behavior -criteria — the design verification in Phase 5 consumes those screenshots. +from both dev handoffs. The qa-engineer first runs the declared gates from +`.claude/qa.json` (or the zero-dep defaults) and reports their results, then +authors tests for whatever the gates don't cover. Remind it to capture rendered +evidence with `tools/browser.js` into `docs/qa/evidence/NNN-/` for +browser-behavior criteria — the design verification in Phase 5 consumes those +screenshots. + +**Tier gate.** Every tier runs the declared gates — QA is never zeroed. Tier 1: +gates + a targeted check of the changed behavior, and **no new test-plan doc**. +Tier 2: gates + authored tests for the gaps + a test plan. Tier 3: the above + +full evidence capture. Pass the run's tier to the qa-engineer so it picks the +right depth. ## Phase 4 — QA fix loop (max 2 iterations) @@ -125,6 +182,12 @@ evidence directory `docs/qa/evidence/NNN-/` (if QA produced one). It writes `docs/design-reviews/NNN-.md` and returns APPROVED or CHANGES REQUIRED. +**Tier gate.** Tier 3: `ux-designer` Mode 2 (full). Tier 2: Mode 2 *delta* — +verify only the changed surface. Tier 1: **skip design verification entirely +unless the change touched layout or tokens**; if it did, run Mode 2 delta. When +skipped, record it in the state file's `Lightened/skipped phases` and go straight +to Phase 7. + ## Phase 6 — Design fix loop (max 2 iterations) If CHANGES REQUIRED — batch findings by owning area exactly as in Phase 4 @@ -133,6 +196,9 @@ If CHANGES REQUIRED — batch findings by owning area exactly as in Phase 4 re-review. If still not APPROVED after 2 iterations, stop and report the open findings to the product owner. +**Tier gate.** This loop only runs when Phase 5 ran and returned CHANGES +REQUIRED. If Phase 5 was skipped (Tier 1, no layout/token change), skip Phase 6. + ## Phase 7 — Documentation Once the design is APPROVED, the feature is at its final shipped state — the @@ -142,6 +208,10 @@ both dev handoffs, and the design-review verdict. It creates or idempotently updates the root `README.md` and the project docs under `docs/project/` (overview + a per-feature note) to match what actually shipped. +**Tier gate.** Tier 3/2: `technical-writer` Mode 1 + Mode 2 (README + overview + +a per-feature note). Tier 1: **overview/README touch only** — update just the +sections the change affects; skip the per-feature note. + The writer's lane covers `README.md` and `docs/project/` only — do not author these docs yourself. Treat its handoff footer like any other agent's: if its OPEN QUESTIONS is anything other than "none" (e.g. a missing `.env.example`, a @@ -151,8 +221,10 @@ answers. ## Phase 8 — Report to the product owner -Present a summary table: phase · agent · artifacts written · status. State -the final QA verdict, design verdict, and the docs updated in Phase 7, list any -open items, and ask the product owner for acceptance. On acceptance, set the +Present a summary table: phase · agent · artifacts written · status. Lead with +the run's **Tier and the phases it lightened or skipped** (so the owner sees what +was traded for speed), then state the QA gate results and final QA verdict, the +design verdict (or that verification was skipped and why), and the docs updated in +Phase 7. List any open items and ask the product owner for acceptance. On acceptance, set the state file's `Status: complete` (and the backlog row to `done`, if one exists). Do not commit unless they ask. diff --git a/plugin/hooks/enforce-lanes.js b/plugin/hooks/enforce-lanes.js index d5b969d..dcc4644 100644 --- a/plugin/hooks/enforce-lanes.js +++ b/plugin/hooks/enforce-lanes.js @@ -57,7 +57,7 @@ const DEFAULT_LANES = { }; // Subagents may not touch the enforcement layer itself. -const DEFAULT_PROTECTED = ['.claude/settings.json', '.claude/hooks/', '.claude/lanes.json']; +const DEFAULT_PROTECTED = ['.claude/settings.json', '.claude/hooks/', '.claude/lanes.json', '.claude/qa.json']; // Bash heuristics, subagents only. (Package managers are handled separately by // the dependency-policy logic, not by this static list.) diff --git a/plugin/templates/lanes.json b/plugin/templates/lanes.json index 6978081..b74daed 100644 --- a/plugin/templates/lanes.json +++ b/plugin/templates/lanes.json @@ -22,7 +22,7 @@ "hint": "You document the shipped state; code, specs, tests, and other docs are other lanes — raise an OPEN QUESTION if you are blocked." } }, - "protected": [".claude/settings.json", ".claude/hooks/", ".claude/lanes.json"], + "protected": [".claude/settings.json", ".claude/hooks/", ".claude/lanes.json", ".claude/qa.json"], "dependencies": { "allow": [], "installers": ["backend-developer"] diff --git a/plugin/templates/qa.json b/plugin/templates/qa.json new file mode 100644 index 0000000..50d43eb --- /dev/null +++ b/plugin/templates/qa.json @@ -0,0 +1,17 @@ +{ + "_note": "Copy to .claude/qa.json and adapt to your stack. Standardized QA gates, product-owner-owned (protected, like lanes.json). The qa-engineer runs each non-null gate under `checks` before authoring any tests, then authors tests only for gaps. null = skip. If .claude/qa.json is absent the qa-engineer falls back to the zero-dependency defaults shown here. QA RUNS declared quality tools but NEVER installs dependencies — adding a linter/tool is a dependency decision (backend-developer installer lane + dependencies.allow), raised under OPEN QUESTIONS.", + "_examples": { + "lint": "eslint .", + "typecheck": "tsc --noEmit", + "unit": "vitest run", + "e2e": "playwright test" + }, + "checks": { + "lint": "node --check", + "typecheck": null, + "unit": "node --test app/test/*.test.js", + "contract": "node tools/http-check.js", + "e2e": "node tools/browser.js check", + "smoke": null + } +} diff --git a/plugin/tools/browser.js b/plugin/tools/browser.js new file mode 100644 index 0000000..428d5c3 --- /dev/null +++ b/plugin/tools/browser.js @@ -0,0 +1,196 @@ +#!/usr/bin/env node +// Zero-dependency headless-Chromium wrapper for QA evidence + assertions. +// +// Usage: +// node tools/browser.js dom # post-JS DOM → stdout +// node tools/browser.js shot [WxH] # screenshot (default 1280x800) +// node tools/browser.js check # assert against the post-JS DOM +// +// Exit codes: 0 success · 1 usage/capture/assertion failure · 2 no browser found +// (on exit 2, fall back to static HTML/CSS checks and note it in the test plan). +'use strict'; + +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +// Executes the page's JS for up to 3s of virtual time before capturing, so +// fetch-driven states (loading → success) have settled. +const BASE_FLAGS = [ + '--headless=new', + '--no-sandbox', + '--disable-gpu', + '--disable-dev-shm-usage', + '--hide-scrollbars', + '--virtual-time-budget=3000', +]; + +// Absolute launcher paths worth probing before PATH lookups — covers common +// macOS app-bundle installs that never land on PATH (the reason QA silently +// fell back to static checks on macOS before). +const ABSOLUTE_CANDIDATES = [ + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + '/Applications/Chromium.app/Contents/MacOS/Chromium', + '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary', +]; + +function findBrowser() { + const candidates = []; + if (process.env.CHROME_BIN) candidates.push(process.env.CHROME_BIN); + const pw = process.env.PLAYWRIGHT_BROWSERS_PATH; + if (pw && fs.existsSync(pw)) { + for (const entry of fs.readdirSync(pw).sort().reverse()) { + if (!entry.startsWith('chromium')) continue; + candidates.push( + path.join(pw, entry), + path.join(pw, entry, 'chrome-linux', 'chrome'), + path.join(pw, entry, 'chrome-linux', 'headless_shell'), + ); + } + } + candidates.push(...ABSOLUTE_CANDIDATES); + candidates.push('chromium', 'chromium-browser', 'google-chrome', 'chrome'); + for (const c of candidates) { + try { + if (spawnSync(c, ['--version'], { stdio: 'ignore', timeout: 10000 }).status === 0) { + return c; + } + } catch { + // not runnable — try the next candidate + } + } + return null; +} + +function usage() { + process.stderr.write( + 'usage: node tools/browser.js dom \n' + + ' node tools/browser.js shot [WxH]\n' + + ' node tools/browser.js check \n', + ); + process.exit(1); +} + +// Dump the post-JS DOM for `url` as a string, or exit with the right code on +// failure (2 = no browser, 1 = capture error). +function dumpDom(url) { + const browser = findBrowser(); + if (!browser) { + process.stderr.write( + 'browser unavailable — no Chromium/Chrome found (checked CHROME_BIN, ' + + 'PLAYWRIGHT_BROWSERS_PATH, macOS app bundles, PATH). Fall back to static checks.\n', + ); + process.exit(2); + } + const res = spawnSync(browser, [...BASE_FLAGS, '--dump-dom', url], { + encoding: 'utf8', + timeout: 30000, + }); + if (res.error || res.status !== 0) { + process.stderr.write( + `capture failed (${res.error ? res.error.message : `exit ${res.status}`})\n${res.stderr || ''}`, + ); + process.exit(1); + } + return res.stdout; +} + +// `check` mode: evaluate a small assertions list against the post-JS DOM. +// assertions.json is a JSON array (or an object with an `assertions` array) of: +// { "name": "...", "contains": "substring in the rendered DOM" } +// { "matches": "regex source" } // tested against the DOM string +// { "absent": "substring that must NOT appear" } +// Prints PASS/FAIL per assertion; exits 1 if any fail. +function runChecks(url, assertionsPath) { + let doc; + try { + doc = JSON.parse(fs.readFileSync(assertionsPath, 'utf8')); + } catch (err) { + process.stderr.write(`cannot read assertions file: ${err.message}\n`); + process.exit(1); + } + const assertions = Array.isArray(doc) ? doc : Array.isArray(doc.assertions) ? doc.assertions : null; + if (!assertions) { + process.stderr.write('assertions file must be a JSON array or an object with an `assertions` array\n'); + process.exit(1); + } + + const dom = dumpDom(url); + let passed = 0; + let failed = 0; + for (let i = 0; i < assertions.length; i++) { + const a = assertions[i]; + let ok; + let label; + if (a.contains !== undefined) { + ok = dom.includes(a.contains); + label = a.name || `contains ${JSON.stringify(a.contains)}`; + } else if (a.matches !== undefined) { + let re; + try { + re = new RegExp(a.matches); + } catch (err) { + console.log(`FAIL ${a.name || `matches ${JSON.stringify(a.matches)}`} — bad regex: ${err.message}`); + failed++; + continue; + } + ok = re.test(dom); + label = a.name || `matches /${a.matches}/`; + } else if (a.absent !== undefined) { + ok = !dom.includes(a.absent); + label = a.name || `absent ${JSON.stringify(a.absent)}`; + } else { + console.log(`FAIL assertion #${i} — needs one of "contains", "matches", or "absent"`); + failed++; + continue; + } + if (ok) { + console.log(`PASS ${label}`); + passed++; + } else { + console.log(`FAIL ${label}`); + failed++; + } + } + console.log(`\n${passed}/${assertions.length} assertions passed`); + process.exit(failed === 0 ? 0 : 1); +} + +const [cmd, url, arg3, arg4] = process.argv.slice(2); +if (!url || !['dom', 'shot', 'check'].includes(cmd)) usage(); + +if (cmd === 'dom') { + process.stdout.write(dumpDom(url)); + process.exit(0); +} + +if (cmd === 'check') { + if (!arg3) usage(); + runChecks(url, arg3); +} else { + // shot + const out = arg3; + if (!out) usage(); + const browser = findBrowser(); + if (!browser) { + process.stderr.write( + 'browser unavailable — no Chromium/Chrome found (checked CHROME_BIN, ' + + 'PLAYWRIGHT_BROWSERS_PATH, macOS app bundles, PATH). Fall back to static checks.\n', + ); + process.exit(2); + } + const [w, h] = (arg4 || '1280x800').split('x'); + fs.mkdirSync(path.dirname(path.resolve(out)), { recursive: true }); + const res = spawnSync( + browser, + [...BASE_FLAGS, `--screenshot=${path.resolve(out)}`, `--window-size=${w || 1280},${h || 800}`, url], + { encoding: 'utf8', timeout: 30000 }, + ); + if (res.error || res.status !== 0) { + process.stderr.write( + `capture failed (${res.error ? res.error.message : `exit ${res.status}`})\n${res.stderr || ''}`, + ); + process.exit(1); + } + process.stderr.write(`screenshot written: ${path.resolve(out)}\n`); +} diff --git a/plugin/tools/http-check.js b/plugin/tools/http-check.js new file mode 100644 index 0000000..39dc9c5 --- /dev/null +++ b/plugin/tools/http-check.js @@ -0,0 +1,173 @@ +#!/usr/bin/env node +// Zero-dependency HTTP contract/smoke runner for QA. The backend counterpart to +// tools/browser.js: drives a checks file against a running server so QA verifies +// the spec's API contract without hand-authoring bespoke fetch tests each run. +// +// Usage: +// node tools/http-check.js +// +// checks.json is a JSON array of check objects (or an object with a `checks` +// array). Each check: +// { +// "name": "human label", // optional +// "method": "GET", // default GET +// "path": "/api/status", // required +// "body": { ... } | "raw string", // optional request body (objects sent as JSON) +// "headers": { ... }, // optional request headers +// "expectStatus": 200, // optional exact status match +// "expectJsonSubset":{ "status": "ok" }, // optional: response JSON must contain this (deep subset) +// "expectContains": "substring" | [ ... ] // optional: raw body must contain each substring +// } +// +// Exit codes: 0 all checks passed · 1 usage/one-or-more-failures. +// Prints one `PASS`/`FAIL` line per check plus a summary; FAIL lines say why. +'use strict'; + +const http = require('node:http'); +const https = require('node:https'); +const fs = require('node:fs'); + +function usage(msg) { + if (msg) process.stderr.write(`${msg}\n`); + process.stderr.write('usage: node tools/http-check.js \n'); + process.exit(1); +} + +// Deep-subset match: every key/value in `expected` must be present and equal in +// `actual`. Arrays must match element-by-element as subsets. Primitives compare +// with strict equality. Returns null on match, or a human path string on mismatch. +function subsetMismatch(expected, actual, pathStr = '') { + if (Array.isArray(expected)) { + if (!Array.isArray(actual)) return `${pathStr || ''}: expected array, got ${typeof actual}`; + for (let i = 0; i < expected.length; i++) { + const m = subsetMismatch(expected[i], actual[i], `${pathStr}[${i}]`); + if (m) return m; + } + return null; + } + if (expected && typeof expected === 'object') { + if (!actual || typeof actual !== 'object') { + return `${pathStr || ''}: expected object, got ${actual === null ? 'null' : typeof actual}`; + } + for (const key of Object.keys(expected)) { + const childPath = pathStr ? `${pathStr}.${key}` : key; + if (!(key in actual)) return `${childPath}: missing`; + const m = subsetMismatch(expected[key], actual[key], childPath); + if (m) return m; + } + return null; + } + // primitive + if (expected !== actual) { + return `${pathStr || ''}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`; + } + return null; +} + +function request(baseUrl, check) { + return new Promise((resolve, reject) => { + let url; + try { + url = new URL(check.path, baseUrl); + } catch (err) { + reject(new Error(`invalid path/base-url: ${err.message}`)); + return; + } + const lib = url.protocol === 'https:' ? https : http; + const headers = { ...(check.headers || {}) }; + let payload; + if (check.body !== undefined && check.body !== null) { + if (typeof check.body === 'string') { + payload = check.body; + } else { + payload = JSON.stringify(check.body); + if (!Object.keys(headers).some((h) => h.toLowerCase() === 'content-type')) { + headers['Content-Type'] = 'application/json'; + } + } + } + const req = lib.request( + url, + { method: (check.method || 'GET').toUpperCase(), headers }, + (res) => { + let raw = ''; + res.setEncoding('utf8'); + res.on('data', (c) => (raw += c)); + res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, raw })); + }, + ); + req.on('error', reject); + if (payload !== undefined) req.write(payload); + req.end(); + }); +} + +function evaluate(check, res) { + const failures = []; + if (check.expectStatus !== undefined && res.status !== check.expectStatus) { + failures.push(`status ${res.status} !== expected ${check.expectStatus}`); + } + if (check.expectContains !== undefined) { + const needles = Array.isArray(check.expectContains) ? check.expectContains : [check.expectContains]; + for (const needle of needles) { + if (!res.raw.includes(needle)) failures.push(`body missing substring ${JSON.stringify(needle)}`); + } + } + if (check.expectJsonSubset !== undefined) { + let parsed; + try { + parsed = JSON.parse(res.raw); + } catch { + failures.push('response body is not valid JSON'); + return failures; + } + const m = subsetMismatch(check.expectJsonSubset, parsed); + if (m) failures.push(`json subset mismatch — ${m}`); + } + return failures; +} + +async function main() { + const [baseUrl, checksPath] = process.argv.slice(2); + if (!baseUrl || !checksPath) usage(); + + let doc; + try { + doc = JSON.parse(fs.readFileSync(checksPath, 'utf8')); + } catch (err) { + usage(`cannot read checks file: ${err.message}`); + } + const checks = Array.isArray(doc) ? doc : Array.isArray(doc.checks) ? doc.checks : null; + if (!checks) usage('checks file must be a JSON array or an object with a `checks` array'); + + let passed = 0; + let failed = 0; + for (let i = 0; i < checks.length; i++) { + const check = checks[i]; + const label = check.name || `${(check.method || 'GET').toUpperCase()} ${check.path}`; + if (!check.path) { + console.log(`FAIL ${label} — check #${i} has no "path"`); + failed++; + continue; + } + try { + const res = await request(baseUrl, check); + const failures = evaluate(check, res); + if (failures.length === 0) { + console.log(`PASS ${label}`); + passed++; + } else { + console.log(`FAIL ${label} — ${failures.join('; ')}`); + failed++; + } + } catch (err) { + console.log(`FAIL ${label} — request error: ${err.message}`); + failed++; + } + } + + console.log(`\n${passed}/${checks.length} checks passed`); + process.exit(failed === 0 ? 0 : 1); +} + +main(); diff --git a/tools/browser.js b/tools/browser.js index 422c439..428d5c3 100644 --- a/tools/browser.js +++ b/tools/browser.js @@ -1,11 +1,12 @@ #!/usr/bin/env node -// Zero-dependency headless-Chromium wrapper for QA evidence capture. +// Zero-dependency headless-Chromium wrapper for QA evidence + assertions. // // Usage: -// node tools/browser.js dom # post-JS DOM → stdout -// node tools/browser.js shot [WxH] # screenshot (default 1280x800) +// node tools/browser.js dom # post-JS DOM → stdout +// node tools/browser.js shot [WxH] # screenshot (default 1280x800) +// node tools/browser.js check # assert against the post-JS DOM // -// Exit codes: 0 success · 1 usage/capture error · 2 no browser found +// Exit codes: 0 success · 1 usage/capture/assertion failure · 2 no browser found // (on exit 2, fall back to static HTML/CSS checks and note it in the test plan). 'use strict'; @@ -24,6 +25,15 @@ const BASE_FLAGS = [ '--virtual-time-budget=3000', ]; +// Absolute launcher paths worth probing before PATH lookups — covers common +// macOS app-bundle installs that never land on PATH (the reason QA silently +// fell back to static checks on macOS before). +const ABSOLUTE_CANDIDATES = [ + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + '/Applications/Chromium.app/Contents/MacOS/Chromium', + '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary', +]; + function findBrowser() { const candidates = []; if (process.env.CHROME_BIN) candidates.push(process.env.CHROME_BIN); @@ -38,6 +48,7 @@ function findBrowser() { ); } } + candidates.push(...ABSOLUTE_CANDIDATES); candidates.push('chromium', 'chromium-browser', 'google-chrome', 'chrome'); for (const c of candidates) { try { @@ -53,47 +64,133 @@ function findBrowser() { function usage() { process.stderr.write( - 'usage: node tools/browser.js dom \n' + - ' node tools/browser.js shot [WxH]\n', + 'usage: node tools/browser.js dom \n' + + ' node tools/browser.js shot [WxH]\n' + + ' node tools/browser.js check \n', ); process.exit(1); } -const [cmd, url, out, size] = process.argv.slice(2); -if (!url || (cmd !== 'dom' && cmd !== 'shot') || (cmd === 'shot' && !out)) usage(); +// Dump the post-JS DOM for `url` as a string, or exit with the right code on +// failure (2 = no browser, 1 = capture error). +function dumpDom(url) { + const browser = findBrowser(); + if (!browser) { + process.stderr.write( + 'browser unavailable — no Chromium/Chrome found (checked CHROME_BIN, ' + + 'PLAYWRIGHT_BROWSERS_PATH, macOS app bundles, PATH). Fall back to static checks.\n', + ); + process.exit(2); + } + const res = spawnSync(browser, [...BASE_FLAGS, '--dump-dom', url], { + encoding: 'utf8', + timeout: 30000, + }); + if (res.error || res.status !== 0) { + process.stderr.write( + `capture failed (${res.error ? res.error.message : `exit ${res.status}`})\n${res.stderr || ''}`, + ); + process.exit(1); + } + return res.stdout; +} -const browser = findBrowser(); -if (!browser) { - process.stderr.write( - 'browser unavailable — no Chromium/Chrome found (checked CHROME_BIN, ' + - 'PLAYWRIGHT_BROWSERS_PATH, PATH). Fall back to static checks.\n', - ); - process.exit(2); +// `check` mode: evaluate a small assertions list against the post-JS DOM. +// assertions.json is a JSON array (or an object with an `assertions` array) of: +// { "name": "...", "contains": "substring in the rendered DOM" } +// { "matches": "regex source" } // tested against the DOM string +// { "absent": "substring that must NOT appear" } +// Prints PASS/FAIL per assertion; exits 1 if any fail. +function runChecks(url, assertionsPath) { + let doc; + try { + doc = JSON.parse(fs.readFileSync(assertionsPath, 'utf8')); + } catch (err) { + process.stderr.write(`cannot read assertions file: ${err.message}\n`); + process.exit(1); + } + const assertions = Array.isArray(doc) ? doc : Array.isArray(doc.assertions) ? doc.assertions : null; + if (!assertions) { + process.stderr.write('assertions file must be a JSON array or an object with an `assertions` array\n'); + process.exit(1); + } + + const dom = dumpDom(url); + let passed = 0; + let failed = 0; + for (let i = 0; i < assertions.length; i++) { + const a = assertions[i]; + let ok; + let label; + if (a.contains !== undefined) { + ok = dom.includes(a.contains); + label = a.name || `contains ${JSON.stringify(a.contains)}`; + } else if (a.matches !== undefined) { + let re; + try { + re = new RegExp(a.matches); + } catch (err) { + console.log(`FAIL ${a.name || `matches ${JSON.stringify(a.matches)}`} — bad regex: ${err.message}`); + failed++; + continue; + } + ok = re.test(dom); + label = a.name || `matches /${a.matches}/`; + } else if (a.absent !== undefined) { + ok = !dom.includes(a.absent); + label = a.name || `absent ${JSON.stringify(a.absent)}`; + } else { + console.log(`FAIL assertion #${i} — needs one of "contains", "matches", or "absent"`); + failed++; + continue; + } + if (ok) { + console.log(`PASS ${label}`); + passed++; + } else { + console.log(`FAIL ${label}`); + failed++; + } + } + console.log(`\n${passed}/${assertions.length} assertions passed`); + process.exit(failed === 0 ? 0 : 1); } -let flags; +const [cmd, url, arg3, arg4] = process.argv.slice(2); +if (!url || !['dom', 'shot', 'check'].includes(cmd)) usage(); + if (cmd === 'dom') { - flags = [...BASE_FLAGS, '--dump-dom', url]; -} else { - const [w, h] = (size || '1280x800').split('x'); - fs.mkdirSync(path.dirname(path.resolve(out)), { recursive: true }); - flags = [ - ...BASE_FLAGS, - `--screenshot=${path.resolve(out)}`, - `--window-size=${w || 1280},${h || 800}`, - url, - ]; + process.stdout.write(dumpDom(url)); + process.exit(0); } -const res = spawnSync(browser, flags, { encoding: 'utf8', timeout: 30000 }); -if (res.error || res.status !== 0) { - process.stderr.write( - `capture failed (${res.error ? res.error.message : `exit ${res.status}`})\n${res.stderr || ''}`, - ); - process.exit(1); -} -if (cmd === 'dom') { - process.stdout.write(res.stdout); +if (cmd === 'check') { + if (!arg3) usage(); + runChecks(url, arg3); } else { + // shot + const out = arg3; + if (!out) usage(); + const browser = findBrowser(); + if (!browser) { + process.stderr.write( + 'browser unavailable — no Chromium/Chrome found (checked CHROME_BIN, ' + + 'PLAYWRIGHT_BROWSERS_PATH, macOS app bundles, PATH). Fall back to static checks.\n', + ); + process.exit(2); + } + const [w, h] = (arg4 || '1280x800').split('x'); + fs.mkdirSync(path.dirname(path.resolve(out)), { recursive: true }); + const res = spawnSync( + browser, + [...BASE_FLAGS, `--screenshot=${path.resolve(out)}`, `--window-size=${w || 1280},${h || 800}`, url], + { encoding: 'utf8', timeout: 30000 }, + ); + if (res.error || res.status !== 0) { + process.stderr.write( + `capture failed (${res.error ? res.error.message : `exit ${res.status}`})\n${res.stderr || ''}`, + ); + process.exit(1); + } process.stderr.write(`screenshot written: ${path.resolve(out)}\n`); } diff --git a/tools/http-check.js b/tools/http-check.js new file mode 100644 index 0000000..39dc9c5 --- /dev/null +++ b/tools/http-check.js @@ -0,0 +1,173 @@ +#!/usr/bin/env node +// Zero-dependency HTTP contract/smoke runner for QA. The backend counterpart to +// tools/browser.js: drives a checks file against a running server so QA verifies +// the spec's API contract without hand-authoring bespoke fetch tests each run. +// +// Usage: +// node tools/http-check.js +// +// checks.json is a JSON array of check objects (or an object with a `checks` +// array). Each check: +// { +// "name": "human label", // optional +// "method": "GET", // default GET +// "path": "/api/status", // required +// "body": { ... } | "raw string", // optional request body (objects sent as JSON) +// "headers": { ... }, // optional request headers +// "expectStatus": 200, // optional exact status match +// "expectJsonSubset":{ "status": "ok" }, // optional: response JSON must contain this (deep subset) +// "expectContains": "substring" | [ ... ] // optional: raw body must contain each substring +// } +// +// Exit codes: 0 all checks passed · 1 usage/one-or-more-failures. +// Prints one `PASS`/`FAIL` line per check plus a summary; FAIL lines say why. +'use strict'; + +const http = require('node:http'); +const https = require('node:https'); +const fs = require('node:fs'); + +function usage(msg) { + if (msg) process.stderr.write(`${msg}\n`); + process.stderr.write('usage: node tools/http-check.js \n'); + process.exit(1); +} + +// Deep-subset match: every key/value in `expected` must be present and equal in +// `actual`. Arrays must match element-by-element as subsets. Primitives compare +// with strict equality. Returns null on match, or a human path string on mismatch. +function subsetMismatch(expected, actual, pathStr = '') { + if (Array.isArray(expected)) { + if (!Array.isArray(actual)) return `${pathStr || ''}: expected array, got ${typeof actual}`; + for (let i = 0; i < expected.length; i++) { + const m = subsetMismatch(expected[i], actual[i], `${pathStr}[${i}]`); + if (m) return m; + } + return null; + } + if (expected && typeof expected === 'object') { + if (!actual || typeof actual !== 'object') { + return `${pathStr || ''}: expected object, got ${actual === null ? 'null' : typeof actual}`; + } + for (const key of Object.keys(expected)) { + const childPath = pathStr ? `${pathStr}.${key}` : key; + if (!(key in actual)) return `${childPath}: missing`; + const m = subsetMismatch(expected[key], actual[key], childPath); + if (m) return m; + } + return null; + } + // primitive + if (expected !== actual) { + return `${pathStr || ''}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`; + } + return null; +} + +function request(baseUrl, check) { + return new Promise((resolve, reject) => { + let url; + try { + url = new URL(check.path, baseUrl); + } catch (err) { + reject(new Error(`invalid path/base-url: ${err.message}`)); + return; + } + const lib = url.protocol === 'https:' ? https : http; + const headers = { ...(check.headers || {}) }; + let payload; + if (check.body !== undefined && check.body !== null) { + if (typeof check.body === 'string') { + payload = check.body; + } else { + payload = JSON.stringify(check.body); + if (!Object.keys(headers).some((h) => h.toLowerCase() === 'content-type')) { + headers['Content-Type'] = 'application/json'; + } + } + } + const req = lib.request( + url, + { method: (check.method || 'GET').toUpperCase(), headers }, + (res) => { + let raw = ''; + res.setEncoding('utf8'); + res.on('data', (c) => (raw += c)); + res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, raw })); + }, + ); + req.on('error', reject); + if (payload !== undefined) req.write(payload); + req.end(); + }); +} + +function evaluate(check, res) { + const failures = []; + if (check.expectStatus !== undefined && res.status !== check.expectStatus) { + failures.push(`status ${res.status} !== expected ${check.expectStatus}`); + } + if (check.expectContains !== undefined) { + const needles = Array.isArray(check.expectContains) ? check.expectContains : [check.expectContains]; + for (const needle of needles) { + if (!res.raw.includes(needle)) failures.push(`body missing substring ${JSON.stringify(needle)}`); + } + } + if (check.expectJsonSubset !== undefined) { + let parsed; + try { + parsed = JSON.parse(res.raw); + } catch { + failures.push('response body is not valid JSON'); + return failures; + } + const m = subsetMismatch(check.expectJsonSubset, parsed); + if (m) failures.push(`json subset mismatch — ${m}`); + } + return failures; +} + +async function main() { + const [baseUrl, checksPath] = process.argv.slice(2); + if (!baseUrl || !checksPath) usage(); + + let doc; + try { + doc = JSON.parse(fs.readFileSync(checksPath, 'utf8')); + } catch (err) { + usage(`cannot read checks file: ${err.message}`); + } + const checks = Array.isArray(doc) ? doc : Array.isArray(doc.checks) ? doc.checks : null; + if (!checks) usage('checks file must be a JSON array or an object with a `checks` array'); + + let passed = 0; + let failed = 0; + for (let i = 0; i < checks.length; i++) { + const check = checks[i]; + const label = check.name || `${(check.method || 'GET').toUpperCase()} ${check.path}`; + if (!check.path) { + console.log(`FAIL ${label} — check #${i} has no "path"`); + failed++; + continue; + } + try { + const res = await request(baseUrl, check); + const failures = evaluate(check, res); + if (failures.length === 0) { + console.log(`PASS ${label}`); + passed++; + } else { + console.log(`FAIL ${label} — ${failures.join('; ')}`); + failed++; + } + } catch (err) { + console.log(`FAIL ${label} — request error: ${err.message}`); + failed++; + } + } + + console.log(`\n${passed}/${checks.length} checks passed`); + process.exit(failed === 0 ? 0 : 1); +} + +main();