ci(release): advisory Playwright web-UI sweep on tag builds (QA gate T2)

.github/workflows/prerelease.yml

@@ -888,8 +888,11 @@ jobs:
           # Create a flat structure to avoid duplicates
           mkdir -p release-files
 
-          # Copy archives (tar.gz and zip files) - only versioned, no latest for prereleases
-          find dist -name "*.tar.gz" -o -name "*.zip" | while read file; do
+          # Copy archives (tar.gz and zip files) - only versioned, no latest for prereleases.
+          # The QA gate runs as a reusable workflow of THIS run, so its artifacts land
+          # in dist/ too; its Playwright HTML report stores traces as
+          # playwright-report/data/<sha>.zip and must never become a release asset.
+          find dist -not -path "*/web-ui-sweep-playwright-report/*" \( -name "*.tar.gz" -o -name "*.zip" \) | while read file; do
             filename=$(basename "$file")
             cp "$file" "release-files/$filename"
           done

.github/workflows/release-qa-gate.yml

@@ -16,9 +16,13 @@ name: Release QA Gate
 # manually against any ref (workflow_dispatch — a dry run that publishes
 # nothing and never counts as qualification for a later tag, FR-001a).
 #
-# T2/T3/T4 extension slots (reserved manifest entries, recorded as
+# T2 landed as an ADVISORY entry — it runs on every tag build and is reported,
+# but cannot block publication (`continue-on-error` + a non-blocking manifest
+# entry):
+#   advisory/web-ui-sweep        — T2 Playwright sweep against the candidate
+#
+# T3/T4 extension slots (reserved manifest entries, recorded as
 # not-run/"not-implemented-yet" until their stage lands):
-#   reserved/web-ui-sweep        — T2 Playwright sweep against the candidate
 #   reserved/macos-smoke         — T3 macOS tray smoke (advisory, FR-019/020)
 #   reserved/surface-consistency — T4 REST/CLI/Web-UI/tray state agreement
 # See docs/development/release-gate.md.
@@ -374,6 +378,76 @@ jobs:
           retention-days: 1
           if-no-files-found: warn
 
+  # ---------------------------------------------------------------------------
+  # T2 (US3, FR-016): Playwright Web UI sweep against the Web UI served by the
+  # CANDIDATE binary (embedded frontend, not a dev server), with a live stdio
+  # mcpfixture upstream so the servers/tools screens have real data.
+  #
+  # ADVISORY, deliberately: `continue-on-error: true` keeps a red sweep out of
+  # this reusable workflow's conclusion, so a publisher whose job `needs:` the
+  # gate still publishes. The sweep is nonetheless REPORTED — the driver runs it
+  # with --advisory, which records `advisory/web-ui-sweep` as `advisory-fail`,
+  # and the verdict job lists it under advisory_failures. Promote to blocking by
+  # flipping that manifest entry to Blocking:true and dropping continue-on-error.
+  #
+  # Setup is not reinvented here: scripts/run-web-smoke.sh is the same launcher
+  # used by hand (docs/development/web-ui-verification.md) — it boots a
+  # throwaway instance, installs Chromium, and runs e2e/web-ui-sweep.
+  # ---------------------------------------------------------------------------
+  web-ui-sweep:
+    name: Web UI sweep (advisory)
+    needs: build-candidate
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    continue-on-error: true  # advisory: never blocks the tag (FR-019 pattern)
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          ref: ${{ inputs.ref || github.sha }}
+
+      - name: Set up Node.js  # the sweep installs @playwright/test + Chromium
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020  # v7.0.0
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Download candidate binaries
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          name: gate-candidate
+          path: dist-bin
+
+      - name: Stage binaries
+        run: chmod +x dist-bin/*
+
+      - name: Run Web UI sweep
+        env:
+          MCPPROXY_BINARY_PATH: ${{ github.workspace }}/dist-bin/mcpproxy
+          MCPPROXY_FIXTURE_PATH: ${{ github.workspace }}/dist-bin/mcpfixture
+          ARTIFACT_DIR: ${{ github.workspace }}/tmp/web-smoke-artifacts
+        run: |
+          ./dist-bin/release-gate run-suite \
+            --name advisory/web-ui-sweep --advisory --report-dir "${GATE_REPORT_DIR}" \
+            -- bash scripts/run-web-smoke.sh
+
+      - name: Upload Playwright report
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: web-ui-sweep-playwright-report
+          path: tmp/web-smoke-artifacts/
+          retention-days: 14
+          if-no-files-found: warn
+
+      - name: Upload report fragment
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: gate-fragment-web-ui-sweep
+          path: ${{ env.GATE_REPORT_DIR }}/
+          retention-days: 1
+          if-no-files-found: warn
+
   # ---------------------------------------------------------------------------
   # Merge every fragment against the hardcoded gatereport manifest → one
   # verdict. Runs even if upstream jobs failed (if: always()) so a missing
@@ -383,7 +457,7 @@ jobs:
   # ---------------------------------------------------------------------------
   verdict:
     name: Gate verdict
-    needs: [build-candidate, suite-api-e2e, suite-race, suite-scan-eval, matrix-invariants]
+    needs: [build-candidate, suite-api-e2e, suite-race, suite-scan-eval, matrix-invariants, web-ui-sweep]
     if: always()
     runs-on: ubuntu-latest
     timeout-minutes: 10

.github/workflows/release.yml

@@ -1298,8 +1298,14 @@ jobs:
           # Create a flat structure to avoid duplicates
           mkdir -p release-files
 
-          # Copy archives (tar.gz and zip files)
-          find dist -name "*.tar.gz" -o -name "*.zip" | while read file; do
+          # Copy archives (tar.gz and zip files).
+          # The QA gate runs as a reusable workflow of THIS run, so its artifacts
+          # land in dist/ too. Its Playwright HTML report stores traces as
+          # playwright-report/data/<sha>.zip (retained on failure — precisely the
+          # case the advisory sweep tolerates), which would otherwise be published
+          # as unnamed release assets. Excluded here, not upstream, so the report
+          # stays downloadable as a CI artifact for post-mortems.
+          find dist -not -path "*/web-ui-sweep-playwright-report/*" \( -name "*.tar.gz" -o -name "*.zip" \) | while read file; do
             filename=$(basename "$file")
             cp "$file" "release-files/$filename"
           done

ROADMAP.md

@@ -105,15 +105,17 @@ graph LR
   release_qa_gate_matrix --> release_qa_gate_consistency
 
   classDef done fill:#1f7a1f,stroke:#0d3d0d,color:#ffffff;
+  classDef in_review fill:#9a6700,stroke:#5c3d00,color:#ffffff;
   classDef todo fill:#6e7781,stroke:#3d4248,color:#ffffff;
   class release_qa_gate_matrix done;
-  class release_qa_gate_playwright,release_qa_gate_macos,release_qa_gate_consistency todo;
+  class release_qa_gate_playwright in_review;
+  class release_qa_gate_macos,release_qa_gate_consistency todo;
 ```
 
 | Task | Status | Refs |
 | --- | --- | --- |
 | T1: tag-blocking release-gate workflow: server-type matrix (stdio/http/sse/docker/oauth) + invariants (activity-log/request-id, token+telemetry counters, quarantine flow, reconnect, upgrade-in-place), publish jobs gated on the verdict, scan-eval unconditional on tags | 🟢 Done | #819 |
-| T2: wire the Playwright Web UI sweep into the gate (currently manual-trigger only) | ⚪ Todo | — |
+| T2: wire the Playwright Web UI sweep into the gate (currently manual-trigger only) | 🟡 In review | — |
 | T3: macOS app smoke on a macos runner, advisory until 3 consecutive passes (today zero CI automation for the tray app) | ⚪ Todo | — |
 | T4: surface-state consistency check (tray/Web UI/CLI agree with core on server states) | ⚪ Todo | — |
 
@@ -801,3 +803,4 @@ Legend: `shipped` ≥95% checked · `in-flight` 1–94% · `drafted` 0% · `—`
 | [097-stored-scripts](./specs/097-stored-scripts/) | `in-flight` | 13/14 (93%) |
 | [098-tools-preflight](./specs/098-tools-preflight/) | `in-flight` | 26/33 (79%) |
 | [099-describe-check-mode](./specs/099-describe-check-mode/) | `in-flight` | 9/10 (90%) |
+| [100-prompt-rugpull-baseline](./specs/100-prompt-rugpull-baseline/) | — | — |

cmd/release-gate/main.go

@@ -76,7 +76,7 @@ usage:
                           [--work-dir DIR] [--cell-timeout 300s]
   release-gate invariants --state-file FILE --report-dir DIR [--prev-binary PATH] [--skip-upgrade] \
                           [--upgrade-repo owner/repo] [--keep-core]
-  release-gate run-suite  --name suite/api-e2e --report-dir DIR -- CMD [ARGS...]
+  release-gate run-suite  --name suite/api-e2e --report-dir DIR [--advisory] -- CMD [ARGS...]
   release-gate report     --report-dir DIR [--out gate-report.json] [--summary summary.md]
 `)
 }
@@ -128,6 +128,7 @@ func cmdRunSuite(ctx context.Context, args []string) (bool, error) {
 	fs := flag.NewFlagSet("run-suite", flag.ExitOnError)
 	name := fs.String("name", "", "manifest entry name (e.g. suite/api-e2e)")
 	reportDir := fs.String("report-dir", "", "directory for report fragments (required)")
+	advisory := fs.Bool("advisory", false, "non-blocking entry: record failures as advisory-fail")
 	if err := fs.Parse(args); err != nil {
 		return false, err
 	}
@@ -138,7 +139,7 @@ func cmdRunSuite(ctx context.Context, args []string) (bool, error) {
 	if len(cmdArgs) == 0 {
 		return false, fmt.Errorf("no command given after flags (use: run-suite --name N --report-dir D -- CMD ARGS)")
 	}
-	return runSuite(ctx, *name, *reportDir, cmdArgs)
+	return runSuite(ctx, *name, *reportDir, *advisory, cmdArgs)
 }
 
 func mustAbs(p string) string {

cmd/release-gate/suite.go

@@ -14,7 +14,13 @@ import (
 // test-api-e2e.sh, go race tests, scan-eval --gate) and records its outcome
 // as a report fragment. The command's output streams through so CI logs stay
 // useful.
-func runSuite(ctx context.Context, name, reportDir string, cmdArgs []string) (bool, error) {
+//
+// advisory marks a non-blocking entry (Spec 081 T2 Web UI sweep, T3 macOS
+// smoke): its failures are recorded as `advisory-fail` so the merged report
+// distinguishes "went red but cannot block the tag" from a real blocking
+// failure. The returned ok is false either way — the CI job still goes red
+// (under continue-on-error) so a maintainer sees it.
+func runSuite(ctx context.Context, name, reportDir string, advisory bool, cmdArgs []string) (bool, error) {
 	frag := &gatereport.Fragment{Name: name, StartedAt: time.Now().UTC()}
 	cmd := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
 	cmd.Stdout = os.Stdout
@@ -26,6 +32,9 @@ func runSuite(ctx context.Context, name, reportDir string, cmdArgs []string) (bo
 	frag.DurationMS = frag.FinishedAt.Sub(frag.StartedAt).Milliseconds()
 	if err != nil {
 		frag.Status = gatereport.StatusFail
+		if advisory {
+			frag.Status = gatereport.StatusAdvisoryFail
+		}
 		frag.Reason = fmt.Sprintf("suite command failed: %v", err)
 		frag.Classification = gatereport.ClassificationProduct
 	} else {

cmd/release-gate/suite_test.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/smart-mcp-proxy/mcpproxy-go/internal/gatereport"
+)
+
+func readFragment(t *testing.T, dir, name string) gatereport.Fragment {
+	t.Helper()
+	data, err := os.ReadFile(filepath.Join(dir, gatereport.FragmentFileName(name)))
+	if err != nil {
+		t.Fatalf("read fragment %s: %v", name, err)
+	}
+	var frag gatereport.Fragment
+	if err := json.Unmarshal(data, &frag); err != nil {
+		t.Fatalf("parse fragment %s: %v", name, err)
+	}
+	return frag
+}
+
+// A failing advisory suite must be recorded as advisory-fail (never plain
+// fail), so the merged report reads honestly: the check ran, it went red, and
+// it did not block the tag (Spec 081 T2/FR-019).
+func TestRunSuite_AdvisoryFailure_RecordsAdvisoryFail(t *testing.T) {
+	dir := t.TempDir()
+	ok, err := runSuite(context.Background(), gatereport.EntryAdvisoryWebUISweep, dir, true,
+		[]string{"sh", "-c", "exit 3"})
+	if err != nil {
+		t.Fatalf("runSuite returned an error: %v", err)
+	}
+	if ok {
+		t.Error("a failing suite must report ok=false even when advisory (the job goes red under continue-on-error)")
+	}
+	frag := readFragment(t, dir, gatereport.EntryAdvisoryWebUISweep)
+	if frag.Status != gatereport.StatusAdvisoryFail {
+		t.Errorf("status=%s want %s", frag.Status, gatereport.StatusAdvisoryFail)
+	}
+	if frag.Reason == "" {
+		t.Error("a non-pass fragment must carry a reason (FR-004)")
+	}
+}
+
+func TestRunSuite_AdvisorySuccess_RecordsPass(t *testing.T) {
+	dir := t.TempDir()
+	ok, err := runSuite(context.Background(), gatereport.EntryAdvisoryWebUISweep, dir, true,
+		[]string{"sh", "-c", "exit 0"})
+	if err != nil || !ok {
+		t.Fatalf("runSuite ok=%v err=%v, want true/nil", ok, err)
+	}
+	if frag := readFragment(t, dir, gatereport.EntryAdvisoryWebUISweep); frag.Status != gatereport.StatusPass {
+		t.Errorf("status=%s want %s", frag.Status, gatereport.StatusPass)
+	}
+}
+
+// Blocking suites keep the original behaviour: a failure is a plain fail.
+func TestRunSuite_BlockingFailure_RecordsFail(t *testing.T) {
+	dir := t.TempDir()
+	if _, err := runSuite(context.Background(), gatereport.EntrySuiteAPIE2E, dir, false,
+		[]string{"sh", "-c", "exit 1"}); err != nil {
+		t.Fatalf("runSuite returned an error: %v", err)
+	}
+	if frag := readFragment(t, dir, gatereport.EntrySuiteAPIE2E); frag.Status != gatereport.StatusFail {
+		t.Errorf("status=%s want %s", frag.Status, gatereport.StatusFail)
+	}
+}