diff --git a/docs/README.md b/docs/README.md index cf9cd945e..5a3a37f0c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -41,6 +41,9 @@ unless this index says otherwise. - [Generic runtime primitives](./generic-runtime-primitives.md) documents the caller-neutral artifact storage, trusted browser origin, materialization, and target-context envelopes shared by runtime integrations. +- [Browser environment matrices](./browser-environment-matrices.md) documents + deterministic hostile-environment expansion, capability fidelity, bounds, + isolated cell artifacts, and replay evidence. - [Runtime profile contract](./runtime-profile-contract.md) documents `wp-codebox/runtime-profile/v1`, the Codebox-owned profile request/result lane for capabilities, components, readiness, diagnostics, and provenance. diff --git a/docs/browser-environment-matrices.md b/docs/browser-environment-matrices.md new file mode 100644 index 000000000..aac0a802a --- /dev/null +++ b/docs/browser-environment-matrices.md @@ -0,0 +1,89 @@ +# Browser environment matrices + +`wp-codebox/browser-environment-matrix/v1` expands hostile browser conditions +into deterministic, bounded cells. It composes existing probe, action, adaptive, +visual, accessibility, and multi-actor executors through one callback rather +than duplicating their execution logic. + +```ts +const matrix = browserEnvironmentMatrix({ + id: "responsive-accessibility", + seed: "ci-42", + dimensions: [ + { id: "viewport", values: [ + { id: "desktop", environment: { viewport: { width: 1280, height: 720 } } }, + { id: "mobile", environment: { viewport: { width: 320, height: 640 }, isMobile: true, hasTouch: true } }, + ] }, + { id: "preferences", values: [ + { id: "default", environment: { colorScheme: "light", reducedMotion: "no-preference" } }, + { id: "hostile", environment: { colorScheme: "dark", reducedMotion: "reduce", forcedColors: "active" } }, + ] }, + ], + limits: { + maxCells: 4, + maxDurationMs: 120_000, + maxCellDurationMs: 30_000, + maxArtifactBytes: 50 * 1024 * 1024, + }, +}) +``` + +## Expansion and bounds + +Dimension and value ids are sorted before cartesian expansion, so declaration +or object insertion order cannot change cell order. Each cell derives its id and +seed from the matrix seed, selected value ids, and canonical requested +environment. Conflicting assignments from two dimensions are rejected. + +Construction fails before execution when the cartesian product exceeds +`maxCells` or when `cells * maxCellDurationMs` exceeds `maxDurationMs`. Execution +is sequential by default. The runner additionally stops on wall-time, +interruption, or aggregate artifact-byte exhaustion while preserving every +completed cell report. + +## Capabilities and fidelity + +Values declare `requiredCapabilities` and `optionalCapabilities` using generic +runtime capability ids. A required unsupported capability fails the cell closed +without executing it. An optional unsupported capability executes when safe and +marks an otherwise passing cell `inconclusive`. Every cell records `exact`, +`emulated`, or `unsupported` fidelity and a reason. + +The Playwright adapter included with WP Codebox 0.15 applies: + +- viewport, named device, DPR, mobile, touch, and orientation context options; +- color scheme, reduced motion, forced colors, and contrast media features; +- locale, timezone, and online/offline state; +- fixed or realtime browser clock behavior; +- page-scale zoom with explicitly `emulated` fidelity; +- caller-declared CPU and network profiles through the browser debugging + protocol with explicitly `emulated` fidelity; +- caller-declared generic capability state through an immutable page global. + +Unknown device, CPU, or network profiles are unsupported rather than silently +falling back. OS-level zoom, browser chrome, server clocks, and transport-level +socket faults are not claimed. Transport degradation continues to use the +existing transport-fault contract. + +## Evidence and replay + +Callers provide a safe `runId`. Cell evidence belongs under: + +```text +browser-matrices///cells/-/ +``` + +The Playwright adapter writes `matrix-report.json` beside the `cells` directory. +Each cell includes requested and effective environments, selected dimensions, +provider/browser/version provenance, capability fidelity, status, findings, +timing, artifact references, and an exact replay contract. Environment cell +identity participates in default finding fingerprints. + +`browserEnvironmentMatrixFailed(report)` preserves fail-on-finding behavior: +incomplete runs always fail, while findings fail unless the declaration sets +`failOnFinding: false`. Completed evidence remains available in either case. + +For multi-actor scenarios, resolve the shared cell once and create one context +per actor from that resolution. Explicitly different actor environments should +be represented as separate named dimensions and resolved before constructing +actor clients; never mix undeclared context settings inside one cell. diff --git a/package.json b/package.json index 54f0d9473..3e61ab671 100644 --- a/package.json +++ b/package.json @@ -178,6 +178,7 @@ "test:bench-command-step-behavior": "tsx tests/bench-command-step-behavior.test.ts", "test:generic-primitives": "npm run test:artifact-path-primitives && npm run test:browser-callback-materialization-contracts && npm run test:source-package-compiler-primitives && npm run test:bench-command-step-behavior && npm run test:generic-ability-runtime-run", "test:browser-artifact-session": "tsx tests/browser-artifact-session.test.ts", + "test:browser-environment-matrix": "tsx --test tests/browser-environment-matrix.test.ts tests/browser-environment-matrix.browser.test.ts", "test:browser-diagnostic-providers": "tsx tests/browser-diagnostic-providers.test.ts", "test:browser-capture-html-diagnostics-reliability": "tsx tests/browser-capture-html-diagnostics-reliability.test.ts", "test:browser-provider-permissions": "tsx tests/browser-provider-permissions.test.ts", diff --git a/packages/runtime-core/src/browser-environment-matrix.ts b/packages/runtime-core/src/browser-environment-matrix.ts new file mode 100644 index 000000000..d72cd44d4 --- /dev/null +++ b/packages/runtime-core/src/browser-environment-matrix.ts @@ -0,0 +1,341 @@ +import { createHash } from "node:crypto" + +import { stableJson, stripUndefined } from "./object-utils.js" + +export const BROWSER_ENVIRONMENT_MATRIX_SCHEMA = "wp-codebox/browser-environment-matrix/v1" as const +export const BROWSER_ENVIRONMENT_MATRIX_REPORT_SCHEMA = "wp-codebox/browser-environment-matrix-report/v1" as const +export const BROWSER_ENVIRONMENT_REPLAY_SCHEMA = "wp-codebox/browser-environment-replay/v1" as const + +export type BrowserEnvironmentFidelity = "exact" | "emulated" | "unsupported" +export type BrowserEnvironmentCellStatus = "passed" | "findings" | "error" | "timed-out" | "inconclusive" + +export interface BrowserEnvironment { + viewport?: { width: number; height: number } + device?: string + deviceScaleFactor?: number + isMobile?: boolean + hasTouch?: boolean + orientation?: "portrait" | "landscape" + zoom?: number + colorScheme?: "light" | "dark" | "no-preference" + reducedMotion?: "reduce" | "no-preference" + forcedColors?: "active" | "none" + contrast?: "more" | "no-preference" + locale?: string + timezone?: string + networkProfile?: string + cpuProfile?: string + online?: boolean + clock?: { mode: "realtime" | "fixed"; at?: string } + capabilities?: Record +} + +export interface BrowserEnvironmentDimensionValue { + id: string + environment: BrowserEnvironment + requiredCapabilities?: string[] + optionalCapabilities?: string[] +} + +export interface BrowserEnvironmentDimension { + id: string + values: BrowserEnvironmentDimensionValue[] +} + +export interface BrowserEnvironmentMatrixLimits { + maxCells: number + maxDurationMs: number + maxCellDurationMs: number + maxArtifactBytes: number +} + +export interface BrowserEnvironmentMatrix { + schema: typeof BROWSER_ENVIRONMENT_MATRIX_SCHEMA + id: string + seed: string + dimensions: BrowserEnvironmentDimension[] + limits: BrowserEnvironmentMatrixLimits + failOnFinding: boolean + replayCommand?: string + metadata?: Record +} + +export interface BrowserEnvironmentCell { + id: string + index: number + seed: string + selections: Record + requested: BrowserEnvironment + requiredCapabilities: string[] + optionalCapabilities: string[] +} + +export interface BrowserEnvironmentCapabilityResult { + id: string + fidelity: BrowserEnvironmentFidelity + reason?: string +} + +export interface ResolvedBrowserEnvironment { + effective: BrowserEnvironment + capabilities: BrowserEnvironmentCapabilityResult[] + provider?: { id: string; browser?: string; channel?: string; version?: string } +} + +export interface BrowserEnvironmentFinding { + code: string + message: string + severity?: string + fingerprint?: string + metadata?: Record +} + +export interface BrowserEnvironmentCellExecution { + status: BrowserEnvironmentCellStatus + findings?: BrowserEnvironmentFinding[] + artifacts?: Array<{ path: string; kind: string; bytes?: number; sha256?: string }> + metadata?: Record +} + +export interface BrowserEnvironmentReplay { + schema: typeof BROWSER_ENVIRONMENT_REPLAY_SCHEMA + matrixId: string + matrixSeed: string + cellId: string + cellSeed: string + selections: Record + requested: BrowserEnvironment + effective: BrowserEnvironment + provider?: ResolvedBrowserEnvironment["provider"] + capabilities: BrowserEnvironmentCapabilityResult[] + artifactNamespace: string + command: string +} + +export interface BrowserEnvironmentMatrixCellReport { + id: string + index: number + seed: string + selections: Record + artifactNamespace: string + requested: BrowserEnvironment + effective: BrowserEnvironment + status: BrowserEnvironmentCellStatus + findings: BrowserEnvironmentFinding[] + artifacts: NonNullable + timing: { startedAt: string; completedAt: string; durationMs: number } + replay: BrowserEnvironmentReplay + capabilities: BrowserEnvironmentCapabilityResult[] + unsupported: string[] + inconclusive: string[] + metadata?: Record +} + +export interface BrowserEnvironmentMatrixReport { + schema: typeof BROWSER_ENVIRONMENT_MATRIX_REPORT_SCHEMA + matrixId: string + runId: string + seed: string + status: "passed" | "findings" | "incomplete" + failOnFinding: boolean + cells: BrowserEnvironmentMatrixCellReport[] + summary: { planned: number; completed: number; passed: number; findings: number; failed: number; inconclusive: number } + resourceUsage: { durationMs: number; artifactBytes: number } + diagnostics: Array<{ code: string; message: string }> +} + +export interface BrowserEnvironmentMatrixRunnerOptions { + runId: string + resolve(cell: BrowserEnvironmentCell): Promise | ResolvedBrowserEnvironment + execute(input: { cell: BrowserEnvironmentCell; resolved: ResolvedBrowserEnvironment; artifactNamespace: string; signal: AbortSignal }): Promise + now?: () => number + signal?: AbortSignal + replayCommand?: (matrix: BrowserEnvironmentMatrix, cell: BrowserEnvironmentCell) => string +} + +export function browserEnvironmentMatrix(input: Omit & { limits?: Partial; failOnFinding?: boolean }): BrowserEnvironmentMatrix { + if (!safeId(input.id) || !input.seed) throw new Error("Browser environment matrices require a safe non-empty id and seed.") + if (!Array.isArray(input.dimensions) || input.dimensions.length === 0) throw new Error("Browser environment matrices require at least one dimension.") + const dimensions = input.dimensions.map(normalizeDimension).sort((left, right) => left.id.localeCompare(right.id)) + if (new Set(dimensions.map(({ id }) => id)).size !== dimensions.length) throw new Error("Browser environment matrix dimension ids must be unique.") + const limits = normalizeLimits(input.limits) + const cells = dimensions.reduce((count, dimension) => count * dimension.values.length, 1) + if (!Number.isSafeInteger(cells) || cells > limits.maxCells) throw new Error(`Browser environment matrix expands to ${cells} cells, exceeding maxCells=${limits.maxCells}.`) + const requestedDuration = cells * limits.maxCellDurationMs + if (requestedDuration > limits.maxDurationMs) throw new Error(`Browser environment matrix reserves ${requestedDuration}ms, exceeding maxDurationMs=${limits.maxDurationMs}.`) + return stripUndefined({ schema: BROWSER_ENVIRONMENT_MATRIX_SCHEMA, id: input.id, seed: input.seed, dimensions, limits, failOnFinding: input.failOnFinding !== false, replayCommand: input.replayCommand, metadata: input.metadata }) +} + +export function expandBrowserEnvironmentMatrix(input: BrowserEnvironmentMatrix): BrowserEnvironmentCell[] { + const matrix = browserEnvironmentMatrix(input) + let selections: Array> = [[]] + for (const dimension of matrix.dimensions) selections = selections.flatMap((selection) => dimension.values.map((value) => [...selection, [dimension.id, value] as [string, BrowserEnvironmentDimensionValue]])) + return selections.map((selection, index) => { + const selected = Object.fromEntries(selection.map(([dimension, value]) => [dimension, value.id])) + const requested = selection.reduce((environment, [, value]) => mergeEnvironment(environment, value.environment), {} as BrowserEnvironment) + const identity = stableJson({ selections: selected, requested }) + const id = createHash("sha256").update(`wp-codebox/browser-environment-cell/v1\n${identity}`).digest("hex").slice(0, 16) + return { + id, + index, + seed: createHash("sha256").update(`wp-codebox/browser-environment-cell-seed/v1\n${matrix.seed}\n${identity}`).digest("hex"), + selections: selected, + requested, + requiredCapabilities: uniqueSorted(selection.flatMap(([, value]) => value.requiredCapabilities ?? [])), + optionalCapabilities: uniqueSorted(selection.flatMap(([, value]) => value.optionalCapabilities ?? [])), + } + }) +} + +export async function runBrowserEnvironmentMatrix(input: BrowserEnvironmentMatrix, options: BrowserEnvironmentMatrixRunnerOptions): Promise { + const matrix = browserEnvironmentMatrix(input) + if (!safeId(options.runId)) throw new Error("Browser environment matrix runId must be a safe non-empty id.") + const cells = expandBrowserEnvironmentMatrix(matrix) + const now = options.now ?? Date.now + const started = now() + const reports: BrowserEnvironmentMatrixCellReport[] = [] + const diagnostics: BrowserEnvironmentMatrixReport["diagnostics"] = [] + let artifactBytes = 0 + let incomplete = false + + for (const cell of cells) { + if (options.signal?.aborted || now() - started >= matrix.limits.maxDurationMs) { + incomplete = true + diagnostics.push({ code: options.signal?.aborted ? "browser-environment-matrix-interrupted" : "browser-environment-matrix-duration-exhausted", message: `Matrix stopped after ${reports.length} completed cells.` }) + break + } + const artifactNamespace = `browser-matrices/${matrix.id}/${options.runId}/cells/${String(cell.index).padStart(4, "0")}-${cell.id}` + const cellStarted = now() + let resolved: ResolvedBrowserEnvironment + let resolutionError: Error | undefined + try { + resolved = await options.resolve(cell) + } catch (error) { + resolutionError = error instanceof Error ? error : new Error(String(error)) + resolved = { effective: cell.requested, capabilities: [] } + } + const capabilityMap = new Map(resolved.capabilities.map((capability) => [capability.id, capability])) + const unsupported = cell.requiredCapabilities.filter((id) => capabilityMap.get(id)?.fidelity === "unsupported" || !capabilityMap.has(id)) + const inconclusive = cell.optionalCapabilities.filter((id) => capabilityMap.get(id)?.fidelity === "unsupported" || !capabilityMap.has(id)) + let execution: BrowserEnvironmentCellExecution + if (resolutionError) { + execution = { status: "error", findings: [{ code: "browser-environment-resolution-error", message: resolutionError.message }] } + } else if (unsupported.length > 0) { + execution = { status: "error", findings: unsupported.map((id) => ({ code: "browser-environment-required-capability-unsupported", message: `Required browser environment capability is unsupported: ${id}` })) } + } else { + execution = await executeBoundedCell(matrix, cell, resolved, artifactNamespace, options) + if (execution.status === "passed" && inconclusive.length > 0) execution = { ...execution, status: "inconclusive" } + } + const findings = (execution.findings ?? []).map((finding) => ({ ...finding, fingerprint: finding.fingerprint ?? browserEnvironmentFindingFingerprint(cell.id, finding) })) + artifactBytes += (execution.artifacts ?? []).reduce((total, artifact) => total + (artifact.bytes ?? 0), 0) + const completedAtMs = now() + reports.push({ + id: cell.id, + index: cell.index, + seed: cell.seed, + selections: cell.selections, + artifactNamespace, + requested: cell.requested, + effective: resolved.effective, + status: execution.status, + findings, + artifacts: execution.artifacts ?? [], + timing: { startedAt: new Date(cellStarted).toISOString(), completedAt: new Date(completedAtMs).toISOString(), durationMs: Math.max(0, completedAtMs - cellStarted) }, + replay: { schema: BROWSER_ENVIRONMENT_REPLAY_SCHEMA, matrixId: matrix.id, matrixSeed: matrix.seed, cellId: cell.id, cellSeed: cell.seed, selections: cell.selections, requested: cell.requested, effective: resolved.effective, provider: resolved.provider, capabilities: resolved.capabilities, artifactNamespace, command: options.replayCommand?.(matrix, cell) ?? matrix.replayCommand ?? `wp-codebox browser-matrix replay --matrix ${matrix.id} --cell ${cell.id} --seed ${cell.seed}` }, + capabilities: resolved.capabilities, + unsupported, + inconclusive, + ...(execution.metadata ? { metadata: execution.metadata } : {}), + }) + if (artifactBytes > matrix.limits.maxArtifactBytes) { + incomplete = true + diagnostics.push({ code: "browser-environment-matrix-artifact-budget-exhausted", message: `Matrix stopped after completed evidence exceeded maxArtifactBytes=${matrix.limits.maxArtifactBytes}.` }) + break + } + } + + const findingCount = reports.reduce((count, cell) => count + cell.findings.length, 0) + const failed = reports.filter((cell) => cell.status === "error" || cell.status === "timed-out").length + return { + schema: BROWSER_ENVIRONMENT_MATRIX_REPORT_SCHEMA, + matrixId: matrix.id, + runId: options.runId, + seed: matrix.seed, + status: incomplete || reports.length < cells.length ? "incomplete" : findingCount > 0 || failed > 0 ? "findings" : "passed", + failOnFinding: matrix.failOnFinding, + cells: reports, + summary: { planned: cells.length, completed: reports.length, passed: reports.filter((cell) => cell.status === "passed").length, findings: findingCount, failed, inconclusive: reports.filter((cell) => cell.status === "inconclusive").length }, + resourceUsage: { durationMs: Math.max(0, now() - started), artifactBytes }, + diagnostics, + } +} + +export function browserEnvironmentFindingFingerprint(cellId: string, finding: Pick): string { + return createHash("sha256").update("wp-codebox/browser-environment-finding/v1\n").update(stableJson({ cellId, finding })).digest("hex") +} + +export function browserEnvironmentMatrixFailed(report: BrowserEnvironmentMatrixReport): boolean { + return report.status === "incomplete" || (report.failOnFinding && report.status === "findings") +} + +async function executeBoundedCell(matrix: BrowserEnvironmentMatrix, cell: BrowserEnvironmentCell, resolved: ResolvedBrowserEnvironment, artifactNamespace: string, options: BrowserEnvironmentMatrixRunnerOptions): Promise { + const controller = new AbortController() + const abort = () => controller.abort(options.signal?.reason) + options.signal?.addEventListener("abort", abort, { once: true }) + let timer: ReturnType | undefined + try { + return await Promise.race([ + options.execute({ cell, resolved, artifactNamespace, signal: controller.signal }), + new Promise((resolve) => { timer = setTimeout(() => { controller.abort(); resolve({ status: "timed-out", findings: [{ code: "browser-environment-cell-timeout", message: `Cell exceeded ${matrix.limits.maxCellDurationMs}ms.` }] }) }, matrix.limits.maxCellDurationMs) }), + ]) + } catch (error) { + return { status: "error", findings: [{ code: "browser-environment-cell-error", message: error instanceof Error ? error.message : String(error) }] } + } finally { + if (timer) clearTimeout(timer) + options.signal?.removeEventListener("abort", abort) + } +} + +function normalizeDimension(dimension: BrowserEnvironmentDimension): BrowserEnvironmentDimension { + if (!safeId(dimension.id) || !Array.isArray(dimension.values) || dimension.values.length === 0) throw new Error(`Invalid browser environment dimension: ${dimension.id}`) + const values = dimension.values.map((value) => { + if (!safeId(value.id)) throw new Error(`Invalid browser environment value id in ${dimension.id}: ${value.id}`) + validateEnvironment(value.environment) + return { id: value.id, environment: canonicalEnvironment(value.environment), requiredCapabilities: uniqueSorted(value.requiredCapabilities ?? []), optionalCapabilities: uniqueSorted(value.optionalCapabilities ?? []) } + }).sort((left, right) => left.id.localeCompare(right.id) || stableJson(left.environment).localeCompare(stableJson(right.environment))) + if (new Set(values.map(({ id }) => id)).size !== values.length) throw new Error(`Browser environment value ids must be unique in ${dimension.id}.`) + return { id: dimension.id, values } +} + +function normalizeLimits(input: Partial | undefined): BrowserEnvironmentMatrixLimits { + return { + maxCells: boundedInteger(input?.maxCells, 32, 1, 10_000), + maxDurationMs: boundedInteger(input?.maxDurationMs, 300_000, 1, 86_400_000), + maxCellDurationMs: boundedInteger(input?.maxCellDurationMs, 30_000, 1, 3_600_000), + maxArtifactBytes: boundedInteger(input?.maxArtifactBytes, 100 * 1_048_576, 1, 4 * 1024 * 1_048_576), + } +} + +function validateEnvironment(environment: BrowserEnvironment): void { + if (!environment || typeof environment !== "object" || Array.isArray(environment)) throw new Error("Browser environment values must be objects.") + if (environment.viewport && (!positiveInteger(environment.viewport.width) || !positiveInteger(environment.viewport.height))) throw new Error("Browser environment viewport width and height must be positive integers.") + if (environment.deviceScaleFactor !== undefined && (!Number.isFinite(environment.deviceScaleFactor) || environment.deviceScaleFactor <= 0)) throw new Error("Browser environment deviceScaleFactor must be positive.") + if (environment.zoom !== undefined && (!Number.isFinite(environment.zoom) || environment.zoom < 0.25 || environment.zoom > 5)) throw new Error("Browser environment zoom must be between 0.25 and 5.") + if (environment.clock?.mode === "fixed" && (!environment.clock.at || !Number.isFinite(Date.parse(environment.clock.at)))) throw new Error("Fixed browser environment clocks require an ISO-compatible at value.") +} + +function canonicalEnvironment(environment: BrowserEnvironment): BrowserEnvironment { + return JSON.parse(stableJson(JSON.parse(JSON.stringify(environment)))) as BrowserEnvironment +} + +function mergeEnvironment(left: BrowserEnvironment, right: BrowserEnvironment): BrowserEnvironment { + const conflicts = Object.keys(right).filter((key) => key !== "capabilities" && key in left && stableJson(left[key as keyof BrowserEnvironment]) !== stableJson(right[key as keyof BrowserEnvironment])) + if (conflicts.length > 0) throw new Error(`Browser environment dimensions assign conflicting values: ${conflicts.sort().join(", ")}.`) + const capabilities = left.capabilities || right.capabilities ? { ...(left.capabilities ?? {}), ...(right.capabilities ?? {}) } : undefined + return canonicalEnvironment({ ...left, ...right, ...(capabilities ? { capabilities } : {}) }) +} + +function safeId(value: string): boolean { return typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(value) } +function positiveInteger(value: number): boolean { return Number.isInteger(value) && value > 0 } +function uniqueSorted(values: string[]): string[] { return [...new Set(values)].sort() } +function boundedInteger(value: number | undefined, fallback: number, minimum: number, maximum: number): number { return Number.isFinite(value) ? Math.max(minimum, Math.min(maximum, Math.floor(value as number))) : fallback } diff --git a/packages/runtime-core/src/contracts.ts b/packages/runtime-core/src/contracts.ts index b09149656..2f35906ee 100644 --- a/packages/runtime-core/src/contracts.ts +++ b/packages/runtime-core/src/contracts.ts @@ -2,6 +2,7 @@ export * from "./browser-probe-contract.js" export * from "./browser-multi-actor-scenario-contracts.js" export * from "./browser-adaptive-exploration.js" +export * from "./browser-environment-matrix.js" export * from "./command-registry.js" export * from "./fuzz-fixture-plan-contracts.js" export * from "./fuzz-coverage-plan-contracts.js" diff --git a/packages/runtime-core/src/index.ts b/packages/runtime-core/src/index.ts index 805f21401..a8ae2fa1a 100644 --- a/packages/runtime-core/src/index.ts +++ b/packages/runtime-core/src/index.ts @@ -49,6 +49,7 @@ export * from "./command-agent-run.js" export * from "./php-worker-runner.js" export * from "./browser-interaction.js" export * from "./browser-adaptive-exploration.js" +export * from "./browser-environment-matrix.js" export * from "./browser-multi-actor-scenario-contracts.js" export * from "./browser-probe-contract.js" export * from "./browser-playground-session-run.js" diff --git a/packages/runtime-core/src/public.ts b/packages/runtime-core/src/public.ts index a095f69a2..54a292d54 100644 --- a/packages/runtime-core/src/public.ts +++ b/packages/runtime-core/src/public.ts @@ -25,6 +25,7 @@ export * from "./browser-artifact-lifecycle.js" export * from "./browser-callback-contracts.js" export * from "./browser-interaction.js" export * from "./browser-adaptive-exploration.js" +export * from "./browser-environment-matrix.js" export * from "./browser-multi-actor-scenario-contracts.js" export * from "./browser-probe-contract.js" export * from "./browser-playground-session-run.js" diff --git a/packages/runtime-playground/src/browser-environment-matrix.ts b/packages/runtime-playground/src/browser-environment-matrix.ts new file mode 100644 index 000000000..322b92b8d --- /dev/null +++ b/packages/runtime-playground/src/browser-environment-matrix.ts @@ -0,0 +1,195 @@ +import { mkdir, writeFile } from "node:fs/promises" +import { join } from "node:path" + +import { + runBrowserEnvironmentMatrix, + type BrowserEnvironment, + type BrowserEnvironmentCapabilityResult, + type BrowserEnvironmentCell, + type BrowserEnvironmentCellExecution, + type BrowserEnvironmentMatrix, + type BrowserEnvironmentMatrixReport, + type ResolvedBrowserEnvironment, +} from "@automattic/wp-codebox-core" +import type { Browser, BrowserContext, BrowserContextOptions, Page } from "playwright" + +export const PLAYWRIGHT_BROWSER_ENVIRONMENT_CAPABILITIES = [ + "browser.environment.viewport", + "browser.environment.device", + "browser.environment.device-scale-factor", + "browser.environment.mobile", + "browser.environment.touch", + "browser.environment.orientation", + "browser.environment.zoom", + "browser.environment.color-scheme", + "browser.environment.reduced-motion", + "browser.environment.forced-colors", + "browser.environment.contrast", + "browser.environment.locale", + "browser.environment.timezone", + "browser.environment.network-profile", + "browser.environment.cpu-profile", + "browser.environment.online-state", + "browser.environment.clock", + "browser.environment.capability-state", +] as const + +export interface BrowserEnvironmentNetworkProfile { + offline?: boolean + latencyMs: number + downloadThroughputBytesPerSecond: number + uploadThroughputBytesPerSecond: number +} + +export interface BrowserEnvironmentCpuProfile { slowdownRate: number } + +export interface PlaywrightBrowserEnvironmentOptions { + providerId?: string + channel?: string + networkProfiles?: Record + cpuProfiles?: Record +} + +export interface PlaywrightBrowserEnvironmentExecutionInput { + browser: Browser + context: BrowserContext + page: Page + cell: BrowserEnvironmentCell + resolved: ResolvedBrowserEnvironment + artifactNamespace: string + signal: AbortSignal +} + +export async function resolvePlaywrightBrowserEnvironment(cell: BrowserEnvironmentCell, browser: Browser, options: PlaywrightBrowserEnvironmentOptions = {}): Promise { + const { devices } = await import("playwright") + const requested = cell.requested + const device = requested.device ? devices[requested.device] : undefined + const capabilities: BrowserEnvironmentCapabilityResult[] = [] + const exact = (id: string) => capabilities.push({ id, fidelity: "exact" }) + const emulated = (id: string, reason: string) => capabilities.push({ id, fidelity: "emulated", reason }) + const unsupported = (id: string, reason: string) => capabilities.push({ id, fidelity: "unsupported", reason }) + + if (requested.viewport) exact("browser.environment.viewport") + if (requested.device) device ? exact("browser.environment.device") : unsupported("browser.environment.device", `Unknown device profile: ${requested.device}`) + if (requested.deviceScaleFactor !== undefined) exact("browser.environment.device-scale-factor") + if (requested.isMobile !== undefined) exact("browser.environment.mobile") + if (requested.hasTouch !== undefined) exact("browser.environment.touch") + if (requested.orientation) exact("browser.environment.orientation") + if (requested.colorScheme) exact("browser.environment.color-scheme") + if (requested.reducedMotion) exact("browser.environment.reduced-motion") + if (requested.forcedColors) exact("browser.environment.forced-colors") + if (requested.contrast) exact("browser.environment.contrast") + if (requested.locale) exact("browser.environment.locale") + if (requested.timezone) exact("browser.environment.timezone") + if (requested.online !== undefined) exact("browser.environment.online-state") + if (requested.clock) exact("browser.environment.clock") + if (requested.capabilities) exact("browser.environment.capability-state") + if (requested.zoom !== undefined) emulated("browser.environment.zoom", "Applied as page scale through the browser debugging protocol; OS-level zoom and browser chrome are outside the page context.") + if (requested.networkProfile) options.networkProfiles?.[requested.networkProfile] ? emulated("browser.environment.network-profile", "Latency and throughput are applied through the browser debugging protocol.") : unsupported("browser.environment.network-profile", `Unknown network profile: ${requested.networkProfile}`) + if (requested.cpuProfile) options.cpuProfiles?.[requested.cpuProfile] ? emulated("browser.environment.cpu-profile", "CPU slowdown is applied through the browser debugging protocol.") : unsupported("browser.environment.cpu-profile", `Unknown CPU profile: ${requested.cpuProfile}`) + + const effective = mergeDeviceEnvironment(requested, device) + return { + effective, + capabilities: capabilities.sort((left, right) => left.id.localeCompare(right.id)), + provider: { id: options.providerId ?? "playwright", browser: browser.browserType().name(), ...(options.channel ? { channel: options.channel } : {}), version: browser.version() }, + } +} + +export async function createPlaywrightBrowserEnvironmentContext(browser: Browser, resolved: ResolvedBrowserEnvironment, options: PlaywrightBrowserEnvironmentOptions = {}): Promise<{ context: BrowserContext; page: Page; close(): Promise }> { + const { devices } = await import("playwright") + const environment = resolved.effective + const device = environment.device ? devices[environment.device] : undefined + const viewport = orientedViewport(environment.viewport ?? device?.viewport ?? undefined, environment.orientation) + const contextOptions: BrowserContextOptions = { + ...(device ?? {}), + ...(viewport ? { viewport, screen: viewport } : {}), + ...(environment.deviceScaleFactor !== undefined ? { deviceScaleFactor: environment.deviceScaleFactor } : {}), + ...(environment.isMobile !== undefined ? { isMobile: environment.isMobile } : {}), + ...(environment.hasTouch !== undefined ? { hasTouch: environment.hasTouch } : {}), + ...(environment.locale ? { locale: environment.locale } : {}), + ...(environment.timezone ? { timezoneId: environment.timezone } : {}), + ...(environment.colorScheme ? { colorScheme: environment.colorScheme } : {}), + ...(environment.reducedMotion ? { reducedMotion: environment.reducedMotion } : {}), + ...(environment.forcedColors ? { forcedColors: environment.forcedColors } : {}), + ...(environment.contrast ? { contrast: environment.contrast } : {}), + ...(environment.online !== undefined ? { offline: !environment.online } : {}), + } + const context = await browser.newContext(contextOptions) + const page = await context.newPage() + await applyPlaywrightPageEnvironment(page, environment, options) + return { context, page, close: () => context.close() } +} + +export async function applyPlaywrightPageEnvironment(page: Page, environment: BrowserEnvironment, options: PlaywrightBrowserEnvironmentOptions = {}): Promise { + if (environment.capabilities) { + await page.addInitScript((capabilities) => { + Object.defineProperty(globalThis, "__browserEnvironmentCapabilities", { value: Object.freeze(capabilities), configurable: false }) + }, environment.capabilities) + } + if (environment.clock?.mode === "fixed" && environment.clock.at) await page.clock.setFixedTime(environment.clock.at) + const network = environment.networkProfile ? options.networkProfiles?.[environment.networkProfile] : undefined + const cpu = environment.cpuProfile ? options.cpuProfiles?.[environment.cpuProfile] : undefined + if (environment.zoom === undefined && !network && !cpu) return + const session = await page.context().newCDPSession(page) + try { + const commands: Array> = [] + if (environment.zoom !== undefined) commands.push(session.send("Emulation.setPageScaleFactor", { pageScaleFactor: environment.zoom })) + if (cpu) commands.push(session.send("Emulation.setCPUThrottlingRate", { rate: cpu.slowdownRate })) + if (network) { + commands.push(session.send("Network.enable")) + commands.push(session.send("Network.emulateNetworkConditions", { offline: network.offline ?? false, latency: network.latencyMs, downloadThroughput: network.downloadThroughputBytesPerSecond, uploadThroughput: network.uploadThroughputBytesPerSecond })) + } + await Promise.all(commands) + } finally { + await session.detach() + } +} + +export async function runPlaywrightBrowserEnvironmentMatrix(input: { + matrix: BrowserEnvironmentMatrix + runId: string + artifactRoot: string + browser: Browser + options?: PlaywrightBrowserEnvironmentOptions + signal?: AbortSignal + execute(input: PlaywrightBrowserEnvironmentExecutionInput): Promise +}): Promise { + const matrixDirectory = join(input.artifactRoot, "browser-matrices", input.matrix.id) + const reportDirectory = join(matrixDirectory, input.runId) + await mkdir(matrixDirectory, { recursive: true }) + await mkdir(reportDirectory) + const report = await runBrowserEnvironmentMatrix(input.matrix, { + runId: input.runId, + signal: input.signal, + resolve: (cell) => resolvePlaywrightBrowserEnvironment(cell, input.browser, input.options), + execute: async ({ cell, resolved, artifactNamespace, signal }) => { + const runtime = await createPlaywrightBrowserEnvironmentContext(input.browser, resolved, input.options) + try { + return await input.execute({ browser: input.browser, context: runtime.context, page: runtime.page, cell, resolved, artifactNamespace, signal }) + } finally { + await runtime.close() + } + }, + }) + await writeFile(join(reportDirectory, "matrix-report.json"), `${JSON.stringify(report, null, 2)}\n`) + return report +} + +function mergeDeviceEnvironment(environment: BrowserEnvironment, device: BrowserContextOptions | undefined): BrowserEnvironment { + const viewport = orientedViewport(environment.viewport ?? device?.viewport ?? undefined, environment.orientation) + return { + ...environment, + ...(viewport ? { viewport } : {}), + ...(device?.deviceScaleFactor !== undefined && environment.deviceScaleFactor === undefined ? { deviceScaleFactor: device.deviceScaleFactor } : {}), + ...(device?.isMobile !== undefined && environment.isMobile === undefined ? { isMobile: device.isMobile } : {}), + ...(device?.hasTouch !== undefined && environment.hasTouch === undefined ? { hasTouch: device.hasTouch } : {}), + } +} + +function orientedViewport(viewport: { width: number; height: number } | undefined, orientation: BrowserEnvironment["orientation"]): { width: number; height: number } | undefined { + if (!viewport || !orientation) return viewport + const long = Math.max(viewport.width, viewport.height) + const short = Math.min(viewport.width, viewport.height) + return orientation === "landscape" ? { width: long, height: short } : { width: short, height: long } +} diff --git a/packages/runtime-playground/src/index.ts b/packages/runtime-playground/src/index.ts index a7b51bc6c..6a74ce646 100644 --- a/packages/runtime-playground/src/index.ts +++ b/packages/runtime-playground/src/index.ts @@ -17,6 +17,7 @@ export { preflightPhpWasmRuntimeAssets, PhpWasmRuntimeAssetIntegrityError, type export { browserPreviewAuthCookieUrls, browserPreviewNetworkPolicySummary, browserPreviewReadinessError, browserPreviewRouting, browserPreviewSecureContextError, browserPreviewTopology, browserPreviewOrigins, resolveBrowserPreviewUrl, type BrowserPreviewNetworkPolicy, type BrowserPreviewTopology } from "./browser-preview-routing.js" export { BROWSER_TRANSPORT_FAULT_CAPABILITIES, applyBrowserTransportFault, createBrowserTransportFaultAdapter, type BrowserTransportFaultAdapter } from "./browser-transport-faults.js" export { PLAYWRIGHT_CLOCK_CONTROL_CAPABILITIES, createBrowserClockController, type BrowserClockController } from "./browser-clock-control.js" +export { PLAYWRIGHT_BROWSER_ENVIRONMENT_CAPABILITIES, applyPlaywrightPageEnvironment, createPlaywrightBrowserEnvironmentContext, resolvePlaywrightBrowserEnvironment, runPlaywrightBrowserEnvironmentMatrix, type BrowserEnvironmentCpuProfile, type BrowserEnvironmentNetworkProfile, type PlaywrightBrowserEnvironmentExecutionInput, type PlaywrightBrowserEnvironmentOptions } from "./browser-environment-matrix.js" export { WORDPRESS_ADVERSARIAL_ADAPTER_SCHEMA, WORDPRESS_ADVERSARIAL_CAPABILITIES, WORDPRESS_ADVERSARIAL_ORACLES, WORDPRESS_CLOCK_CONTROL_CAPABILITIES, WORDPRESS_HTTP_TRANSPORT_FAULT_CAPABILITIES, createWordPressAdversarialAdapter, evaluateWordPressAdversarialOracles, negotiateWordPressHttpTransportFaults, wordpressAdversarialActionSpec, wordpressHttpFaultConfigurationAction, wordpressNoveltySignals, wordpressSchedulerClockAction, type WordPressAdapterFidelity, type WordPressAdversarialAction, type WordPressAdversarialAdapter, type WordPressAdversarialCapability, type WordPressAdversarialSurface } from "./wordpress-adversarial-adapter.js" export { normalizePreviewReviewerAccess, previewReviewerAccess } from "./preview-reviewer-access.js" export { applyVfsMountSnapshots, materializePlaygroundMountsFromVfs, materializePlaygroundStagedInputs, type HostMountSnapshot, type MountMaterializationResult, type StagedInputMaterializationResult, type VfsMountSnapshot } from "./mount-materialization.js" diff --git a/tests/browser-environment-matrix.browser.test.ts b/tests/browser-environment-matrix.browser.test.ts new file mode 100644 index 000000000..c4fd8087e --- /dev/null +++ b/tests/browser-environment-matrix.browser.test.ts @@ -0,0 +1,83 @@ +import assert from "node:assert/strict" +import { mkdtemp, readFile, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import test from "node:test" + +import { chromium } from "playwright" + +import { browserEnvironmentMatrix } from "../packages/runtime-core/src/browser-environment-matrix.js" +import { runPlaywrightBrowserEnvironmentMatrix } from "../packages/runtime-playground/src/browser-environment-matrix.js" + +test("real browser matrix applies viewport, media, locale, timezone, touch, zoom, and throttling", async () => { + const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-matrix-")) + const browser = await chromium.launch({ headless: true }) + try { + const matrix = browserEnvironmentMatrix({ + id: "browser-smoke", + seed: "browser-smoke-seed", + dimensions: [ + { id: "device", values: [{ id: "narrow-touch", environment: { viewport: { width: 320, height: 640 }, deviceScaleFactor: 2, isMobile: true, hasTouch: true, orientation: "portrait", zoom: 2 }, requiredCapabilities: ["browser.environment.viewport", "browser.environment.touch", "browser.environment.zoom"] }] }, + { id: "locale", values: [{ id: "french", environment: { locale: "fr-FR", timezone: "Europe/Paris" }, requiredCapabilities: ["browser.environment.locale", "browser.environment.timezone"] }] }, + { id: "media", values: [{ id: "hostile", environment: { colorScheme: "dark", reducedMotion: "reduce", forcedColors: "active", contrast: "more" }, requiredCapabilities: ["browser.environment.color-scheme", "browser.environment.reduced-motion", "browser.environment.forced-colors"] }] }, + { id: "resources", values: [{ id: "slow", environment: { networkProfile: "slow", cpuProfile: "slow" }, optionalCapabilities: ["browser.environment.network-profile", "browser.environment.cpu-profile"] }] }, + ], + limits: { maxCells: 1, maxDurationMs: 30_000, maxCellDurationMs: 30_000, maxArtifactBytes: 1_048_576 }, + }) + const report = await runPlaywrightBrowserEnvironmentMatrix({ + matrix, + runId: "smoke", + artifactRoot, + browser, + options: { + networkProfiles: { slow: { latencyMs: 20, downloadThroughputBytesPerSecond: 100_000, uploadThroughputBytesPerSecond: 50_000 } }, + cpuProfiles: { slow: { slowdownRate: 2 } }, + }, + execute: async ({ page }) => { + await page.goto("data:text/html,
matrix
") + const observed = await page.evaluate(() => ({ + width: innerWidth, + language: navigator.language, + timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, + touch: navigator.maxTouchPoints, + dark: matchMedia("(prefers-color-scheme: dark)").matches, + reduced: matchMedia("(prefers-reduced-motion: reduce)").matches, + forced: matchMedia("(forced-colors: active)").matches, + contrast: matchMedia("(prefers-contrast: more)").matches, + })) + assert.equal(observed.width, 320) + assert.equal(observed.language, "fr-FR") + assert.equal(observed.timezone, "Europe/Paris") + assert(observed.touch > 0) + assert.equal(observed.dark, true) + assert.equal(observed.reduced, true) + assert.equal(observed.forced, true) + assert.equal(observed.contrast, true) + return { status: "passed" } + }, + }) + assert.equal(report.status, "passed") + assert.equal(report.cells[0]?.effective.zoom, 2) + assert.equal(report.cells[0]?.capabilities.find(({ id }) => id === "browser.environment.zoom")?.fidelity, "emulated") + assert.equal(report.cells[0]?.capabilities.find(({ id }) => id === "browser.environment.cpu-profile")?.fidelity, "emulated") + const persisted = JSON.parse(await readFile(join(artifactRoot, "browser-matrices/browser-smoke/smoke/matrix-report.json"), "utf8")) + assert.deepEqual(persisted, report) + } finally { + await browser.close() + await rm(artifactRoot, { recursive: true, force: true }) + } +}) + +test("real browser matrix refuses a colliding run artifact namespace", async () => { + const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-matrix-collision-")) + const browser = await chromium.launch({ headless: true }) + const matrix = browserEnvironmentMatrix({ id: "collision", seed: "seed", dimensions: [{ id: "default", values: [{ id: "default", environment: {} }] }], limits: { maxCells: 1, maxDurationMs: 10_000, maxCellDurationMs: 10_000, maxArtifactBytes: 1024 } }) + try { + const run = () => runPlaywrightBrowserEnvironmentMatrix({ matrix, runId: "same", artifactRoot, browser, execute: async () => ({ status: "passed" }) }) + await run() + await assert.rejects(run(), (error: NodeJS.ErrnoException) => error.code === "EEXIST") + } finally { + await browser.close() + await rm(artifactRoot, { recursive: true, force: true }) + } +}) diff --git a/tests/browser-environment-matrix.test.ts b/tests/browser-environment-matrix.test.ts new file mode 100644 index 000000000..c757537a7 --- /dev/null +++ b/tests/browser-environment-matrix.test.ts @@ -0,0 +1,155 @@ +import assert from "node:assert/strict" +import test from "node:test" + +import { + browserEnvironmentMatrix, + browserEnvironmentMatrixFailed, + expandBrowserEnvironmentMatrix, + runBrowserEnvironmentMatrix, + type BrowserEnvironmentMatrix, + type ResolvedBrowserEnvironment, +} from "../packages/runtime-core/src/browser-environment-matrix.js" + +const limits = { maxCells: 16, maxDurationMs: 16_000, maxCellDurationMs: 1_000, maxArtifactBytes: 1024 } + +function fixtureMatrix(dimensions: BrowserEnvironmentMatrix["dimensions"]): BrowserEnvironmentMatrix { + return browserEnvironmentMatrix({ id: "environment-fixture", seed: "fixture-seed", dimensions, limits }) +} + +const dimensions: BrowserEnvironmentMatrix["dimensions"] = [ + { + id: "viewport", + values: [ + { id: "wide", environment: { viewport: { width: 1280, height: 720 } }, requiredCapabilities: ["browser.environment.viewport"] }, + { id: "narrow", environment: { viewport: { width: 320, height: 640 }, isMobile: true, hasTouch: true }, requiredCapabilities: ["browser.environment.viewport"] }, + ], + }, + { + id: "media", + values: [ + { id: "reduced", environment: { reducedMotion: "reduce" }, optionalCapabilities: ["browser.environment.reduced-motion"] }, + { id: "dark", environment: { colorScheme: "dark" }, optionalCapabilities: ["browser.environment.color-scheme"] }, + ], + }, +] + +test("environment matrices expand canonically independent of declaration order", () => { + const first = expandBrowserEnvironmentMatrix(fixtureMatrix(dimensions)) + const reordered = expandBrowserEnvironmentMatrix(fixtureMatrix([...dimensions].reverse().map((dimension) => ({ ...dimension, values: [...dimension.values].reverse() })))) + assert.deepEqual(first, reordered) + assert.deepEqual(first.map((cell) => cell.selections), [ + { media: "dark", viewport: "narrow" }, + { media: "dark", viewport: "wide" }, + { media: "reduced", viewport: "narrow" }, + { media: "reduced", viewport: "wide" }, + ]) + assert.equal(new Set(first.map((cell) => cell.seed)).size, 4) +}) + +test("environment matrices reject combinatorial and duration budget explosions", () => { + assert.throws(() => browserEnvironmentMatrix({ id: "too-many", seed: "seed", dimensions, limits: { ...limits, maxCells: 3 } }), /exceeding maxCells=3/) + assert.throws(() => browserEnvironmentMatrix({ id: "too-slow", seed: "seed", dimensions, limits: { ...limits, maxDurationMs: 3_999 } }), /exceeding maxDurationMs=3999/) +}) + +test("environment dimensions merge independent capability state", () => { + const [cell] = expandBrowserEnvironmentMatrix(fixtureMatrix([ + { id: "first", values: [{ id: "first", environment: { capabilities: { alpha: true } } }] }, + { id: "second", values: [{ id: "second", environment: { capabilities: { beta: "enabled" } } }] }, + ])) + assert.deepEqual(cell?.requested.capabilities, { alpha: true, beta: "enabled" }) +}) + +test("matrix cells use isolated run and cell artifact namespaces", async () => { + const namespaces: string[] = [] + const report = await runBrowserEnvironmentMatrix(fixtureMatrix(dimensions), { + runId: "run-a", + resolve: resolveAll, + execute: async ({ artifactNamespace }) => { namespaces.push(artifactNamespace); return { status: "passed" } }, + }) + const other = await runBrowserEnvironmentMatrix(fixtureMatrix(dimensions), { runId: "run-b", resolve: resolveAll, execute: async () => ({ status: "passed" }) }) + assert.equal(new Set(namespaces).size, 4) + assert(namespaces.every((namespace) => namespace.startsWith("browser-matrices/environment-fixture/run-a/cells/"))) + assert.notEqual(report.cells[0]?.artifactNamespace, other.cells[0]?.artifactNamespace) +}) + +test("matrix execution retains completed evidence after failures and artifact exhaustion", async () => { + let now = 0 + const report = await runBrowserEnvironmentMatrix(fixtureMatrix(dimensions), { + runId: "partial", + now: () => now += 10, + resolve: resolveAll, + execute: async ({ cell, artifactNamespace }) => cell.index === 1 + ? Promise.reject(new Error("fixture failure")) + : { status: "passed", artifacts: [{ path: `${artifactNamespace}/summary.json`, kind: "json", bytes: cell.index === 2 ? 2048 : 10 }] }, + }) + assert.equal(report.status, "incomplete") + assert.equal(report.cells.length, 3) + assert.equal(report.cells[0]?.status, "passed") + assert.equal(report.cells[1]?.status, "error") + assert.match(report.cells[1]?.findings[0]?.message ?? "", /fixture failure/) + assert.equal(report.cells[2]?.artifacts[0]?.bytes, 2048) + assert(report.diagnostics.some(({ code }) => code === "browser-environment-matrix-artifact-budget-exhausted")) + assert.equal(browserEnvironmentMatrixFailed(report), true) +}) + +test("capability resolution errors retain earlier completed cells", async () => { + const matrix = fixtureMatrix([{ id: "scheme", values: [ + { id: "dark", environment: { colorScheme: "dark" } }, + { id: "light", environment: { colorScheme: "light" } }, + ] }]) + const report = await runBrowserEnvironmentMatrix(matrix, { + runId: "resolution-error", + resolve: (cell) => { if (cell.index === 1) throw new Error("resolver unavailable"); return resolveAll(cell) }, + execute: async () => ({ status: "passed" }), + }) + assert.equal(report.cells.length, 2) + assert.equal(report.cells[0]?.status, "passed") + assert.equal(report.cells[1]?.status, "error") + assert.match(report.cells[1]?.findings[0]?.message ?? "", /resolver unavailable/) +}) + +test("required capabilities fail closed while optional capabilities are inconclusive", async () => { + const matrix = fixtureMatrix([{ + id: "capability", + values: [ + { id: "optional", environment: { reducedMotion: "reduce" }, optionalCapabilities: ["browser.environment.reduced-motion"] }, + { id: "required", environment: { forcedColors: "active" }, requiredCapabilities: ["browser.environment.forced-colors"] }, + ], + }]) + let executions = 0 + const report = await runBrowserEnvironmentMatrix(matrix, { + runId: "unsupported", + resolve: (cell) => ({ effective: cell.requested, capabilities: [] }), + execute: async () => { executions += 1; return { status: "passed" } }, + }) + assert.equal(executions, 1) + assert.equal(report.cells[0]?.status, "inconclusive") + assert.deepEqual(report.cells[0]?.inconclusive, ["browser.environment.reduced-motion"]) + assert.equal(report.cells[1]?.status, "error") + assert.deepEqual(report.cells[1]?.unsupported, ["browser.environment.forced-colors"]) +}) + +test("finding fingerprints include the environment cell and fail-on-finding remains configurable", async () => { + const matrix = fixtureMatrix([{ id: "scheme", values: [ + { id: "dark", environment: { colorScheme: "dark" } }, + { id: "light", environment: { colorScheme: "light" } }, + ] }]) + const report = await runBrowserEnvironmentMatrix(matrix, { + runId: "findings", + resolve: resolveAll, + execute: async () => ({ status: "findings", findings: [{ code: "fixture", message: "same finding" }] }), + }) + assert.equal(report.status, "findings") + assert.notEqual(report.cells[0]?.findings[0]?.fingerprint, report.cells[1]?.findings[0]?.fingerprint) + assert.equal(browserEnvironmentMatrixFailed(report), true) + const advisory = await runBrowserEnvironmentMatrix({ ...matrix, failOnFinding: false }, { runId: "advisory", resolve: resolveAll, execute: async () => ({ status: "findings", findings: [{ code: "fixture", message: "same finding" }] }) }) + assert.equal(browserEnvironmentMatrixFailed(advisory), false) +}) + +function resolveAll(cell: { requested: BrowserEnvironmentMatrix["dimensions"][number]["values"][number]["environment"] }): ResolvedBrowserEnvironment { + return { effective: cell.requested, capabilities: [ + { id: "browser.environment.viewport", fidelity: "exact" }, + { id: "browser.environment.reduced-motion", fidelity: "exact" }, + { id: "browser.environment.color-scheme", fidelity: "exact" }, + ] } +} diff --git a/tests/public-api-contract.test.ts b/tests/public-api-contract.test.ts index 7ecb30c7b..273d4f41c 100644 --- a/tests/public-api-contract.test.ts +++ b/tests/public-api-contract.test.ts @@ -160,6 +160,7 @@ assert.deepEqual(barrelExportModules(publicBarrel), [ "./browser-callback-contracts.js", "./browser-interaction.js", "./browser-adaptive-exploration.js", + "./browser-environment-matrix.js", "./browser-multi-actor-scenario-contracts.js", "./browser-probe-contract.js", "./browser-playground-session-run.js", @@ -244,6 +245,7 @@ assert.deepEqual(barrelExportModules(contractsBarrel), [ "./browser-probe-contract.js", "./browser-multi-actor-scenario-contracts.js", "./browser-adaptive-exploration.js", + "./browser-environment-matrix.js", "./command-registry.js", "./fuzz-fixture-plan-contracts.js", "./fuzz-coverage-plan-contracts.js",