From 86a8e2982541b8d0e00f2e1a3091a89eddad8710 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Tue, 21 Jul 2026 12:39:45 -0400 Subject: [PATCH 01/17] Execute bounded PHP commands in clean Playground processes --- .../runtime-playground/src/php-bootstrap.ts | 10 ++-- .../src/phpunit-command-handlers.ts | 5 ++ .../src/playground-cli-runner.ts | 54 +++---------------- .../src/playground-runtime.ts | 49 +++-------------- .../runtime-playground/src/preview-server.ts | 2 +- ...isposable-mysql-mysqli.integration.test.ts | 2 +- tests/phpunit-project-autoload.test.ts | 1 + ...layground-cli-runner-bootstrap-ini.test.ts | 20 ++----- 8 files changed, 34 insertions(+), 109 deletions(-) diff --git a/packages/runtime-playground/src/php-bootstrap.ts b/packages/runtime-playground/src/php-bootstrap.ts index 3995da35..9c46fafe 100644 --- a/packages/runtime-playground/src/php-bootstrap.ts +++ b/packages/runtime-playground/src/php-bootstrap.ts @@ -21,11 +21,15 @@ ${phpBody(code)}` } export function bootstrapPhpCode(spec: RuntimeCreateSpec, code: string, args: string[], wpCliBridge?: PhpBootstrapBridge, failureDiagnosticFile?: string): string { + const command = splitLeadingStrictTypesDeclare(code) if (argValue(args, "bootstrap") === "none") { - return code + return ` or declare phpunit bootstrap'); diff --git a/packages/runtime-playground/src/playground-cli-runner.ts b/packages/runtime-playground/src/playground-cli-runner.ts index 2f3266e3..cbd8bd8f 100644 --- a/packages/runtime-playground/src/playground-cli-runner.ts +++ b/packages/runtime-playground/src/playground-cli-runner.ts @@ -3,7 +3,7 @@ import { PlaygroundCliExitError, type PlaygroundCliBufferedOutput } from "./play import { PlaygroundPreviewPortUnavailableError, assertPreviewPortAvailable, errorHasCode, withPreviewProxy, type PlaygroundCliServer } from "./preview-server.js" import { startProgrammaticPlaygroundServer } from "./programmatic-playground-runner.js" import { normalizeLiveProgressEvent, previewLease, type BrowserStartupProgressEvent, type BrowserStartupProgressPhase, type BrowserStartupProgressStatus, type MountSpec, type PreviewLease, type RuntimeCreateSpec, type RuntimePreviewLeaseProvider } from "@automattic/wp-codebox-core" -import { randomBytes, randomInt } from "node:crypto" +import { randomInt } from "node:crypto" import { existsSync } from "node:fs" import { createServer as createHttpServer, type Server as HttpServer } from "node:http" import { mkdir, readFile, rename, rm, stat, unlink, utimes, writeFile } from "node:fs/promises" @@ -80,11 +80,6 @@ export async function startPlaygroundCliServer(spec: RuntimeCreateSpec, mounts: const wordpressInstallMode = spec.environment.wordpressInstallMode ?? "install-from-existing-files" const bootstrapIniEntries = runtimeBootstrapPhpIniEntries(spec) const useProgrammaticRunner = shouldUseProgrammaticPlaygroundRunner(spec, options) - const requestWorkerEndpoint = useProgrammaticRunner ? undefined : { - route: `/wp-codebox-execute-${randomBytes(12).toString("hex")}.php`, - token: randomBytes(32).toString("base64url"), - payloadDirectory: join(spec.artifactsDirectory ?? "artifacts", "playground-internal-shared"), - } usesArchiveCache = !wordpressDirectory && !spec.environment.assets?.wordpressZip readonlyMountStaging = await stageReadonlyPlaygroundMounts(mounts) const stagedMounts = readonlyMountStaging.mounts @@ -137,7 +132,7 @@ export async function startPlaygroundCliServer(spec: RuntimeCreateSpec, mounts: }, Boolean(spec.preview?.port)) : await startPlaygroundCliWithDynamicPortRetry(async (port) => { const { runCLI } = options.cliModule ?? (await import("@wp-playground/cli")) as unknown as PlaygroundCliModule const localAssetServer = wordpressStartupAsset?.localPath ? await serveLocalStartupAsset(wordpressStartupAsset.localPath) : undefined - const bootstrapSharedMounts = await pluginRuntimeBootstrapSharedMounts(spec, requestWorkerEndpoint) + const bootstrapSharedMounts = await pluginRuntimeBootstrapSharedMounts(spec) try { return await runCLI({ command: "server", @@ -179,7 +174,7 @@ export async function startPlaygroundCliServer(spec: RuntimeCreateSpec, mounts: fixedPreviewPort: spec.preview?.port ?? null, }) - const proxiedServer = await withPreviewLeaseProvider(await withPreviewProxy({ ...server, ...(requestWorkerEndpoint ? { requestWorkerEndpoint } : {}) }, spec.preview?.port ?? 0, spec.preview?.bind), spec) + const proxiedServer = await withPreviewLeaseProvider(await withPreviewProxy(server, spec.preview?.port ?? 0, spec.preview?.bind), spec) emitProgress("preview:ready", "complete", "Preview ready", { localUrl: proxiedServer.serverUrl, upstreamUrl: server.serverUrl, @@ -303,9 +298,10 @@ export function shouldUseProgrammaticPlaygroundRunner(spec: RuntimeCreateSpec, o && (Boolean(runtimeBootstrapPhpIniEntries(spec)) || Boolean(spec.environment.extensions?.length)) } -async function pluginRuntimeBootstrapSharedMounts(spec: RuntimeCreateSpec, requestWorkerEndpoint?: { route: string; token: string; payloadDirectory: string }): Promise> { +async function pluginRuntimeBootstrapSharedMounts(spec: RuntimeCreateSpec): Promise> { const iniEntries = runtimeBootstrapPhpIniEntries(spec) - if (!iniEntries && !requestWorkerEndpoint) { + const externalWpConfig = externalDatabaseWpConfig(spec) + if (!iniEntries && !externalWpConfig) { return [] } @@ -315,8 +311,6 @@ async function pluginRuntimeBootstrapSharedMounts(spec: RuntimeCreateSpec, reque await writeFile(join(directory, "php.ini"), phpIniContent(iniEntries), "utf8") await writeFile(join(directory, "wp-codebox-auto-prepend.php"), runtimeAutoPrependPhp(spec), "utf8") } - if (requestWorkerEndpoint) await writeFile(join(directory, "request-worker.php"), requestWorkerPhp(requestWorkerEndpoint.token), "utf8") - const externalWpConfig = externalDatabaseWpConfig(spec) if (externalWpConfig) await writeFile(join(directory, "wp-config.php"), externalWpConfig, "utf8") return [ @@ -324,46 +318,10 @@ async function pluginRuntimeBootstrapSharedMounts(spec: RuntimeCreateSpec, reque { hostPath: join(directory, "php.ini"), vfsPath: "/internal/shared/php.ini" }, { hostPath: join(directory, "wp-codebox-auto-prepend.php"), vfsPath: "/internal/shared/wp-codebox-auto-prepend.php" }, ] : []), - ...(requestWorkerEndpoint ? [ - { hostPath: directory, vfsPath: "/internal/wp-codebox" }, - { hostPath: join(directory, "request-worker.php"), vfsPath: `/wordpress${requestWorkerEndpoint.route}` }, - ] : []), ...(externalWpConfig ? [{ hostPath: join(directory, "wp-config.php"), vfsPath: "/wordpress/wp-config.php" }] : []), ] } -function requestWorkerPhp(token: string): string { - return ` $wp_codebox_value) { - if (!is_string($wp_codebox_name) || !is_string($wp_codebox_value)) { - http_response_code(400); - exit; - } - putenv($wp_codebox_name . '=' . $wp_codebox_value); - $_ENV[$wp_codebox_name] = $wp_codebox_value; - $_SERVER[$wp_codebox_name] = $wp_codebox_value; -} -eval('?>' . $wp_codebox_code); -` -} - function runtimeBootstrapPhpIniEntries(spec: RuntimeCreateSpec): Record | undefined { const entries = pluginRuntimeBootstrapPhpIniEntries(spec) ?? {} if (Object.keys(entries).length === 0 && !runtimeAutoPrependPhpBody(spec)) { diff --git a/packages/runtime-playground/src/playground-runtime.ts b/packages/runtime-playground/src/playground-runtime.ts index f0598ad5..076092c8 100644 --- a/packages/runtime-playground/src/playground-runtime.ts +++ b/packages/runtime-playground/src/playground-runtime.ts @@ -1,6 +1,6 @@ import { randomBytes } from "node:crypto" import { AsyncLocalStorage } from "node:async_hooks" -import { mkdir, readFile, realpath, unlink, writeFile } from "node:fs/promises" +import { mkdir, readFile, realpath, writeFile } from "node:fs/promises" import type { IncomingMessage, ServerResponse } from "node:http" import { dirname, join, resolve } from "node:path" import { HostToolRegistry, PREVIEW_LEASE_SCHEMA, RUNTIME_EPISODE_OBSERVATION_SCHEMA, RUNTIME_EPISODE_SNAPSHOT_SCHEMA, RuntimeActionExecutionError, assertRuntimeCommandAllowed, commandAgentRunResultJson, createCommandAgentRunResult, createHostToolRegistry, createRuntimeCommandResultEnvelope, parseCommandAgentRunRequest, previewLease, resolveArtifactPath, resolveCommandPath, runtimeCommandResultEnvelopeFromOutput, runtimeEpisodeDigest } from "@automattic/wp-codebox-core" @@ -251,8 +251,7 @@ class PlaygroundRuntime implements Runtime { private cliServerPromise?: Promise private readonly activeExecutionAbortControllers = new Set() private readonly executionSignals = new AsyncLocalStorage() - private readonly requestWorkerExecutions = new AsyncLocalStorage>() - private requestWorkerReady?: Promise + private readonly isolatedProcessExecutions = new AsyncLocalStorage() private reviewerAuthBootstrapRouteRegistered = false private readonly reviewerAuthBootstraps = new Map() @@ -365,9 +364,7 @@ class PlaygroundRuntime implements Runtime { this.activeExecutionAbortControllers.add(abortController) try { const executionSpec = executionSpecWithEnvironment(spec) - const output = await this.executionSignals.run(abortController.signal, async () => spec.processIdentity - ? await this.requestWorkerExecutions.run(spec.environment ?? {}, async () => await timeoutPlaygroundCommand(executePlaygroundCommand(this, executionSpec, this.hostTools), spec, abortController)) - : await timeoutPlaygroundCommand(executePlaygroundCommand(this, executionSpec, this.hostTools), spec, abortController)) + const output = await this.executionSignals.run(abortController.signal, async () => await this.isolatedProcessExecutions.run(Boolean(spec.processIdentity), async () => await timeoutPlaygroundCommand(executePlaygroundCommand(this, executionSpec, this.hostTools), spec, abortController))) const finishedAt = now() const envelope = typeof output === "string" ? runtimeCommandResultEnvelopeFromOutput({ @@ -1769,11 +1766,11 @@ class PlaygroundRuntime implements Runtime { private async runPlaygroundCommand(command: string, server: PlaygroundCliServer, options: { code: string } | { scriptPath: string }): Promise { try { - const requestWorkerEnvironment = this.requestWorkerExecutions.getStore() - if (requestWorkerEnvironment && "code" in options && server.requestWorkerEndpoint) { - await this.prepareRequestWorker(server) - const response = await this.executeRequestWorker(server, options.code, requestWorkerEnvironment, this.executionSignals.getStore()) - return { text: response.text, exitCode: response.ok ? 0 : 1, ...(!response.ok ? { errors: response.text } : {}) } + if (this.isolatedProcessExecutions.getStore() && "code" in options) { + if (!server.playground.runInFreshProcess) { + throw new Error("The Playground runtime does not support clean PHP process execution.") + } + return await abortable(server.playground.runInFreshProcess(options), this.executionSignals.getStore()) } return await abortable(server.playground.run(options), this.executionSignals.getStore()) } catch (error) { @@ -1786,36 +1783,6 @@ class PlaygroundRuntime implements Runtime { } } - private async prepareRequestWorker(server: PlaygroundCliServer): Promise { - if (!server.requestWorkerEndpoint) return - this.requestWorkerReady ??= (async () => { - const response = await this.executeRequestWorker(server, ", signal?: AbortSignal): Promise<{ ok: boolean; status: number; text: string }> { - const endpoint = server.requestWorkerEndpoint - if (!endpoint) throw new Error("Playground request worker endpoint is unavailable.") - const payloadId = randomBytes(16).toString("hex") - const payloadPath = join(endpoint.payloadDirectory, `execution-${payloadId}.json`) - await writeFile(payloadPath, JSON.stringify({ code, environment }), "utf8") - try { - const response = await fetch(new URL(endpoint.route, server.serverUrl), { - method: "POST", - headers: { - "X-WP-Codebox-Execution-Token": endpoint.token, - "X-WP-Codebox-Execution-Payload": payloadId, - }, - signal, - }) - return { ok: response.ok, status: response.status, text: await response.text() } - } finally { - await unlink(payloadPath).catch(() => undefined) - } - } - async inspectMountedInputs(): Promise { const server = await this.bootPlayground() const response = await server.playground.run({ diff --git a/packages/runtime-playground/src/preview-server.ts b/packages/runtime-playground/src/preview-server.ts index 79e3b454..8601d400 100644 --- a/packages/runtime-playground/src/preview-server.ts +++ b/packages/runtime-playground/src/preview-server.ts @@ -11,12 +11,12 @@ export interface PlaygroundServerRunResponse { export interface PlaygroundCliServer { playground: { run(options: { code: string } | { scriptPath: string }): Promise + runInFreshProcess?(options: { code: string }): Promise onMessage?(listener: (data: string) => Promise | string | void): Promise<(() => Promise | void) | void> | (() => Promise | void) | void readFileAsText?(path: string): string | Promise writeFile?(path: string, contents: string): Promise } serverUrl: string - requestWorkerEndpoint?: { route: string; token: string; payloadDirectory: string } previewLease?: PreviewLease previewRoutes?: PlaygroundPreviewRouteRegistry previewProxyDiagnostics?: PlaygroundPreviewProxyDiagnostics diff --git a/tests/disposable-mysql-mysqli.integration.test.ts b/tests/disposable-mysql-mysqli.integration.test.ts index 0a404e0a..92c4b861 100644 --- a/tests/disposable-mysql-mysqli.integration.test.ts +++ b/tests/disposable-mysql-mysqli.integration.test.ts @@ -77,7 +77,7 @@ if (!await dockerAvailable()) { await mkdir(join(plugin, "tests"), { recursive: true }) await writeFile(join(plugin, "bounded-phpunit-fixture.php"), "tests\n") - await writeFile(join(plugin, "tests", "bootstrap.php"), "xpath('//testsuite/file') ?: array() as $file)")) assert.ok(projectModeCode.includes("list($directories, $suffixes, $prefixes, $excludes, $configured_files) = wp_codebox_phpunit_parse_config")) assert.ok(projectModeCode.includes("$test_files = wp_codebox_phpunit_discover($directories, $suffixes, $prefixes, $excludes, $configured_files);")) diff --git a/tests/playground-cli-runner-bootstrap-ini.test.ts b/tests/playground-cli-runner-bootstrap-ini.test.ts index 918491de..9e94d29e 100644 --- a/tests/playground-cli-runner-bootstrap-ini.test.ts +++ b/tests/playground-cli-runner-bootstrap-ini.test.ts @@ -64,14 +64,12 @@ try { await server[Symbol.asyncDispose]() assert.equal(calls.length, 1) - assert.equal(calls[0]["mount-before-install"]?.length, 6) + assert.equal(calls[0]["mount-before-install"]?.length, 4) assert.equal(calls[0]["mount-before-install"]?.[0]?.vfsPath, "/internal/shared/php.ini") assert.equal(calls[0]["mount-before-install"]?.[1]?.vfsPath, "/internal/shared/wp-codebox-auto-prepend.php") - assert.equal(calls[0]["mount-before-install"]?.[2]?.vfsPath, "/internal/wp-codebox") - assert.match(calls[0]["mount-before-install"]?.[3]?.vfsPath ?? "", /^\/wordpress\/wp-codebox-execute-[a-f0-9]{24}\.php$/) // A wordpress-develop checkout is the runtime root, not an ordinary post-startup mount. - assert.equal(calls[0]["mount-before-install"]?.[4]?.vfsPath, "/wordpress/wp-config.php") - assert.deepEqual(calls[0]["mount-before-install"]?.[5], { hostPath: wordpressDevelopDirectory, vfsPath: "/wordpress" }) + assert.equal(calls[0]["mount-before-install"]?.[2]?.vfsPath, "/wordpress/wp-config.php") + assert.deepEqual(calls[0]["mount-before-install"]?.[3], { hostPath: wordpressDevelopDirectory, vfsPath: "/wordpress" }) assert.deepEqual(calls[0].mount, []) assert.equal(calls[0].workers, 6) assert.equal(calls[0].wordpressInstallMode, "do-not-attempt-installing") @@ -92,13 +90,7 @@ try { const sharedAutoPrepend = await readFile(sharedAutoPrependPath as string, "utf8") assert.match(sharedAutoPrepend, /require_once '\/internal\/shared\/auto_prepend_file\.php'/) assert.match(sharedAutoPrepend, /putenv\("TC_MYSQL_PORT=33060"\);/) - const requestWorkerPath = calls[0]["mount-before-install"]?.[3]?.hostPath - assert.equal(typeof requestWorkerPath, "string") - const requestWorker = await readFile(requestWorkerPath as string, "utf8") - assert.match(requestWorker, /HTTP_X_WP_CODEBOX_EXECUTION_TOKEN/) - assert.match(requestWorker, /hash_equals/) - assert.match(requestWorker, /\$_ENV\[\$wp_codebox_name\] = \$wp_codebox_value/) - const externalWpConfigPath = calls[0]["mount-before-install"]?.[4]?.hostPath + const externalWpConfigPath = calls[0]["mount-before-install"]?.[2]?.hostPath assert.equal(typeof externalWpConfigPath, "string") const externalWpConfig = await readFile(externalWpConfigPath as string, "utf8") assert.match(externalWpConfig, /define\('DB_HOST', "127\.0\.0\.1:33061"\)/) @@ -134,9 +126,7 @@ try { await downloadedWordPressServer[Symbol.asyncDispose]() assert.equal(calls.length, 1) - assert.equal(calls[0]["mount-before-install"]?.length, 2) - assert.equal(calls[0]["mount-before-install"]?.[0]?.vfsPath, "/internal/wp-codebox") - assert.match(calls[0]["mount-before-install"]?.[1]?.vfsPath ?? "", /^\/wordpress\/wp-codebox-execute-[a-f0-9]{24}\.php$/) + assert.equal(calls[0]["mount-before-install"], undefined) assert.equal(calls[0].wordpressInstallMode, undefined) assert.equal(shouldUseProgrammaticPlaygroundRunner(downloadedWordPressSpec), false) From 8662de9974e0002b7c89f8b3d17ed2820b6dc008 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Tue, 21 Jul 2026 13:10:40 -0400 Subject: [PATCH 02/17] Decode wrapped PHPUnit bootstrap assertions --- tests/phpunit-project-autoload.test.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/phpunit-project-autoload.test.ts b/tests/phpunit-project-autoload.test.ts index 8a248812..63ab6fea 100644 --- a/tests/phpunit-project-autoload.test.ts +++ b/tests/phpunit-project-autoload.test.ts @@ -540,12 +540,17 @@ echo json_encode(array($legacy_project_autoload_file, $harness_autoload_file)); `) assert.deepEqual(JSON.parse(execFileSync("php", [canonicalHarnessProbe], { encoding: "utf8" })), ["", "/tmp/wp-codebox-inputs/0-wp-codebox-vendor-73845ca47d2f/autoload.php"], "a canonical staged harness path remains the harness in project mode") +function decodeSubmittedBootstrap(code: string): string { + const encodedBootstrap = code.match(/base64_decode\("([A-Za-z0-9+/=]+)"\)/)?.[1] + return encodedBootstrap ? Buffer.from(encodedBootstrap, "base64").toString("utf8") : code +} + let capturedCanonicalHarnessCode = "" await runPhpunitCommand({ artifactRoot: mkdtempSync(join(tmpdir(), "wp-codebox-phpunit-artifacts-")), mounts: [], runPlaygroundCommand: async (_command, _server, input) => { - capturedCanonicalHarnessCode = input.code + capturedCanonicalHarnessCode = decodeSubmittedBootstrap(input.code) return { text: "ok", exitCode: 0 } }, runtimeSpec: phpunitRuntimeSpec, @@ -573,7 +578,7 @@ await runPhpunitCommand({ artifactRoot: mkdtempSync(join(tmpdir(), "wp-codebox-phpunit-artifacts-")), mounts: [], runPlaygroundCommand: async (_command, _server, input) => { - capturedExplicitCode = input.code + capturedExplicitCode = decodeSubmittedBootstrap(input.code) return { text: "ok", exitCode: 0 } }, runtimeSpec: phpunitRuntimeSpec, From 72e2acd3cfda5ff7a7a16b164e3d6d49aad502b3 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Tue, 21 Jul 2026 18:09:53 -0400 Subject: [PATCH 03/17] Preserve strict PHPUnit config handling after rebase --- .../src/phpunit-command-handlers.ts | 3 +++ tests/phpunit-project-autoload.test.ts | 11 +++-------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/runtime-playground/src/phpunit-command-handlers.ts b/packages/runtime-playground/src/phpunit-command-handlers.ts index 0a92b984..78013154 100644 --- a/packages/runtime-playground/src/phpunit-command-handlers.ts +++ b/packages/runtime-playground/src/phpunit-command-handlers.ts @@ -837,6 +837,9 @@ function pg_run_project_bootstrap_stage(array $cfg): void { $from_config = $bootstrap !== ''; } if ($bootstrap === '') { + if (!$phpunit_xml_is_default && !is_readable($phpunit_xml)) { + throw new RuntimeException('explicit PHPUnit config is not readable: ' . $phpunit_xml); + } pg_log('NOTICE:project bootstrap not declared; continuing without one'); pg_stage_ok('project_bootstrap'); return; diff --git a/tests/phpunit-project-autoload.test.ts b/tests/phpunit-project-autoload.test.ts index 63ab6fea..1f325e89 100644 --- a/tests/phpunit-project-autoload.test.ts +++ b/tests/phpunit-project-autoload.test.ts @@ -540,17 +540,12 @@ echo json_encode(array($legacy_project_autoload_file, $harness_autoload_file)); `) assert.deepEqual(JSON.parse(execFileSync("php", [canonicalHarnessProbe], { encoding: "utf8" })), ["", "/tmp/wp-codebox-inputs/0-wp-codebox-vendor-73845ca47d2f/autoload.php"], "a canonical staged harness path remains the harness in project mode") -function decodeSubmittedBootstrap(code: string): string { - const encodedBootstrap = code.match(/base64_decode\("([A-Za-z0-9+/=]+)"\)/)?.[1] - return encodedBootstrap ? Buffer.from(encodedBootstrap, "base64").toString("utf8") : code -} - let capturedCanonicalHarnessCode = "" await runPhpunitCommand({ artifactRoot: mkdtempSync(join(tmpdir(), "wp-codebox-phpunit-artifacts-")), mounts: [], runPlaygroundCommand: async (_command, _server, input) => { - capturedCanonicalHarnessCode = decodeSubmittedBootstrap(input.code) + capturedCanonicalHarnessCode = input.code return { text: "ok", exitCode: 0 } }, runtimeSpec: phpunitRuntimeSpec, @@ -571,14 +566,14 @@ const decodedCanonicalHarnessCode = decodedBootstrapWrapper(capturedCanonicalHar assert.ok(decodedCanonicalHarnessCode.includes('$autoload_file = "/tmp/wp-codebox-inputs/0-wp-codebox-vendor-73845ca47d2f/autoload.php";')) assert.ok(decodedCanonicalHarnessCode.includes('$autoload_file_role = "harness";')) assert.ok(decodedCanonicalHarnessCode.includes('putenv("TC_MYSQL_PORT=3306");'), "runtime service environment is passed to the PHP executed by wordpress.phpunit") -assert.ok(decodedCanonicalHarnessCode.indexOf('putenv("TC_MYSQL_PORT=3306");') < decodedCanonicalHarnessCode.indexOf("require_once '/wordpress/wp-load.php';"), "runtime environment is available to project bootstrap code") +assert.ok(!decodedCanonicalHarnessCode.includes("require_once '/wordpress/wp-load.php';"), "project bootstrap mode does not load the managed WordPress runtime first") let capturedExplicitCode = "" await runPhpunitCommand({ artifactRoot: mkdtempSync(join(tmpdir(), "wp-codebox-phpunit-artifacts-")), mounts: [], runPlaygroundCommand: async (_command, _server, input) => { - capturedExplicitCode = decodeSubmittedBootstrap(input.code) + capturedExplicitCode = input.code return { text: "ok", exitCode: 0 } }, runtimeSpec: phpunitRuntimeSpec, From 9025ac7109346e0eceb1dde644718dd89c485fce Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Tue, 21 Jul 2026 20:58:51 -0400 Subject: [PATCH 04/17] Reject incomplete PHPUnit bootstrap runs --- packages/runtime-playground/src/playground-runtime.ts | 4 +++- packages/runtime-playground/src/wordpress-command-runners.ts | 3 +++ tests/phpunit-runtime-failure-diagnostics.test.ts | 4 ++-- .../playground-phpunit-bootstrap-failure.integration.test.ts | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/runtime-playground/src/playground-runtime.ts b/packages/runtime-playground/src/playground-runtime.ts index 076092c8..ad38f06e 100644 --- a/packages/runtime-playground/src/playground-runtime.ts +++ b/packages/runtime-playground/src/playground-runtime.ts @@ -1898,9 +1898,11 @@ echo json_encode(array('command' => 'inspect-mounted-inputs', 'mounts' => $inspe function executionSpecWithEnvironment(spec: ExecutionSpec): ExecutionSpec { if (!spec.environment || Object.keys(spec.environment).length === 0) return spec + const args = spec.args ?? [] + const runtimeEnvironment = JSON.parse(argValue(args, "runtime-env-json") ?? "{}") as Record return { ...spec, - args: [...(spec.args ?? []).filter((argument) => !argument.startsWith("runtime-env-json=")), `runtime-env-json=${JSON.stringify(spec.environment)}`], + args: [...args.filter((argument) => !argument.startsWith("runtime-env-json=")), `runtime-env-json=${JSON.stringify({ ...runtimeEnvironment, ...spec.environment })}`], } } diff --git a/packages/runtime-playground/src/wordpress-command-runners.ts b/packages/runtime-playground/src/wordpress-command-runners.ts index a85fccf1..6036bbd1 100644 --- a/packages/runtime-playground/src/wordpress-command-runners.ts +++ b/packages/runtime-playground/src/wordpress-command-runners.ts @@ -966,6 +966,9 @@ export async function runPhpunitCommand({ } throw error } + if (structured) { + throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit terminated before completing bootstrap"), "wordpress.phpunit structured diagnostics", structured) + } return response.text } diff --git a/tests/phpunit-runtime-failure-diagnostics.test.ts b/tests/phpunit-runtime-failure-diagnostics.test.ts index 93ec3c23..4b3d6516 100644 --- a/tests/phpunit-runtime-failure-diagnostics.test.ts +++ b/tests/phpunit-runtime-failure-diagnostics.test.ts @@ -15,7 +15,7 @@ await assert.rejects( mounts: [], runPlaygroundCommand: async (_command, _server, input) => { submittedCode = input.code - return { exitCode: 1, errors: "", text: "" } + return { exitCode: 0, errors: "", text: "" } }, runtimeSpec: { environment: { kind: "wordpress", name: "test", version: "latest" }, policy: { commands: ["wordpress.phpunit"] } } as never, server: { @@ -29,7 +29,7 @@ await assert.rejects( spec: { command: "wordpress.phpunit", args: ["plugin-slug=demo-plugin"] }, }), (error: Error) => { - assert.match(error.message, /wordpress\.phpunit failed with exit code 1/) + assert.match(error.message, /wordpress\.phpunit terminated before completing bootstrap/) assert.match(error.message, /wordpress\.phpunit structured diagnostics/) assert.match(error.message, /Bootstrap failed with token: \[redacted\]/) assert.match(error.message, /\[diagnostic truncated\]/) diff --git a/tests/playground-phpunit-bootstrap-failure.integration.test.ts b/tests/playground-phpunit-bootstrap-failure.integration.test.ts index d1a3d140..a4c7031f 100644 --- a/tests/playground-phpunit-bootstrap-failure.integration.test.ts +++ b/tests/playground-phpunit-bootstrap-failure.integration.test.ts @@ -52,7 +52,7 @@ try { assert.match(commandLog, /PHPUnit bootstrap fixture token: \[redacted\]/) assert.doesNotMatch(commandLog, new RegExp(secret)) - await writeFile(fatalMuPlugin, ` Date: Tue, 21 Jul 2026 22:56:53 -0400 Subject: [PATCH 05/17] Require real scoped PHPUnit execution --- packages/cli/src/recipe-validation.ts | 10 +++-- packages/cli/src/runtime-services.ts | 4 +- packages/runtime-core/src/recipe-schema.ts | 7 +++- .../runtime-core/src/runtime-contracts.ts | 4 +- .../src/phpunit-command-handlers.ts | 42 ++++++++++++++++--- .../src/wordpress-command-runners.ts | 6 ++- tests/phpunit-project-autoload.test.ts | 28 +++++++++++-- ...hpunit-runtime-failure-diagnostics.test.ts | 13 ++++++ tests/runtime-services.test.ts | 9 +++- 9 files changed, 104 insertions(+), 19 deletions(-) diff --git a/packages/cli/src/recipe-validation.ts b/packages/cli/src/recipe-validation.ts index 74411ff4..ad10c3cd 100644 --- a/packages/cli/src/recipe-validation.ts +++ b/packages/cli/src/recipe-validation.ts @@ -743,11 +743,13 @@ function validateRecipeRuntimeServices(recipe: WorkspaceRecipe, addIssue: (code: if (ids.has(service.id)) addIssue("duplicate-runtime-service-id", `${path}.id`, `Runtime service ids must be unique: ${service.id}`) ids.add(service.id) if (service.kind !== "mysql") addIssue("unsupported-runtime-service-kind", `${path}.kind`, `Unsupported managed runtime service kind: ${service.kind}`) - for (const [output, name] of Object.entries(service.outputs)) { + for (const [output, names] of Object.entries(service.outputs)) { if (!/^(host|port|username|password|database)$/.test(output)) addIssue("unknown-runtime-service-output", `${path}.outputs.${output}`, `Unsupported ${service.kind} service output: ${output}`) - if (!/^[A-Z_][A-Z0-9_]*$/.test(name)) addIssue("invalid-runtime-service-env", `${path}.outputs.${output}`, "Runtime service environment variable names must match /^[A-Z_][A-Z0-9_]*$/.") - if (environment.has(name)) addIssue("duplicate-runtime-service-env", `${path}.outputs.${output}`, `Runtime service output environment variable is already declared: ${name}`) - environment.add(name) + for (const name of Array.isArray(names) ? names : [names]) { + if (!/^[A-Z_][A-Z0-9_]*$/.test(name)) addIssue("invalid-runtime-service-env", `${path}.outputs.${output}`, "Runtime service environment variable names must match /^[A-Z_][A-Z0-9_]*$/.") + if (environment.has(name)) addIssue("duplicate-runtime-service-env", `${path}.outputs.${output}`, `Runtime service output environment variable is already declared: ${name}`) + environment.add(name) + } } } } diff --git a/packages/cli/src/runtime-services.ts b/packages/cli/src/runtime-services.ts index 90ed2758..fc955371 100644 --- a/packages/cli/src/runtime-services.ts +++ b/packages/cli/src/runtime-services.ts @@ -61,7 +61,7 @@ const defaultDependencies: RuntimeServiceDependencies = { randomBytes, } -export function runtimeServicePlan(services: WorkspaceRecipeRuntimeService[]): Array<{ id: string; kind: string; provider: string; version: string; bind: "loopback"; port: "ephemeral"; persistentVolume: false; configuration?: WorkspaceRecipeRuntimeService["configuration"]; outputs: Record }> { +export function runtimeServicePlan(services: WorkspaceRecipeRuntimeService[]): Array<{ id: string; kind: string; provider: string; version: string; bind: "loopback"; port: "ephemeral"; persistentVolume: false; configuration?: WorkspaceRecipeRuntimeService["configuration"]; outputs: Record }> { return services.map((service) => { const provider = runtimeServiceProvider(service.kind) return { id: service.id, kind: service.kind, provider: provider.name, version: provider.version(service), bind: "loopback", port: "ephemeral", persistentVolume: false, ...(service.configuration ? { configuration: service.configuration } : {}), outputs: service.outputs } @@ -177,7 +177,7 @@ async function provisionMysqlDockerService(service: WorkspaceRecipeRuntimeServic evidence.readiness = "ready" evidence.lifecycle = "provisioned" const values: Record = { host: "127.0.0.1", port: String(port), username: "runtime", password, database: "runtime" } - return { env: Object.fromEntries(Object.entries(service.outputs).map(([output, name]) => [name, values[output] ?? ""])), evidence, async release() { await releaseService(container, evidence, dependencies) } } + return { env: Object.fromEntries(Object.entries(service.outputs).flatMap(([output, names]) => (Array.isArray(names) ? names : [names]).map((name) => [name, values[output] ?? ""]))), evidence, async release() { await releaseService(container, evidence, dependencies) } } } catch (error) { evidence.readiness = "failed" evidence.lifecycle = "failed" diff --git a/packages/runtime-core/src/recipe-schema.ts b/packages/runtime-core/src/recipe-schema.ts index 083b7d3a..ecce763a 100644 --- a/packages/runtime-core/src/recipe-schema.ts +++ b/packages/runtime-core/src/recipe-schema.ts @@ -930,7 +930,12 @@ export function createWorkspaceRecipeJsonSchema(options: WorkspaceRecipeJsonSche outputs: { type: "object", minProperties: 1, - additionalProperties: { type: "string", pattern: "^[A-Z_][A-Z0-9_]*$" }, + additionalProperties: { + anyOf: [ + { type: "string", pattern: "^[A-Z_][A-Z0-9_]*$" }, + { type: "array", minItems: 1, uniqueItems: true, items: { type: "string", pattern: "^[A-Z_][A-Z0-9_]*$" } }, + ], + }, }, }, }, diff --git a/packages/runtime-core/src/runtime-contracts.ts b/packages/runtime-core/src/runtime-contracts.ts index 3e018a64..6d40a0e3 100644 --- a/packages/runtime-core/src/runtime-contracts.ts +++ b/packages/runtime-core/src/runtime-contracts.ts @@ -142,8 +142,8 @@ export interface WorkspaceRecipeRuntimeService { rootAuthentication?: "generated-password" | "empty-password" foreignKeyTargetPolicy?: "unique-only" | "indexed" } - /** Explicit map from a provider output (for example `port`) to a runtime env name. */ - outputs: Record + /** Explicit map from a provider output (for example `port`) to one or more runtime env names. */ + outputs: Record } export interface WorkspaceRecipeRuntimeStack { diff --git a/packages/runtime-playground/src/phpunit-command-handlers.ts b/packages/runtime-playground/src/phpunit-command-handlers.ts index 78013154..30c2c05c 100644 --- a/packages/runtime-playground/src/phpunit-command-handlers.ts +++ b/packages/runtime-playground/src/phpunit-command-handlers.ts @@ -97,7 +97,7 @@ function phpunitConfigDiscoveryPhp(options: PhpunitConfigDiscoveryPhpOptions): s const suffixAssignment = options.replaceDefaultMatchers ? "$suffixes = $config_suffixes;" : "$suffixes = array_merge($suffixes, $config_suffixes);" const prefixAssignment = options.replaceDefaultMatchers ? "$prefixes = $config_prefixes;" : "$prefixes = array_merge($prefixes, $config_prefixes);" - return `function ${options.functionName}($xml_path, $test_dir_default) { + return `function ${options.functionName}($xml_path, $test_dir_default, $requested_suites = array()) { $directories = array($test_dir_default); $suffixes = array('Test.php'); $prefixes = array('test-'); @@ -118,11 +118,23 @@ function phpunitConfigDiscoveryPhp(options: PhpunitConfigDiscoveryPhpOptions): s $first = $errors ? trim($errors[0]->message) : 'unknown'; throw new RuntimeException('PHPUnit config could not be parsed at ' . $xml_path . ': ' . $first); } + $suite_nodes = $xml->xpath('//testsuite') ?: array(); + if (!empty($requested_suites)) { + $requested_lookup = array_fill_keys(array_map('strval', $requested_suites), true); + $suite_nodes = array_values(array_filter($suite_nodes, static function($suite) use ($requested_lookup) { + return isset($requested_lookup[(string) ($suite['name'] ?? '')]); + })); + if (empty($suite_nodes)) { + throw new RuntimeException('Requested PHPUnit testsuite was not found in config: ' . implode(',', array_keys($requested_lookup))); + } + } + $directories = array(); $base = ${options.basePathExpression}; $config_dirs = array(); $config_suffixes = array(); $config_prefixes = array(); - foreach ($xml->xpath('//testsuite/directory') ?: array() as $dir) { + foreach ($suite_nodes as $suite) { + foreach ($suite->xpath('./directory') ?: array() as $dir) { $raw = trim((string) $dir);${directoryRestriction} $config_dirs[] = $raw[0] === '/' ? rtrim($raw, '/') : rtrim($base . '/' . $raw, '/'); foreach (explode(',', (string) ($dir['suffix'] ?? '')) as $suffix) { @@ -137,18 +149,23 @@ function phpunitConfigDiscoveryPhp(options: PhpunitConfigDiscoveryPhpOptions): s $config_prefixes[] = $prefix; } } + } } - foreach ($xml->xpath('//testsuite/exclude') ?: array() as $exclude) { + foreach ($suite_nodes as $suite) { + foreach ($suite->xpath('./exclude') ?: array() as $exclude) { $raw = trim((string) $exclude); if ($raw !== '') { $excludes[] = $raw[0] === '/' ? rtrim($raw, '/') : rtrim($base . '/' . $raw, '/'); } + } } - foreach ($xml->xpath('//testsuite/file') ?: array() as $file) { + foreach ($suite_nodes as $suite) { + foreach ($suite->xpath('./file') ?: array() as $file) { $raw = trim((string) $file); if ($raw !== '') { $files[] = $raw[0] === '/' ? $raw : $base . '/' . $raw; } + } } if (!empty($config_dirs)) { $directories = $config_dirs; @@ -1227,13 +1244,28 @@ ${phpunitConfigDiscoveryPhp({ ${phpunitDiscoveryPhp("wp_codebox_phpunit_discover", "pg_log")} +function wp_codebox_requested_phpunit_suites($args): array { + $suites = array(); + if (!is_array($args)) { + return $suites; + } + foreach ($args as $index => $arg) { + if ($arg === '--testsuite' && isset($args[$index + 1])) { + $suites = array_merge($suites, explode(',', (string) $args[$index + 1])); + } elseif (is_string($arg) && strpos($arg, '--testsuite=') === 0) { + $suites = array_merge($suites, explode(',', substr($arg, strlen('--testsuite=')))); + } + } + return array_values(array_filter(array_map('trim', $suites), static function($suite) { return $suite !== ''; })); +} + pg_stage_begin('discover_tests'); try { $test_dir = $test_root; if (!is_dir($test_dir)) { throw new RuntimeException('configured PHPUnit test root is not a readable directory: ' . $test_dir); } - list($directories, $suffixes, $prefixes, $excludes, $configured_files) = wp_codebox_phpunit_parse_config(${JSON.stringify(options.phpunitXml)}, $test_dir); + list($directories, $suffixes, $prefixes, $excludes, $configured_files) = wp_codebox_phpunit_parse_config(${JSON.stringify(options.phpunitXml)}, $test_dir, wp_codebox_requested_phpunit_suites($phpunit_args_raw)); $test_files = wp_codebox_phpunit_discover($directories, $suffixes, $prefixes, $excludes, $configured_files); $test_files = pg_filter_changed_test_files($test_files, $changed_test_files_raw, $test_dir); if ($selected_test_file !== '') { diff --git a/packages/runtime-playground/src/wordpress-command-runners.ts b/packages/runtime-playground/src/wordpress-command-runners.ts index 6036bbd1..ee9c80f7 100644 --- a/packages/runtime-playground/src/wordpress-command-runners.ts +++ b/packages/runtime-playground/src/wordpress-command-runners.ts @@ -913,6 +913,7 @@ export async function runPhpunitCommand({ const bootstrapMode = argValue(args, "bootstrap-mode")?.trim() || "managed" const autoloadFile = argValue(args, "autoload-file")?.trim() || (bootstrapMode === "project" ? "" : "/wp-codebox-vendor/autoload.php") const autoloadFileRole = argValue(args, "autoload-file-role")?.trim() === "harness" ? "harness" : undefined + const phpunitArgs = jsonArrayArg(args, "phpunit-args-json").filter((value): value is string => typeof value === "string") const processIdentity = boundedProcessIdentity(spec.processIdentity) const resultFile = processIdentity ? `/tmp/wp-codebox-phpunit-result-${processIdentity}.txt` : PLUGIN_PHPUNIT_RESULT_FILE const diagnosticHostFile = `/wordpress/wp-content/plugins/${pluginSlug}/.pg-test-result${processIdentity ? `-${processIdentity}` : ""}.txt` @@ -928,7 +929,7 @@ export async function runPhpunitCommand({ phpunitXmlIsDefault: phpunitXmlArg === undefined || booleanArg(args, "phpunit-xml-default"), selectedTestFile: argValue(args, "test-file")?.trim() || "", changedTestFiles: changedTestFilesArg(args), - phpunitArgs: jsonArrayArg(args, "phpunit-args-json").filter((value): value is string => typeof value === "string"), + phpunitArgs, env: jsonObjectArg(args, "env-json"), wpConfigDefines: jsonObjectArg(args, "wp-config-defines-json"), dependencyMounts: commaListArg(args, "dependency-mounts"), @@ -969,6 +970,9 @@ export async function runPhpunitCommand({ if (structured) { throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit terminated before completing bootstrap"), "wordpress.phpunit structured diagnostics", structured) } + if (!phpunitArgs.includes("--list-tests") && !/\bOK(?:, but there were issues!)? \([1-9]\d* tests?,/m.test(response.text)) { + throw new Error("wordpress.phpunit exited successfully without a non-zero PHPUnit test summary") + } return response.text } diff --git a/tests/phpunit-project-autoload.test.ts b/tests/phpunit-project-autoload.test.ts index 1f325e89..efa8e27c 100644 --- a/tests/phpunit-project-autoload.test.ts +++ b/tests/phpunit-project-autoload.test.ts @@ -136,6 +136,27 @@ echo "ok\n"; assert.equal(execFileSync("php", [scriptPath], { encoding: "utf8" }), "ok\n") } +function assertNamedTestsuiteScopesDiscovery(source: string): void { + const tempDir = mkdtempSync(join(tmpdir(), "wp-codebox-named-testsuite-")) + const selected = join(tempDir, "SelectedTest.php") + const unrelated = join(tempDir, "UnrelatedTest.php") + const config = join(tempDir, "phpunit.xml") + const scriptPath = join(tempDir, "assert-named-testsuite.php") + writeFileSync(selected, "SelectedTest.phpUnrelatedTest.php`) + writeFileSync(scriptPath, `xpath('//testsuite/file') ?: array() as $file)")) +assert.ok(projectModeCode.includes("foreach ($suite->xpath('./file') ?: array() as $file)")) assert.ok(projectModeCode.includes("list($directories, $suffixes, $prefixes, $excludes, $configured_files) = wp_codebox_phpunit_parse_config")) assert.ok(projectModeCode.includes("$test_files = wp_codebox_phpunit_discover($directories, $suffixes, $prefixes, $excludes, $configured_files);")) assert.ok(projectModeCode.includes("' files=' . count($configured_files)")) @@ -492,6 +513,7 @@ assert.equal(projectModeCode.match(/return array\(\$directories, \$suffixes, \$p assert.equal(projectModeCode.match(/return \$return_values\(\);/g)?.length, 1) assert.match(projectModeCode, /configured PHPUnit test root is not a readable directory/) assertPhpunitConfigurationAndDiscoveryFailures(projectModeCode, "wp_codebox_phpunit_parse_config", "wp_codebox_phpunit_discover", "pg_log", false) +assertNamedTestsuiteScopesDiscovery(projectModeCode) assertChangedScopeNoOp(projectModeCode, "pg_filter_changed_test_files", "pg_component_relative_path") assertSelectedTestFileResolution(projectModeCode) @@ -546,7 +568,7 @@ await runPhpunitCommand({ mounts: [], runPlaygroundCommand: async (_command, _server, input) => { capturedCanonicalHarnessCode = input.code - return { text: "ok", exitCode: 0 } + return { text: "OK (1 test, 1 assertion)", exitCode: 0 } }, runtimeSpec: phpunitRuntimeSpec, server: { playground: {} } as never, @@ -574,7 +596,7 @@ await runPhpunitCommand({ mounts: [], runPlaygroundCommand: async (_command, _server, input) => { capturedExplicitCode = input.code - return { text: "ok", exitCode: 0 } + return { text: "OK (1 test, 1 assertion)", exitCode: 0 } }, runtimeSpec: phpunitRuntimeSpec, server: { playground: {} } as never, diff --git a/tests/phpunit-runtime-failure-diagnostics.test.ts b/tests/phpunit-runtime-failure-diagnostics.test.ts index 4b3d6516..f6fd7373 100644 --- a/tests/phpunit-runtime-failure-diagnostics.test.ts +++ b/tests/phpunit-runtime-failure-diagnostics.test.ts @@ -49,4 +49,17 @@ const preBootstrapRecorder = decodedBootstrap.indexOf("STAGE_FATAL:bootstrap:") const wordpressBootstrap = decodedBootstrap.indexOf("require_once '/wordpress/wp-load.php';") assert.ok(preBootstrapRecorder >= 0 && preBootstrapRecorder < wordpressBootstrap, "fatal diagnostics must be recorded before the WordPress bootstrap boundary") +const emptySuccessArtifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-phpunit-empty-success-")) +await assert.rejects( + () => runPhpunitCommand({ + artifactRoot: emptySuccessArtifactRoot, + mounts: [], + runPlaygroundCommand: async () => ({ exitCode: 0, errors: "", text: "WPCOM Codebox PHPUnit shutdown: mysql_port=unset" }), + runtimeSpec: { environment: { kind: "wordpress", name: "test", version: "latest" }, policy: { commands: ["wordpress.phpunit"] } } as never, + server: { playground: { readFileAsText: async () => { throw new Error("missing") } } } as never, + spec: { command: "wordpress.phpunit", args: ["plugin-slug=demo-plugin"] }, + }), + /exited successfully without a non-zero PHPUnit test summary/, +) + console.log("phpunit runtime failure diagnostics ok") diff --git a/tests/runtime-services.test.ts b/tests/runtime-services.test.ts index 27552adc..23af5540 100644 --- a/tests/runtime-services.test.ts +++ b/tests/runtime-services.test.ts @@ -18,6 +18,9 @@ assert.throws(() => parseLoopbackPort("0.0.0.0:3306"), /loopback/) const valid = validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [service] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }) assert.equal(valid.valid, true) +const aliasedPortService = { ...service, outputs: { ...service.outputs, port: ["DB_PORT", "TC_MYSQL_PORT"] } } +assert.equal(validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [aliasedPortService] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }).valid, true) +assert.equal(validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [{ ...service, outputs: { port: [] } }] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }).valid, false) const unsafe = validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [{ ...service, outputs: { port: "bad-name" } }] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }) assert.equal(unsafe.valid, false) const emptyRootService = { ...service, configuration: { rootAuthentication: "empty-password" as const } } @@ -103,6 +106,10 @@ const dependencies: RuntimeServiceDependencies = { const provisioned = await provisionRuntimeServices([service], { dependencies }) assert.equal(provisioned.env.DB_PORT, "41001") assert.equal(provisioned.env.DB_PASSWORD, Buffer.alloc(24, 7).toString("base64url")) +const aliasedPort = await provisionRuntimeServices([aliasedPortService], { dependencies }) +assert.equal(aliasedPort.env.DB_PORT, "41001") +assert.equal(aliasedPort.env.TC_MYSQL_PORT, "41001") +await aliasedPort.release() const runCall = calls.find((call) => call.args[0] === "run") assert.ok(runCall?.args.includes("MYSQL_PASSWORD")) assert.ok(runCall?.args.includes("127.0.0.1::3306"), "Docker publishes MySQL on a loopback ephemeral port") @@ -118,7 +125,7 @@ assert.equal(runCall?.env?.DOCKER_HOST, process.env.DOCKER_HOST, "Docker provide assert.equal(calls[0]?.args[0], "image", "the provider checks the image before starting the service") await provisioned.release() await provisioned.release() -assert.equal(calls.filter((call) => call.args[0] === "rm").length, 1, "release is idempotent") +assert.equal(calls.filter((call) => call.args[0] === "rm").length, 2, "each service is released exactly once") const emptyRootCalls: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = [] const emptyRootDependencies: RuntimeServiceDependencies = { From e7b41708937199fdbda73728de2cea06ec5ec921 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Wed, 22 Jul 2026 18:33:07 -0400 Subject: [PATCH 06/17] Preserve PHPUnit zero-summary diagnostics --- .../src/wordpress-command-runners.ts | 6 +++++- ...hpunit-runtime-failure-diagnostics.test.ts | 20 +++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/packages/runtime-playground/src/wordpress-command-runners.ts b/packages/runtime-playground/src/wordpress-command-runners.ts index ee9c80f7..0b87aed7 100644 --- a/packages/runtime-playground/src/wordpress-command-runners.ts +++ b/packages/runtime-playground/src/wordpress-command-runners.ts @@ -971,7 +971,11 @@ export async function runPhpunitCommand({ throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit terminated before completing bootstrap"), "wordpress.phpunit structured diagnostics", structured) } if (!phpunitArgs.includes("--list-tests") && !/\bOK(?:, but there were issues!)? \([1-9]\d* tests?,/m.test(response.text)) { - throw new Error("wordpress.phpunit exited successfully without a non-zero PHPUnit test summary") + const diagnostics = [ + response.errors?.trim() ? `--- stderr ---\n${response.errors.trim()}` : "", + response.text.trim() ? `--- stdout ---\n${response.text.trim()}` : "", + ].filter(Boolean).join("\n") + throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit exited successfully without a non-zero PHPUnit test summary"), "wordpress.phpunit successful response diagnostics", diagnostics) } return response.text diff --git a/tests/phpunit-runtime-failure-diagnostics.test.ts b/tests/phpunit-runtime-failure-diagnostics.test.ts index f6fd7373..32b68ee0 100644 --- a/tests/phpunit-runtime-failure-diagnostics.test.ts +++ b/tests/phpunit-runtime-failure-diagnostics.test.ts @@ -50,16 +50,32 @@ const wordpressBootstrap = decodedBootstrap.indexOf("require_once '/wordpress/wp assert.ok(preBootstrapRecorder >= 0 && preBootstrapRecorder < wordpressBootstrap, "fatal diagnostics must be recorded before the WordPress bootstrap boundary") const emptySuccessArtifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-phpunit-empty-success-")) +const successfulResponseSecret = "ghp_abcdefghijklmnopqrstuvwxyz1234567890" await assert.rejects( () => runPhpunitCommand({ artifactRoot: emptySuccessArtifactRoot, mounts: [], - runPlaygroundCommand: async () => ({ exitCode: 0, errors: "", text: "WPCOM Codebox PHPUnit shutdown: mysql_port=unset" }), + runPlaygroundCommand: async () => ({ + exitCode: 0, + errors: `PHPUnit stderr token=${successfulResponseSecret}`, + text: `WPCOM Codebox PHPUnit shutdown: mysql_port=unset\n${"x".repeat(25_000)}`, + }), runtimeSpec: { environment: { kind: "wordpress", name: "test", version: "latest" }, policy: { commands: ["wordpress.phpunit"] } } as never, server: { playground: { readFileAsText: async () => { throw new Error("missing") } } } as never, spec: { command: "wordpress.phpunit", args: ["plugin-slug=demo-plugin"] }, }), - /exited successfully without a non-zero PHPUnit test summary/, + (error: Error) => { + assert.match(error.message, /exited successfully without a non-zero PHPUnit test summary/) + assert.match(error.message, /wordpress\.phpunit successful response diagnostics/) + assert.match(error.message, /--- stderr ---/) + assert.match(error.message, /--- stdout ---/) + assert.match(error.message, /WPCOM Codebox \[redacted\] shutdown: mysql_port=unset/) + assert.match(error.message, /PHPUnit stderr token=\[redacted\]/) + assert.match(error.message, /\[diagnostic truncated\]/) + assert.doesNotMatch(error.message, new RegExp(successfulResponseSecret)) + assert.ok(error.message.length < 21_000, "successful response diagnostics must remain bounded") + return true + }, ) console.log("phpunit runtime failure diagnostics ok") From ddae9e8513ec81c3227b21694ef816bda4691967 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Wed, 22 Jul 2026 18:44:34 -0400 Subject: [PATCH 07/17] Bound PHPUnit response streams independently --- .../src/wordpress-command-runners.ts | 13 +++++++++---- tests/phpunit-runtime-failure-diagnostics.test.ts | 6 +++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/runtime-playground/src/wordpress-command-runners.ts b/packages/runtime-playground/src/wordpress-command-runners.ts index 0b87aed7..b26952ad 100644 --- a/packages/runtime-playground/src/wordpress-command-runners.ts +++ b/packages/runtime-playground/src/wordpress-command-runners.ts @@ -971,16 +971,21 @@ export async function runPhpunitCommand({ throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit terminated before completing bootstrap"), "wordpress.phpunit structured diagnostics", structured) } if (!phpunitArgs.includes("--list-tests") && !/\bOK(?:, but there were issues!)? \([1-9]\d* tests?,/m.test(response.text)) { - const diagnostics = [ - response.errors?.trim() ? `--- stderr ---\n${response.errors.trim()}` : "", - response.text.trim() ? `--- stdout ---\n${response.text.trim()}` : "", - ].filter(Boolean).join("\n") + const diagnostics = successfulPhpunitResponseDiagnostics(response) throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit exited successfully without a non-zero PHPUnit test summary"), "wordpress.phpunit successful response diagnostics", diagnostics) } return response.text } +function successfulPhpunitResponseDiagnostics(response: PlaygroundRunResponse): string { + const boundedStream = (value: string): string => value.length > 9_000 ? `${value.slice(0, 9_000)}\n[stream truncated]` : value + return [ + response.errors?.trim() ? `--- stderr ---\n${boundedStream(response.errors.trim())}` : "", + response.text.trim() ? `--- stdout ---\n${boundedStream(response.text.trim())}` : "", + ].filter(Boolean).join("\n") +} + function boundedProcessIdentity(value: string | undefined): string { if (!value) return "" if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value)) throw new Error(`Invalid PHPUnit process identity: ${value}`) diff --git a/tests/phpunit-runtime-failure-diagnostics.test.ts b/tests/phpunit-runtime-failure-diagnostics.test.ts index 32b68ee0..ff21cb22 100644 --- a/tests/phpunit-runtime-failure-diagnostics.test.ts +++ b/tests/phpunit-runtime-failure-diagnostics.test.ts @@ -57,7 +57,7 @@ await assert.rejects( mounts: [], runPlaygroundCommand: async () => ({ exitCode: 0, - errors: `PHPUnit stderr token=${successfulResponseSecret}`, + errors: `PHPUnit stderr token=${successfulResponseSecret}\n${"e".repeat(25_000)}`, text: `WPCOM Codebox PHPUnit shutdown: mysql_port=unset\n${"x".repeat(25_000)}`, }), runtimeSpec: { environment: { kind: "wordpress", name: "test", version: "latest" }, policy: { commands: ["wordpress.phpunit"] } } as never, @@ -71,9 +71,9 @@ await assert.rejects( assert.match(error.message, /--- stdout ---/) assert.match(error.message, /WPCOM Codebox \[redacted\] shutdown: mysql_port=unset/) assert.match(error.message, /PHPUnit stderr token=\[redacted\]/) - assert.match(error.message, /\[diagnostic truncated\]/) + assert.match(error.message, /\[stream truncated\]/) assert.doesNotMatch(error.message, new RegExp(successfulResponseSecret)) - assert.ok(error.message.length < 21_000, "successful response diagnostics must remain bounded") + assert.ok(error.message.length < 19_000, "successful response diagnostics must remain bounded") return true }, ) From 66ae688bebaac1f04d74a6d54bd7b1fd0c69c780 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Wed, 22 Jul 2026 23:08:41 -0400 Subject: [PATCH 08/17] Support TeamCity PHPUnit runtime contracts --- packages/cli/src/commands/recipe-build.ts | 4 +++- packages/runtime-core/src/recipe-builders.ts | 4 +++- .../runtime-playground/src/wordpress-command-runners.ts | 6 +++++- tests/phpunit-runtime-failure-diagnostics.test.ts | 5 ++++- tests/runtime-services.test.ts | 1 + 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/commands/recipe-build.ts b/packages/cli/src/commands/recipe-build.ts index 40dee3eb..9e091aba 100644 --- a/packages/cli/src/commands/recipe-build.ts +++ b/packages/cli/src/commands/recipe-build.ts @@ -1,5 +1,5 @@ import { readFile, writeFile } from "node:fs/promises" -import { buildGenericAbilityRuntimeRunRecipe, buildRuntimePackageRunRecipe, buildWordPressBenchRecipe, buildWordPressPhpunitRecipe, compileRecipeTemplate, type GenericAbilityRuntimeRunOptions, type RecipeTemplateInput, type RuntimePackageRunRecipeOptions, type RuntimeWordPressInstallMode, type WorkspaceRecipe, type WorkspaceRecipeExtraPlugin, type WorkspaceRecipeMount, type WorkspaceRecipePHPWasmExtensionManifest, type WorkspaceRecipeRuntimeBackendPackage, type WorkspaceRecipeRuntimeService, type WorkspaceRecipeStep } from "@automattic/wp-codebox-core" +import { buildGenericAbilityRuntimeRunRecipe, buildRuntimePackageRunRecipe, buildWordPressBenchRecipe, buildWordPressPhpunitRecipe, compileRecipeTemplate, type GenericAbilityRuntimeRunOptions, type RecipeTemplateInput, type RuntimePackageRunRecipeOptions, type RuntimeWordPressInstallMode, type WorkspaceRecipe, type WorkspaceRecipeExtraPlugin, type WorkspaceRecipeMount, type WorkspaceRecipePHPWasmExtensionManifest, type WorkspaceRecipePluginRuntime, type WorkspaceRecipeRuntimeBackendPackage, type WorkspaceRecipeRuntimeService, type WorkspaceRecipeStep } from "@automattic/wp-codebox-core" interface RecipeBuildOptions { recipeType: "phpunit" | "bench" | "template" | "generic-ability-runtime-run" | "runtime-package-run" @@ -16,6 +16,7 @@ interface WordPressPhpunitBuilderOptions { backendPackage?: WorkspaceRecipeRuntimeBackendPackage mounts?: WorkspaceRecipeMount[] services?: WorkspaceRecipeRuntimeService[] + pluginRuntime?: WorkspaceRecipePluginRuntime extra_plugins?: WorkspaceRecipeExtraPlugin[] pluginSource?: string pluginSlug: string @@ -86,6 +87,7 @@ function buildRecipe(recipeType: RecipeBuildOptions["recipeType"], options: Word backendPackage: phpunitOptions.backendPackage, mounts: Array.isArray(phpunitOptions.mounts) ? phpunitOptions.mounts : [], services: Array.isArray(phpunitOptions.services) ? phpunitOptions.services : [], + pluginRuntime: phpunitOptions.pluginRuntime ? plainObject(phpunitOptions.pluginRuntime) as WorkspaceRecipePluginRuntime : undefined, extra_plugins: Array.isArray(phpunitOptions.extra_plugins) ? phpunitOptions.extra_plugins : [], pluginSource: stringOrUndefined(phpunitOptions.pluginSource), pluginSlug: requiredString(phpunitOptions.pluginSlug, "pluginSlug"), diff --git a/packages/runtime-core/src/recipe-builders.ts b/packages/runtime-core/src/recipe-builders.ts index da5b18e1..fe6b890c 100644 --- a/packages/runtime-core/src/recipe-builders.ts +++ b/packages/runtime-core/src/recipe-builders.ts @@ -1,4 +1,4 @@ -import type { RuntimePreviewSpec, RuntimeWordPressInstallMode, WorkspaceRecipe, WorkspaceRecipeExtraPlugin, WorkspaceRecipeMount, WorkspaceRecipePHPWasmExtensionManifest, WorkspaceRecipeRuntimeBackendPackage, WorkspaceRecipeRuntimeService, WorkspaceRecipeStep } from "./runtime-contracts.js" +import type { RuntimePreviewSpec, RuntimeWordPressInstallMode, WorkspaceRecipe, WorkspaceRecipeExtraPlugin, WorkspaceRecipeMount, WorkspaceRecipePHPWasmExtensionManifest, WorkspaceRecipePluginRuntime, WorkspaceRecipeRuntimeBackendPackage, WorkspaceRecipeRuntimeService, WorkspaceRecipeStep } from "./runtime-contracts.js" import { commandArg, commandJsonArg, commandStringListArg } from "./command-codecs.js" export { buildRuntimePackageRunRecipe, CODEBOX_RUN_RUNTIME_PACKAGE_ABILITY, RUNTIME_PACKAGE_ARTIFACT_DECLARATION_SCHEMA, RUNTIME_PACKAGE_EXECUTION_INPUT_SCHEMA, RUNTIME_PACKAGE_EXECUTION_RESULT_SCHEMA, RUNTIME_PACKAGE_OUTPUT_PROJECTION_SCHEMA, runtimePackageExecutionInput, type RuntimePackageArtifactDeclaration, type RuntimePackageExecutionInput, type RuntimePackageOutputProjection, type RuntimePackageRunRecipeOptions } from "./runtime-package-execution.js" export { RUNTIME_PACKAGE_DIAGNOSTIC_SCHEMA, RUNTIME_PACKAGE_RESULT_SCHEMA, RUNTIME_PACKAGE_TASK_SCHEMA, normalizeRuntimePackageResult, normalizeRuntimePackageTask, validateRuntimePackageTask, type RuntimePackageDiagnostic, type RuntimePackageResult, type RuntimePackageTask } from "./runtime-package-contracts.js" @@ -21,6 +21,7 @@ export interface WordPressPhpunitRecipeOptions { preview?: RuntimePreviewSpec mounts?: WorkspaceRecipeMount[] services?: WorkspaceRecipeRuntimeService[] + pluginRuntime?: WorkspaceRecipePluginRuntime extra_plugins?: WorkspaceRecipeExtraPlugin[] pluginSource?: string pluginSlug: string @@ -88,6 +89,7 @@ export function buildWordPressPhpunitRecipe(options: WordPressPhpunitRecipeOptio inputs: { extra_plugins: normalizeExtraPlugins(options.extra_plugins), ...(options.services && options.services.length > 0 ? { services: options.services } : {}), + ...(options.pluginRuntime ? { pluginRuntime: options.pluginRuntime } : {}), mounts: normalizeRecipeMounts([ ...(options.pluginSource ? [{ source: options.pluginSource, target: pluginTarget } satisfies WorkspaceRecipeMount] : []), ...(options.mounts ?? []), diff --git a/packages/runtime-playground/src/wordpress-command-runners.ts b/packages/runtime-playground/src/wordpress-command-runners.ts index b26952ad..4cdbba94 100644 --- a/packages/runtime-playground/src/wordpress-command-runners.ts +++ b/packages/runtime-playground/src/wordpress-command-runners.ts @@ -970,7 +970,7 @@ export async function runPhpunitCommand({ if (structured) { throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit terminated before completing bootstrap"), "wordpress.phpunit structured diagnostics", structured) } - if (!phpunitArgs.includes("--list-tests") && !/\bOK(?:, but there were issues!)? \([1-9]\d* tests?,/m.test(response.text)) { + if (!phpunitArgs.includes("--list-tests") && !hasSuccessfulPhpunitSummary(response.text)) { const diagnostics = successfulPhpunitResponseDiagnostics(response) throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit exited successfully without a non-zero PHPUnit test summary"), "wordpress.phpunit successful response diagnostics", diagnostics) } @@ -978,6 +978,10 @@ export async function runPhpunitCommand({ return response.text } +export function hasSuccessfulPhpunitSummary(output: string): boolean { + return /\bOK(?:, but (?:there were issues!|incomplete, skipped, or risky tests!))?(?: \([1-9]\d* tests?,|\s+Tests:\s*[1-9]\d*,\s*Assertions:)/m.test(output) +} + function successfulPhpunitResponseDiagnostics(response: PlaygroundRunResponse): string { const boundedStream = (value: string): string => value.length > 9_000 ? `${value.slice(0, 9_000)}\n[stream truncated]` : value return [ diff --git a/tests/phpunit-runtime-failure-diagnostics.test.ts b/tests/phpunit-runtime-failure-diagnostics.test.ts index ff21cb22..dd59b3c1 100644 --- a/tests/phpunit-runtime-failure-diagnostics.test.ts +++ b/tests/phpunit-runtime-failure-diagnostics.test.ts @@ -3,7 +3,7 @@ import { mkdtemp, readFile } from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" import { PLUGIN_PHPUNIT_RESULT_FILE } from "../packages/runtime-playground/src/phpunit-command-handlers.js" -import { runPhpunitCommand } from "../packages/runtime-playground/src/wordpress-command-runners.js" +import { hasSuccessfulPhpunitSummary, runPhpunitCommand } from "../packages/runtime-playground/src/wordpress-command-runners.js" const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-phpunit-runtime-diagnostics-")) const secret = "sk-abcdefghijklmnopqrstuvwxyz" @@ -51,6 +51,9 @@ assert.ok(preBootstrapRecorder >= 0 && preBootstrapRecorder < wordpressBootstrap const emptySuccessArtifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-phpunit-empty-success-")) const successfulResponseSecret = "ghp_abcdefghijklmnopqrstuvwxyz1234567890" +assert.equal(hasSuccessfulPhpunitSummary("OK (1 test, 1 assertion)"), true) +assert.equal(hasSuccessfulPhpunitSummary("OK, but incomplete, skipped, or risky tests!\nTests: 21, Assertions: 27398, Skipped: 1."), true) +assert.equal(hasSuccessfulPhpunitSummary("Tests: 21, Assertions: 27398, Failures: 1."), false) await assert.rejects( () => runPhpunitCommand({ artifactRoot: emptySuccessArtifactRoot, diff --git a/tests/runtime-services.test.ts b/tests/runtime-services.test.ts index 23af5540..1ee14f0e 100644 --- a/tests/runtime-services.test.ts +++ b/tests/runtime-services.test.ts @@ -33,6 +33,7 @@ assert.equal(runtimeServicePlan([mariaDbService])[0]?.version, "mariadb:11.4") assert.equal(validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [{ ...service, configuration: { foreignKeyTargetPolicy: "anything" } }] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }).valid, false) assert.deepEqual(buildWordPressPhpunitRecipe({ pluginSlug: "example", services: [emptyRootService] }).inputs?.services, [emptyRootService]) assert.equal(buildWordPressPhpunitRecipe({ pluginSlug: "example", wordpressInstallMode: "do-not-attempt-installing" }).runtime?.wordpressInstallMode, "do-not-attempt-installing") +assert.deepEqual(buildWordPressPhpunitRecipe({ pluginSlug: "example", pluginRuntime: { php: { memoryLimit: "2G" } } }).inputs?.pluginRuntime, { php: { memoryLimit: "2G" } }) const builderDirectory = await mkdtemp(join(tmpdir(), "wp-codebox-phpunit-builder-")) try { const optionsPath = join(builderDirectory, "options.json") From 7d85c9780b3299867ad54e2bb19649e8d81af220 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Thu, 23 Jul 2026 03:10:43 -0400 Subject: [PATCH 09/17] Honor runtime memory during Playground startup --- .../src/playground-cli-runner.ts | 19 ++++++++++++++++++- ...layground-cli-runner-bootstrap-ini.test.ts | 7 ++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/packages/runtime-playground/src/playground-cli-runner.ts b/packages/runtime-playground/src/playground-cli-runner.ts index cbd8bd8f..dc90d73f 100644 --- a/packages/runtime-playground/src/playground-cli-runner.ts +++ b/packages/runtime-playground/src/playground-cli-runner.ts @@ -332,16 +332,33 @@ function runtimeBootstrapPhpIniEntries(spec: RuntimeCreateSpec): Record | undefined { - return pluginRuntimePhpEntries(spec, "bootstrapIniEntries") + const memoryLimit = pluginRuntimePhpMemoryLimit(spec) + return { + ...(memoryLimit ? { memory_limit: memoryLimit } : {}), + ...(pluginRuntimePhpEntries(spec, "bootstrapIniEntries") ?? {}), + } } function pluginRuntimePhpIniEntries(spec: RuntimeCreateSpec): Record | undefined { + const memoryLimit = pluginRuntimePhpMemoryLimit(spec) return { ...DEFAULT_RUNTIME_PHP_INI_ENTRIES, + ...(memoryLimit ? { memory_limit: memoryLimit } : {}), ...(pluginRuntimePhpEntries(spec, "iniEntries") ?? {}), } } +function pluginRuntimePhpMemoryLimit(spec: RuntimeCreateSpec): string | undefined { + const pluginRuntime = spec.metadata?.recipe && typeof spec.metadata.recipe === "object" && !Array.isArray(spec.metadata.recipe) + ? (spec.metadata.recipe as { inputs?: { pluginRuntime?: unknown } }).inputs?.pluginRuntime + : undefined + const php = pluginRuntime && typeof pluginRuntime === "object" && !Array.isArray(pluginRuntime) + ? (pluginRuntime as { php?: Record }).php + : undefined + const memoryLimit = php?.memoryLimit + return typeof memoryLimit === "string" && /^[0-9]+[KMG]?$/.test(memoryLimit) ? memoryLimit : undefined +} + function pluginRuntimePhpEntries(spec: RuntimeCreateSpec, key: "iniEntries" | "bootstrapIniEntries"): Record | undefined { const pluginRuntime = spec.metadata?.recipe && typeof spec.metadata.recipe === "object" && !Array.isArray(spec.metadata.recipe) ? (spec.metadata.recipe as { inputs?: { pluginRuntime?: unknown } }).inputs?.pluginRuntime diff --git a/tests/playground-cli-runner-bootstrap-ini.test.ts b/tests/playground-cli-runner-bootstrap-ini.test.ts index 9e94d29e..b2f2a4cb 100644 --- a/tests/playground-cli-runner-bootstrap-ini.test.ts +++ b/tests/playground-cli-runner-bootstrap-ini.test.ts @@ -49,7 +49,8 @@ try { inputs: { pluginRuntime: { php: { - iniEntries: { memory_limit: "512M" }, + memoryLimit: "2G", + iniEntries: { max_input_vars: 2048 }, bootstrapIniEntries: { "opcache.file_cache": "/tmp/opcache" }, }, }, @@ -75,7 +76,7 @@ try { assert.equal(calls[0].wordpressInstallMode, "do-not-attempt-installing") assert.equal(calls[0].skipSqliteSetup, true) assert.equal(shouldUseProgrammaticPlaygroundRunner(spec), false) - assert.deepEqual(calls[0].phpIniEntries, { memory_limit: "512M" }) + assert.deepEqual(calls[0].phpIniEntries, { memory_limit: "2G", max_input_vars: "2048" }) assert.deepEqual(calls[0].phpExtension, ["/tmp/sodium/manifest.json"]) const sharedPhpIniPath = calls[0]["mount-before-install"]?.[0]?.hostPath const sharedAutoPrependPath = calls[0]["mount-before-install"]?.[1]?.hostPath @@ -85,7 +86,7 @@ try { assert.match(sharedPhpIni, /opcache\.file_cache = \/tmp\/opcache/) // The runtime default memory ceiling stays high enough for collect_artifacts to // base64 heavy snapshot/declared-artifact files without a hard PHP fatal. - assert.match(sharedPhpIni, /memory_limit=512M/) + assert.match(sharedPhpIni, /memory_limit = 2G/) assert.match(sharedPhpIni, /auto_prepend_file=\/internal\/shared\/wp-codebox-auto-prepend\.php/) const sharedAutoPrepend = await readFile(sharedAutoPrependPath as string, "utf8") assert.match(sharedAutoPrepend, /require_once '\/internal\/shared\/auto_prepend_file\.php'/) From 8926d510b4a9849768428bf50b8c7dadaa8e493b Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Thu, 23 Jul 2026 03:21:53 -0400 Subject: [PATCH 10/17] Retain invalid materialization responses --- .../src/mount-materialization.ts | 3 ++- tests/staged-input-materialization.test.ts | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/runtime-playground/src/mount-materialization.ts b/packages/runtime-playground/src/mount-materialization.ts index be3d496d..9c9f1459 100644 --- a/packages/runtime-playground/src/mount-materialization.ts +++ b/packages/runtime-playground/src/mount-materialization.ts @@ -371,7 +371,8 @@ function parseMaterializationJson(text: string, s try { parsed = JSON.parse(text || "{}") } catch (error) { - throw new Error(`${command} returned invalid JSON: ${errorMessage(error)}`) + const excerpt = text.trim().replace(/\s+/g, " ").slice(0, 500) + throw new Error(`${command} returned invalid JSON: ${errorMessage(error)}${excerpt ? `; response excerpt: ${excerpt}` : ""}`) } if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) || (parsed as { schema?: unknown }).schema !== schema) { throw new Error(`${command} did not return ${schema}; received ${text.trim() || "empty response"}`) diff --git a/tests/staged-input-materialization.test.ts b/tests/staged-input-materialization.test.ts index 015e4c48..06886f50 100644 --- a/tests/staged-input-materialization.test.ts +++ b/tests/staged-input-materialization.test.ts @@ -115,4 +115,24 @@ await withTempDir("wp-codebox-staged-input-materialization-binary-", async (root assert.ok(binaryWriteCode.includes(contents.toString("base64")), "the PHP materializer receives the exact base64 payload") }) +await withTempDir("wp-codebox-staged-input-materialization-invalid-response-", async (root) => { + const source = join(root, "image.bin") + await writeFile(source, Buffer.from([0xff])) + const server = { + playground: { + async run({ code }: { code: string }) { + if (code.includes("wp-codebox/host-mount-materialization/v1")) { + return { text: "
\nFatal error: materialization failed" } + } + return { text: JSON.stringify({ schema: "wp-codebox/host-mount-directory-materialization/v1", created: 1, skipped: 0 }) } + }, + }, + } + + await assert.rejects( + materializePlaygroundStagedInputs(server as never, [{ type: "file", source, target: "/wordpress/image.bin", mode: "readwrite" }]), + /response excerpt:
Fatal error<\/b>: materialization failed/, + ) +}) + console.log("staged input materialization ok") From b3427cf44bbfca2d133e8d818169fa30596f6143 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Thu, 23 Jul 2026 03:32:17 -0400 Subject: [PATCH 11/17] Keep materialization responses machine-readable --- packages/runtime-playground/src/mount-materialization.ts | 8 ++++---- tests/staged-input-materialization.test.ts | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/runtime-playground/src/mount-materialization.ts b/packages/runtime-playground/src/mount-materialization.ts index 9c9f1459..c5991ee8 100644 --- a/packages/runtime-playground/src/mount-materialization.ts +++ b/packages/runtime-playground/src/mount-materialization.ts @@ -488,7 +488,7 @@ foreach (($payload['directories'] ?? array()) as $directory) { $skipped++; continue; } - if (is_dir($directory) || mkdir($directory, 0777, true) || is_dir($directory)) { + if (is_dir($directory) || @mkdir($directory, 0777, true) || is_dir($directory)) { $created++; continue; } @@ -502,12 +502,12 @@ foreach (($payload['files'] ?? array()) as $file) { continue; } $directory = dirname($target); - if (!is_dir($directory) && !mkdir($directory, 0777, true) && !is_dir($directory)) { + if (!is_dir($directory) && !@mkdir($directory, 0777, true) && !is_dir($directory)) { $skipped++; continue; } $decoded = base64_decode($contents, true); - if (false === $decoded || false === file_put_contents($target, $decoded)) { + if (false === $decoded || false === @file_put_contents($target, $decoded)) { $skipped++; continue; } @@ -532,7 +532,7 @@ foreach (($payload['directories'] ?? array()) as $directory) { $skipped++; continue; } - if (is_dir($directory) || mkdir($directory, 0777, true) || is_dir($directory)) { + if (is_dir($directory) || @mkdir($directory, 0777, true) || is_dir($directory)) { $created++; continue; } diff --git a/tests/staged-input-materialization.test.ts b/tests/staged-input-materialization.test.ts index 06886f50..992edfc6 100644 --- a/tests/staged-input-materialization.test.ts +++ b/tests/staged-input-materialization.test.ts @@ -113,6 +113,8 @@ await withTempDir("wp-codebox-staged-input-materialization-binary-", async (root assert.equal(result.skipped, 0) assert.equal(directWrites, 0, "arbitrary bytes bypass the UTF-8-only direct writer") assert.ok(binaryWriteCode.includes(contents.toString("base64")), "the PHP materializer receives the exact base64 payload") + assert.match(binaryWriteCode, /@mkdir\(/, "expected filesystem failures do not corrupt the JSON response channel") + assert.match(binaryWriteCode, /@file_put_contents\(/, "expected write failures remain structured skipped results") }) await withTempDir("wp-codebox-staged-input-materialization-invalid-response-", async (root) => { From 532e3f781aebc40fc2da50a69cc4952b0fdeb397 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Thu, 23 Jul 2026 05:10:02 -0400 Subject: [PATCH 12/17] Handle dangling readonly mount links --- .../src/mount-materialization.ts | 19 +++++++++++++++++-- tests/playground-readonly-mounts.test.ts | 13 +++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/packages/runtime-playground/src/mount-materialization.ts b/packages/runtime-playground/src/mount-materialization.ts index c5991ee8..b820ae08 100644 --- a/packages/runtime-playground/src/mount-materialization.ts +++ b/packages/runtime-playground/src/mount-materialization.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto" -import { cp, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises" +import { cp, lstat, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path" import { materializationPhaseResult, namedFileTreeSkipPolicyNames, phpStringArrayLiteral, type MaterializationPhaseResult, type MountSpec } from "@automattic/wp-codebox-core" @@ -83,7 +83,22 @@ export async function stageReadonlyPlaygroundMounts(mounts: MountSpec[]): Promis return mount } const source = join(root, `${index}-${basename(mount.source) || "mount"}`) - await cp(mount.source, source, { recursive: mount.type !== "file", dereference: true }) + await stat(mount.source) + await cp(mount.source, source, { + recursive: mount.type !== "file", + dereference: true, + filter: async (candidate) => { + const entry = await lstat(candidate) + if (!entry.isSymbolicLink()) return true + try { + await stat(candidate) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false + throw error + } + }, + }) return { ...mount, source } })) const failedMount = stagedMountResults.find((result) => result.status === "rejected") diff --git a/tests/playground-readonly-mounts.test.ts b/tests/playground-readonly-mounts.test.ts index 8d6243d1..a87e5e2e 100644 --- a/tests/playground-readonly-mounts.test.ts +++ b/tests/playground-readonly-mounts.test.ts @@ -53,11 +53,20 @@ try { await symlink("missing.php", join(danglingSource, "build.sh")) await Promise.all(Array.from({ length: 500 }, (_, index) => writeFile(join(largeSource, `File${index}.php`), ` Date: Thu, 23 Jul 2026 06:20:19 -0400 Subject: [PATCH 13/17] Recognize TeamCity PHPUnit completion --- packages/runtime-playground/src/wordpress-command-runners.ts | 1 + tests/phpunit-runtime-failure-diagnostics.test.ts | 2 ++ 2 files changed, 3 insertions(+) diff --git a/packages/runtime-playground/src/wordpress-command-runners.ts b/packages/runtime-playground/src/wordpress-command-runners.ts index 4cdbba94..ca1e005d 100644 --- a/packages/runtime-playground/src/wordpress-command-runners.ts +++ b/packages/runtime-playground/src/wordpress-command-runners.ts @@ -980,6 +980,7 @@ export async function runPhpunitCommand({ export function hasSuccessfulPhpunitSummary(output: string): boolean { return /\bOK(?:, but (?:there were issues!|incomplete, skipped, or risky tests!))?(?: \([1-9]\d* tests?,|\s+Tests:\s*[1-9]\d*,\s*Assertions:)/m.test(output) + || (/##teamcity\[testStarted\b/m.test(output) && /##teamcity\[testSuiteFinished\b/m.test(output)) } function successfulPhpunitResponseDiagnostics(response: PlaygroundRunResponse): string { diff --git a/tests/phpunit-runtime-failure-diagnostics.test.ts b/tests/phpunit-runtime-failure-diagnostics.test.ts index dd59b3c1..04628af8 100644 --- a/tests/phpunit-runtime-failure-diagnostics.test.ts +++ b/tests/phpunit-runtime-failure-diagnostics.test.ts @@ -53,6 +53,8 @@ const emptySuccessArtifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-phpuni const successfulResponseSecret = "ghp_abcdefghijklmnopqrstuvwxyz1234567890" assert.equal(hasSuccessfulPhpunitSummary("OK (1 test, 1 assertion)"), true) assert.equal(hasSuccessfulPhpunitSummary("OK, but incomplete, skipped, or risky tests!\nTests: 21, Assertions: 27398, Skipped: 1."), true) +assert.equal(hasSuccessfulPhpunitSummary("##teamcity[testSuiteStarted name='example']\n##teamcity[testStarted name='works']\n##teamcity[testFinished name='works']\n##teamcity[testSuiteFinished name='example']"), true) +assert.equal(hasSuccessfulPhpunitSummary("##teamcity[testSuiteStarted name='example']\n##teamcity[testSuiteFinished name='example']"), false) assert.equal(hasSuccessfulPhpunitSummary("Tests: 21, Assertions: 27398, Failures: 1."), false) await assert.rejects( () => runPhpunitCommand({ From cdd5eb93702d30ca0a4a5af3a7ee7cf6ea5b5fcd Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Thu, 23 Jul 2026 16:42:00 -0400 Subject: [PATCH 14/17] Persist bounded plan progress --- packages/cli/src/bounded-recipe-plan.ts | 42 +++++++++++++++++-- .../runtime-core/src/bounded-runtime-plan.ts | 3 ++ tests/bounded-recipe-plan.test.ts | 5 +++ tests/bounded-runtime-plan.test.ts | 3 ++ 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/bounded-recipe-plan.ts b/packages/cli/src/bounded-recipe-plan.ts index aa8f99ee..69e8621f 100644 --- a/packages/cli/src/bounded-recipe-plan.ts +++ b/packages/cli/src/bounded-recipe-plan.ts @@ -1,6 +1,6 @@ -import { mkdir, readFile, writeFile } from "node:fs/promises" +import { mkdir, readFile, rename, writeFile } from "node:fs/promises" import { join } from "node:path" -import { commandArgValue, executeBoundedRuntimePlan, parseCommandJsonObject, type BoundedRuntimePlan, type BoundedRuntimePlanResult, type ExecutionResult, type Runtime } from "@automattic/wp-codebox-core" +import { commandArgValue, executeBoundedRuntimePlan, parseCommandJsonObject, type BoundedRuntimePlan, type BoundedRuntimePlanEntryResult, type BoundedRuntimePlanResult, type ExecutionResult, type Runtime } from "@automattic/wp-codebox-core" import { recipeExecutionSpec } from "./agent-sandbox.js" export interface BoundedRecipePlanExecutionOptions { @@ -8,7 +8,14 @@ export interface BoundedRecipePlanExecutionOptions { recipeDirectory: string } +const BOUNDED_RUNTIME_PLAN_PROGRESS_SCHEMA = "wp-codebox/bounded-runtime-plan-progress/v1" + export async function executeBoundedRecipePlan(runtime: Runtime, plan: BoundedRuntimePlan, options: BoundedRecipePlanExecutionOptions): Promise { + const progressPath = join(options.artifactRoot, "bounded-plan", "progress.json") + const completed = new Map() + let progressWrite = Promise.resolve() + await mkdir(join(options.artifactRoot, "bounded-plan"), { recursive: true }) + await writeBoundedPlanProgress(progressPath, plan, completed, false) const aggregate = await executeBoundedRuntimePlan(plan, { async materialize() { return { workspace: options.recipeDirectory, runtime } }, async startServices() { return undefined }, @@ -50,10 +57,15 @@ export async function executeBoundedRecipePlan(runtime: Runtime, plan: BoundedRu }, null, 2)}\n`, "utf8") return { success: exitCode === 0, exitCode, message: stderr, stdoutRef, stderrRef, resultRef, artifactRefs } }, + async onEntryResult(result) { + completed.set(result.id, result) + progressWrite = progressWrite.then(async () => writeBoundedPlanProgress(progressPath, plan, completed, false)) + await progressWrite + }, async stopServices() {}, async dispose() {}, }) - await mkdir(join(options.artifactRoot, "bounded-plan"), { recursive: true }) + await writeBoundedPlanProgress(progressPath, plan, completed, true) await writeFile(join(options.artifactRoot, "bounded-plan/result.json"), `${JSON.stringify(aggregate, null, 2)}\n`, "utf8") return aggregate } @@ -76,3 +88,27 @@ function runtimeArtifactRefs(execution: ExecutionResult | undefined): string[] { return typeof value === "string" && value ? [value] : [] }) } + +async function writeBoundedPlanProgress(path: string, plan: BoundedRuntimePlan, completed: Map, complete: boolean): Promise { + const entries = plan.entries.flatMap((entry) => { + const result = completed.get(entry.id) + return result ? [result] : [] + }) + const progress = { + schema: BOUNDED_RUNTIME_PLAN_PROGRESS_SCHEMA, + complete, + concurrency: Math.min(plan.concurrency, plan.entries.length), + counts: { + total: plan.entries.length, + succeeded: entries.filter((entry) => entry.status === "succeeded").length, + failed: entries.filter((entry) => entry.status === "failed").length, + timedOut: entries.filter((entry) => entry.status === "timed_out").length, + cancelled: entries.filter((entry) => entry.status === "cancelled").length, + unfinished: plan.entries.length - entries.length, + }, + entries, + } + const temporaryPath = `${path}.tmp` + await writeFile(temporaryPath, `${JSON.stringify(progress, null, 2)}\n`, "utf8") + await rename(temporaryPath, path) +} diff --git a/packages/runtime-core/src/bounded-runtime-plan.ts b/packages/runtime-core/src/bounded-runtime-plan.ts index bca5bea0..c7f3e7b2 100644 --- a/packages/runtime-core/src/bounded-runtime-plan.ts +++ b/packages/runtime-core/src/bounded-runtime-plan.ts @@ -64,6 +64,7 @@ export interface BoundedRuntimePlanAdapter startServices(context: { workspace: TWorkspace; runtime: TRuntime }): Promise execute(context: BoundedRuntimePlanExecution): Promise<{ success: boolean; exitCode?: number; message?: string; stdoutRef?: string; stderrRef?: string; resultRef?: string; artifactRefs?: string[] }> + onEntryResult?(result: BoundedRuntimePlanEntryResult): Promise stopServices(context: { workspace: TWorkspace; runtime: TRuntime; services: TServices }): Promise dispose(context: { workspace: TWorkspace; runtime: TRuntime }): Promise } @@ -134,10 +135,12 @@ async function executeEntries(plan: BoundedRunt const entry = plan.entries[index]! if (failed && plan.failFast) { results[index] = cancelledResult(entry) + await adapter.onEntryResult?.(results[index]) continue } const result = await executeEntry(entry, materialized, services, adapter) results[index] = result + await adapter.onEntryResult?.(result) if (!result.success) failed = true } } diff --git a/tests/bounded-recipe-plan.test.ts b/tests/bounded-recipe-plan.test.ts index 7d5769f7..56f3a960 100644 --- a/tests/bounded-recipe-plan.test.ts +++ b/tests/bounded-recipe-plan.test.ts @@ -57,6 +57,11 @@ try { assert.equal(await readFile(join(root, "entries/one/stdout.txt"), "utf8"), "password=[redacted]\n") assert.equal(await readFile(join(root, "entries/failed/stderr.txt"), "utf8"), "database [redacted] failed\n") assert.equal(JSON.parse(await readFile(join(root, "bounded-plan/result.json"), "utf8")).schema, "wp-codebox/bounded-runtime-plan-result/v1") + const progress = JSON.parse(await readFile(join(root, "bounded-plan/progress.json"), "utf8")) + assert.equal(progress.schema, "wp-codebox/bounded-runtime-plan-progress/v1") + assert.equal(progress.complete, true) + assert.deepEqual(progress.counts, { total: 2, succeeded: 1, failed: 1, timedOut: 0, cancelled: 0, unfinished: 0 }) + assert.deepEqual(progress.entries.map((entry: { id: string }) => entry.id), ["one", "failed"]) } finally { await rm(root, { recursive: true, force: true }) } diff --git a/tests/bounded-runtime-plan.test.ts b/tests/bounded-runtime-plan.test.ts index 88599de7..d232aa68 100644 --- a/tests/bounded-runtime-plan.test.ts +++ b/tests/bounded-runtime-plan.test.ts @@ -13,6 +13,7 @@ const plan: BoundedRuntimePlan = { } const lifecycle: string[] = [] +const completedEntries: string[] = [] let active = 0 let maximumActive = 0 const executed: string[] = [] @@ -31,6 +32,7 @@ const adapter: BoundedRuntimePlanAdapter<{ root: string }, { id: string }, { id: active-- } }, + async onEntryResult(result) { completedEntries.push(result.id) }, async stopServices() { lifecycle.push("stop-services") }, async dispose() { lifecycle.push("dispose") }, } @@ -44,6 +46,7 @@ assert.deepEqual(result.counts, { total: 4, succeeded: 2, failed: 1, timedOut: 1 assert.equal(result.entries.every((entry) => Number.isInteger(entry.durationMs) && entry.durationMs >= 0), true) assert.deepEqual(lifecycle, ["materialize", "start-services", "stop-services", "dispose"], "workspace, service, and runtime lifecycles are each owned once") assert.deepEqual(executed.sort(), ["failed", "first", "last", "slow"], "one failed entry does not isolate unrelated entries") +assert.deepEqual(completedEntries.sort(), ["failed", "first", "last", "slow"], "each terminal entry is reported to the adapter") const retry = retryBoundedRuntimePlan(plan, result) assert.deepEqual(retry.entries.map((entry) => entry.id), ["failed", "slow"], "retry selects only unsuccessful prior entries") From 7c2b47f61feed81f7967071542c8d8a597aa35db Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Fri, 24 Jul 2026 16:19:15 -0400 Subject: [PATCH 15/17] Skip concrete PHPUnit helper classes --- .../src/phpunit-command-handlers.ts | 20 +++++++++++++++++-- tests/phpunit-project-autoload.test.ts | 19 +++++++++++++++++- ...phpunit-readonly-cache.integration.test.ts | 2 +- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/packages/runtime-playground/src/phpunit-command-handlers.ts b/packages/runtime-playground/src/phpunit-command-handlers.ts index 30c2c05c..6f8d7cdc 100644 --- a/packages/runtime-playground/src/phpunit-command-handlers.ts +++ b/packages/runtime-playground/src/phpunit-command-handlers.ts @@ -350,6 +350,20 @@ function ${functionName}(array $argv) { }` } +function phpunitClassHasTestsPhp(functionName: string): string { + return `function ${functionName}(ReflectionClass $class): bool { + if ($class->hasMethod('suite') && $class->getMethod('suite')->isStatic()) { + return true; + } + foreach ((new PHPUnit\\Util\\Reflection())->publicMethodsInTestClass($class) as $method) { + if (PHPUnit\\Util\\Test::isTestMethod($method)) { + return true; + } + } + return false; +}` +} + export function phpunitRunCode(options: PhpunitRunCodeOptions): string { return `error_reporting(E_ALL); ini_set('display_errors', '1'); @@ -1314,10 +1328,11 @@ try { exit(1); } $after_classes = get_declared_classes(); +${phpunitClassHasTestsPhp("pg_phpunit_class_has_tests")} foreach (array_diff($after_classes, $before_classes) as $class_name) { try { $ref = new ReflectionClass($class_name); - if (!$ref->isAbstract() && $ref->isSubclassOf('PHPUnit\\Framework\\TestCase')) { + if (!$ref->isAbstract() && $ref->isSubclassOf('PHPUnit\\Framework\\TestCase') && pg_phpunit_class_has_tests($ref)) { $suite->addTestSuite($ref); } } catch (Throwable $e) { @@ -1637,10 +1652,11 @@ try { exit(1); } $after_classes = get_declared_classes(); +${phpunitClassHasTestsPhp("core_pg_phpunit_class_has_tests")} foreach (array_diff($after_classes, $before_classes) as $class_name) { try { $ref = new ReflectionClass($class_name); - if (!$ref->isAbstract() && $ref->isSubclassOf('PHPUnit\\Framework\\TestCase')) { + if (!$ref->isAbstract() && $ref->isSubclassOf('PHPUnit\\Framework\\TestCase') && core_pg_phpunit_class_has_tests($ref)) { $suite->addTestSuite($ref); } } catch (Throwable $e) { diff --git a/tests/phpunit-project-autoload.test.ts b/tests/phpunit-project-autoload.test.ts index efa8e27c..b2edc371 100644 --- a/tests/phpunit-project-autoload.test.ts +++ b/tests/phpunit-project-autoload.test.ts @@ -325,7 +325,11 @@ function assertDiscoveredTestExecutes(source: string, stagePrefix: "pg" | "core_ : `public function __construct($name) { $this->name = $name; }` writeFileSync(testFile, `tests[] = $class->newInstance(); } + public function addTestSuite(\\ReflectionClass $class): void { + if ($class->getName() === 'ConcreteHelper') { throw new \\RuntimeException('concrete helper was scheduled as a test'); } + $this->tests[] = $class->newInstance(); + } public function tests(): array { return $this->tests; } public function count(): int { return count($this->tests); } } } +namespace PHPUnit\\Util { +final class Reflection { + public function publicMethodsInTestClass(\\ReflectionClass $class): array { return $class->getMethods(\\ReflectionMethod::IS_PUBLIC); } +} +final class Test { + public static function isTestMethod(\\ReflectionMethod $method): bool { + return strpos($method->getName(), 'test') === 0 || strpos((string) $method->getDocComment(), '@test') !== false; + } +} +} namespace PHPUnit\\TextUI { final class TestResult { private $count; diff --git a/tests/playground-phpunit-readonly-cache.integration.test.ts b/tests/playground-phpunit-readonly-cache.integration.test.ts index 8588ddd8..4d0597d2 100644 --- a/tests/playground-phpunit-readonly-cache.integration.test.ts +++ b/tests/playground-phpunit-readonly-cache.integration.test.ts @@ -64,7 +64,7 @@ async function writeFixture(): Promise { await writeFile(join(plugin, "readonly-phpunit-fixture.php"), "\ntests\n") await writeFile(join(plugin, "source-sentinel.bin"), sentinel) - await writeFile(join(plugin, "tests", "ReadonlyCacheTest.php"), "assertTrue(is_multisite()); } public function test_sentinel_is_available(): void { $this->assertGreaterThan(0, filesize(dirname(__DIR__) . \'/source-sentinel.bin\')); } public function test_dependency_activation_runs_after_install(): void { $this->assertGreaterThanOrEqual(1, get_option(\'wp_codebox_dependency_activation_users\')); } public function test_dependency_plugins_loaded_runs_once(): void { $this->assertSame(1, (int) get_option(\'wp_codebox_dependency_plugins_loaded_count\')); } public function test_wp_cli_namespaced_stdout_is_available(): void { $this->assertTrue(eval(\'namespace cli; return is_resource(STDOUT);\')); } }\n") + await writeFile(join(plugin, "tests", "ReadonlyCacheTest.php"), "assertTrue(is_multisite()); } public function test_sentinel_is_available(): void { $this->assertGreaterThan(0, filesize(dirname(__DIR__) . \'/source-sentinel.bin\')); } public function test_dependency_activation_runs_after_install(): void { $this->assertGreaterThanOrEqual(1, get_option(\'wp_codebox_dependency_activation_users\')); } public function test_dependency_plugins_loaded_runs_once(): void { $this->assertSame(1, (int) get_option(\'wp_codebox_dependency_plugins_loaded_count\')); } public function test_wp_cli_namespaced_stdout_is_available(): void { $this->assertTrue(eval(\'namespace cli; return is_resource(STDOUT);\')); } }\n") await writeFile(join(dependency, "activation-dependency.php"), " 1)))); });\n") } From f64b8a3978e26b81cebf3515874ff046eac3769b Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Fri, 24 Jul 2026 16:26:05 -0400 Subject: [PATCH 16/17] Ignore inherited PHPUnit suite factories --- .../runtime-playground/src/phpunit-command-handlers.ts | 7 +++++-- tests/phpunit-project-autoload.test.ts | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/runtime-playground/src/phpunit-command-handlers.ts b/packages/runtime-playground/src/phpunit-command-handlers.ts index 6f8d7cdc..54830899 100644 --- a/packages/runtime-playground/src/phpunit-command-handlers.ts +++ b/packages/runtime-playground/src/phpunit-command-handlers.ts @@ -352,8 +352,11 @@ function ${functionName}(array $argv) { function phpunitClassHasTestsPhp(functionName: string): string { return `function ${functionName}(ReflectionClass $class): bool { - if ($class->hasMethod('suite') && $class->getMethod('suite')->isStatic()) { - return true; + if ($class->hasMethod('suite')) { + $suite_method = $class->getMethod('suite'); + if ($suite_method->isStatic() && $suite_method->getDeclaringClass()->getName() === $class->getName()) { + return true; + } } foreach ((new PHPUnit\\Util\\Reflection())->publicMethodsInTestClass($class) as $method) { if (PHPUnit\\Util\\Test::isTestMethod($method)) { diff --git a/tests/phpunit-project-autoload.test.ts b/tests/phpunit-project-autoload.test.ts index b2edc371..73d52e48 100644 --- a/tests/phpunit-project-autoload.test.ts +++ b/tests/phpunit-project-autoload.test.ts @@ -325,10 +325,13 @@ function assertDiscoveredTestExecutes(source: string, stagePrefix: "pg" | "core_ : `public function __construct($name) { $this->name = $name; }` writeFileSync(testFile, ` Date: Fri, 24 Jul 2026 19:50:49 -0400 Subject: [PATCH 17/17] Match PHPUnit test file ownership --- .../src/phpunit-command-handlers.ts | 32 +++++++++++++++++-- tests/phpunit-project-autoload.test.ts | 12 ++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/runtime-playground/src/phpunit-command-handlers.ts b/packages/runtime-playground/src/phpunit-command-handlers.ts index 54830899..aa6c0f7b 100644 --- a/packages/runtime-playground/src/phpunit-command-handlers.ts +++ b/packages/runtime-playground/src/phpunit-command-handlers.ts @@ -367,6 +367,32 @@ function phpunitClassHasTestsPhp(functionName: string): string { }` } +function phpunitClassesOwnedByTestFilesPhp(functionName: string): string { + return `function ${functionName}(array $class_names): array { + $classes = array(); + $owners = array(); + foreach ($class_names as $class_name) { + try { + $class = new ReflectionClass($class_name); + $file = $class->getFileName(); + if ($file === false) { + continue; + } + $classes[$class_name] = array($class, $file); + if ($class->getShortName() === pathinfo($file, PATHINFO_FILENAME)) { + $owners[$file] = $class_name; + } + } catch (Throwable $e) { + continue; + } + } + return array_values(array_filter(array_keys($classes), static function($class_name) use ($classes, $owners) { + $file = $classes[$class_name][1]; + return !isset($owners[$file]) || $owners[$file] === $class_name; + })); +}` +} + export function phpunitRunCode(options: PhpunitRunCodeOptions): string { return `error_reporting(E_ALL); ini_set('display_errors', '1'); @@ -1332,7 +1358,8 @@ try { } $after_classes = get_declared_classes(); ${phpunitClassHasTestsPhp("pg_phpunit_class_has_tests")} -foreach (array_diff($after_classes, $before_classes) as $class_name) { +${phpunitClassesOwnedByTestFilesPhp("pg_phpunit_classes_owned_by_test_files")} +foreach (pg_phpunit_classes_owned_by_test_files(array_diff($after_classes, $before_classes)) as $class_name) { try { $ref = new ReflectionClass($class_name); if (!$ref->isAbstract() && $ref->isSubclassOf('PHPUnit\\Framework\\TestCase') && pg_phpunit_class_has_tests($ref)) { @@ -1656,7 +1683,8 @@ try { } $after_classes = get_declared_classes(); ${phpunitClassHasTestsPhp("core_pg_phpunit_class_has_tests")} -foreach (array_diff($after_classes, $before_classes) as $class_name) { +${phpunitClassesOwnedByTestFilesPhp("core_pg_phpunit_classes_owned_by_test_files")} +foreach (core_pg_phpunit_classes_owned_by_test_files(array_diff($after_classes, $before_classes)) as $class_name) { try { $ref = new ReflectionClass($class_name); if (!$ref->isAbstract() && $ref->isSubclassOf('PHPUnit\\Framework\\TestCase') && core_pg_phpunit_class_has_tests($ref)) { diff --git a/tests/phpunit-project-autoload.test.ts b/tests/phpunit-project-autoload.test.ts index 73d52e48..6ebc7c2b 100644 --- a/tests/phpunit-project-autoload.test.ts +++ b/tests/phpunit-project-autoload.test.ts @@ -313,6 +313,7 @@ echo "BOUNDARY_OK\n"; function assertDiscoveredTestExecutes(source: string, stagePrefix: "pg" | "core_pg", privateConstructor: boolean): void { const tempDir = mkdtempSync(join(tmpdir(), `wp-codebox-${stagePrefix}-testsuite-`)) const testFile = join(tempDir, "DiscoveredTest.php") + const legacyTestFile = join(tempDir, "LegacyTest.php") const scriptPath = join(tempDir, "run-generated-harness.php") const executionMarker = join(tempDir, "executed.txt") const stageLog = join(tempDir, "stages.txt") @@ -337,6 +338,15 @@ class DiscoveredTest extends ProjectTestCase { file_put_contents(getenv('EXECUTION_MARKER'), 'executed'); } } +`) + writeFileSync(legacyTestFile, `