diff --git a/docs/native-mariadb-runtime-service.md b/docs/native-mariadb-runtime-service.md new file mode 100644 index 00000000..bcb5378e --- /dev/null +++ b/docs/native-mariadb-runtime-service.md @@ -0,0 +1,86 @@ +# Native MariaDB Runtime Service + +`provider: "native"` provisions a short-lived MariaDB process for MySQL-compatible workloads on Docker-free hosts. The generic runtime-service layer owns its complete lifecycle. Recipes may select only `provider: "native"` and `engine: "mariadb"`; host, port, credentials, sockets, configuration files, and filesystem paths are not configurable. + +## Isolation Contract + +- The provider rejects UID 0 and resolves and validates `mariadbd`, `mariadb-install-db`, `mariadb`, util-linux `prlimit`, `truncate`, `mkfs.ext4`, `fuse2fs`, and `fusermount3` identities before allocation. +- Default executable discovery ignores the caller's `PATH`, searches fixed system directories, resolves symlinks, and requires every executable and ancestor to be UID-0-owned and not group/other writable. Child processes receive a fixed minimal `PATH`. +- Every run gets a mode-`0700` `mkdtemp` root. MariaDB's data directory, temporary directory, socket, PID file, error log, plugin directory, secure-file directory, home, and working directory are all inside the bounded image. +- Initialization, daemon, and administrative clients run through fixed hard rlimits; each database command itself begins with `--no-defaults`. No shell or default socket is used. +- Administration uses only the private Unix socket. Workloads receive a generated least-privilege `runtime` account over loopback TCP through the existing ephemeral connector-secret channel. +- Initialization, FUSE, and daemon commands each run in a new owned process group. Cleanup addresses the retained group, not a PID file; it waits for the complete group to disappear after graceful shutdown, then applies group-wide `SIGTERM` and `SIGKILL` as needed. Linux captures the leader start-time token and revalidates it while the leader is alive. Root device/inode identity and a symlink-free tree are revalidated before recursive removal. +- Failures, aborts, timeouts, startup crashes, and partial initialization all enter the same graceful-shutdown, forced-shutdown, wait, and verified-removal state machine. Cleanup failure is terminal and retained in bounded lifecycle evidence. +- Address space is capped at 2 GiB, CPU at 300 seconds, individual daemon files at 128 MiB, open files at 512, and processes/threads at 512. Core files and locked memory are disabled. The datadir is a provider-owned 256 MiB ext4 image formatted with 4,096 inodes and mounted through unprivileged FUSE; device, byte, and inode geometry must be proven before initialization. Hosts without this containment fail closed. +- Recipes may declare at most two native services, bounding the aggregate native ceiling to two 2-GiB address spaces, two 256-MiB images, and the corresponding process/file limits. +- The daemon uses an empty bounded plugin directory and a bounded `secure-file-priv` directory. Startup fails unless every enabled storage engine is on the fixed local-only allowlist; FEDERATED, CONNECT, SPIDER, S3, and unknown enabled engines are rejected. The runtime account has privileges only on its generated database and cannot install plugins. +- Runtime descriptor discovery distinguishes package support from host availability. It advertises the active native capability only after trusted tools and a real create, mount, geometry, write, unmount, process-group-exit, and removal probe succeed; unavailable reasons are stable codes without private paths. +- Cleanup is single-flight for concurrent callers. A failed attempt may be retried, and evidence transitions from `teardown: failed` to a consistent released/completed state only after the retry proves cleanup. +- Evidence contains service ID, engine/provider version, lifecycle state, and memory measurements only. It never contains credentials or private absolute paths. + +## Daemon Arguments + +The provider starts `mariadbd` through `prlimit` with these fixed controls plus provider-owned absolute paths and a freshly allocated loopback port: + +```text +--no-defaults +--datadir=/database +--socket=/runtime/server.sock +--pid-file=/runtime/server.pid +--log-error=/runtime/server.log +--tmpdir=/tmp +--plugin-dir=/plugins +--secure-file-priv=/files +--bind-address=127.0.0.1 +--port= +--skip-name-resolve +--skip-log-bin +--skip-host-cache +--skip-slave-start +--skip-symbolic-links +--local-infile=OFF +--performance-schema=OFF +--skip-feedback +--innodb-buffer-pool-size=32M +--innodb-buffer-pool-size-max=32M +--innodb-log-buffer-size=4M +--key-buffer-size=8M +--aria-pagecache-buffer-size=8M +--thread-handling=pool-of-threads +--thread-pool-size=1 +--aria-log-file-size=16M +--innodb-file-per-table=OFF +--innodb-data-file-path=ibdata1:32M:autoextend:max:96M +--innodb-temp-data-file-path=ibtmp1:16M:autoextend:max:32M +--innodb-log-file-size=32M +--max-connections=10 +--max-prepared-stmt-count=256 +--max-session-mem-used=32M +--open-files-limit=512 +--thread-cache-size=0 +--table-open-cache=128 +--table-definition-cache=128 +--tmp-table-size=8M +--max-heap-table-size=8M +``` + +The hard address-space ceiling is 2 GiB. Linux runs record the owned daemon's post-readiness RSS in evidence; the MariaDB 10.11 integration gate additionally requires observed RSS to remain at or below 128 MiB. + +## Integration Contract + +Homeboy Extensions issue #2412 may select this provider by emitting the generic runtime-service recipe declaration below. WP Codebox performs all provisioning and cleanup; callers do not supply or manage native process details. + +```json +{ + "id": "wordpress-database", + "kind": "mysql", + "configuration": { "provider": "native", "engine": "mariadb" }, + "outputs": { + "host": "DB_HOST", + "port": "DB_PORT", + "username": "DB_USER", + "password": "DB_PASSWORD", + "database": "DB_NAME" + } +} +``` diff --git a/package.json b/package.json index 54f0d947..9b8e79ab 100644 --- a/package.json +++ b/package.json @@ -248,8 +248,9 @@ "test:generic-ability-runtime-run": "tsx tests/generic-ability-runtime-run.test.ts", "test:provider-runtime-contracts": "tsx tests/provider-runtime-contracts.test.ts", "test:runtime-requirements-readiness": "tsx tests/runtime-requirements-readiness.test.ts", - "test:runtime-services": "tsx tests/runtime-services.test.ts && tsx tests/external-mysql-runtime-service.test.ts", + "test:runtime-services": "tsx tests/runtime-services.test.ts && tsx tests/external-mysql-runtime-service.test.ts && tsx tests/native-mariadb-runtime-service.test.ts", "test:runtime-services-lifecycle": "tsx tests/runtime-services-lifecycle.test.ts", + "test:native-mariadb-runtime-service-integration": "tsx tests/native-mariadb-runtime-service.integration.test.ts", "test:disposable-mysql-mysqli-e2e": "tsx tests/disposable-mysql-mysqli.integration.test.ts", "test:runtime-contract-manifest": "tsx tests/runtime-contract-manifest.test.ts", "test:runtime-contract-package-exports": "tsx tests/runtime-contract-package-exports.test.ts", diff --git a/packages/cli/src/commands/discovery.ts b/packages/cli/src/commands/discovery.ts index cdca0f43..9735abc9 100644 --- a/packages/cli/src/commands/discovery.ts +++ b/packages/cli/src/commands/discovery.ts @@ -2,6 +2,7 @@ import { createWorkspaceRecipeJsonSchema, runtimeDescriptor, type RuntimeDescrip import { commandRegistry, type CommandDefinition } from "@automattic/wp-codebox-core/contracts" import { printCommandCatalogHumanOutput, printRecipeSchemaHumanOutput, printRuntimeDescriptorHumanOutput } from "../output.js" import { cliRuntimeBackendRecipePolicy, listCliRecipeCommandDefinitions, listCliRuntimeBackendKinds } from "../runtime-backends.js" +import { nativeMariaDbHostReadiness } from "../runtime-services.js" interface CommandCatalogOutput { schema: "wp-codebox/command-catalog/v1" @@ -40,7 +41,7 @@ export async function runRecipeSchemaCommand(args: string[]): Promise { export async function runRuntimeDescriptorCommand(args: string[]): Promise { const json = parseDiscoveryJsonOption(args) - const output = runtimeDescriptorOutput() + const output = await runtimeDescriptorOutput() if (!json) { printRuntimeDescriptorHumanOutput(output) return 0 @@ -109,6 +110,6 @@ function recipeSchemaOutput(): RecipeSchemaOutput { } } -function runtimeDescriptorOutput(): RuntimeDescriptor { - return runtimeDescriptor() +async function runtimeDescriptorOutput(): Promise { + return runtimeDescriptor({ nativeMariaDb: await nativeMariaDbHostReadiness() }) } diff --git a/packages/cli/src/commands/recipe-run-artifact-pointers.ts b/packages/cli/src/commands/recipe-run-artifact-pointers.ts index ec2b853d..24dd8511 100644 --- a/packages/cli/src/commands/recipe-run-artifact-pointers.ts +++ b/packages/cli/src/commands/recipe-run-artifact-pointers.ts @@ -4,6 +4,7 @@ import type { ArtifactBundle, RecipeRunSummary, RuntimeInfo } from "@automattic/ import { stripUndefined } from "@automattic/wp-codebox-core/internals" import type { RunOutput } from "../runtime-command-wrappers.js" import type { RecipeArtifactPointerCommandStatus, RecipeArtifactPointerState, RecipeBrowserEvidence, RecipeDiagnosticArtifactRef, RecipePhaseEvidence } from "./recipe-run-types.js" +import type { RuntimeServiceEvidence } from "../runtime-services.js" export class RecipeArtifactPointerTracker { private command: string | undefined @@ -15,6 +16,7 @@ export class RecipeArtifactPointerTracker { private browserEvidence: RecipeBrowserEvidence[] = [] private diagnosticArtifacts: RecipeDiagnosticArtifactRef[] = [] private result: RecipeRunSummary | undefined + private managedRuntimeServices: RuntimeServiceEvidence[] = [] constructor(private readonly directory: string | undefined, private readonly runId: string, private readonly recipePath: string, private readonly startedAt: string) {} @@ -32,6 +34,7 @@ export class RecipeArtifactPointerTracker { this.browserEvidence = state.browserEvidence ?? this.browserEvidence this.diagnosticArtifacts = state.diagnosticArtifacts ?? this.diagnosticArtifacts this.result = state.result ?? this.result + this.managedRuntimeServices = state.managedRuntimeServices ?? this.managedRuntimeServices const pointer = stripUndefined({ schema: "wp-codebox/recipe-run-artifact-pointer/v1", @@ -47,6 +50,7 @@ export class RecipeArtifactPointerTracker { failure: this.failure, failurePhase: recipeArtifactPointerFailurePhase(this.failure, this.phases), result: this.result, + managedRuntimeServices: this.managedRuntimeServices.length > 0 ? this.managedRuntimeServices : undefined, browserEvidence: this.browserEvidence.length > 0 ? this.browserEvidence : undefined, diagnosticArtifacts: this.diagnosticArtifacts.length > 0 ? this.diagnosticArtifacts : undefined, ...await recipeArtifactPointerArtifactState(this.directory, this.runtime, this.artifacts), diff --git a/packages/cli/src/commands/recipe-run-finalizer.ts b/packages/cli/src/commands/recipe-run-finalizer.ts index b416f861..1125ed73 100644 --- a/packages/cli/src/commands/recipe-run-finalizer.ts +++ b/packages/cli/src/commands/recipe-run-finalizer.ts @@ -67,6 +67,7 @@ interface RecipeRunCommonOutputFields { replayStatus?: RecipeRunOutput["replayStatus"] fuzzRun?: RecipeRunOutput["fuzzRun"] provenance?: RecipeRunOutput["provenance"] + managedRuntimeServices?: RecipeRunOutput["managedRuntimeServices"] artifacts?: ArtifactBundle interruption?: RecipeRunOutput["interruption"] } @@ -194,7 +195,7 @@ export async function finalizeCompletedRecipeRun(args: FinalizeCompletedRecipeRu }) as RecipeRunOutput) runRecord = await args.runRegistry.update(args.runRecord.runId, { result: runtimeRunResultFromRecipeSummary(output.result!) }) output.run = runRecord - await args.artifactPointer.update({ commandStatus: args.success ? "completed" : "failed", runtime: args.runtime, artifacts: args.artifacts, failure: args.failure, phases: args.phaseEvidence, stepFailures: args.output.stepFailures, browserEvidence: args.browserEvidence, result: output.result }) + await args.artifactPointer.update({ commandStatus: args.success ? "completed" : "failed", runtime: args.runtime, artifacts: args.artifacts, failure: args.failure, phases: args.phaseEvidence, stepFailures: args.output.stepFailures, browserEvidence: args.browserEvidence, managedRuntimeServices: args.output.managedRuntimeServices, result: output.result }) return output } @@ -220,19 +221,19 @@ export async function finalizeRecoveredRecipeFailure(args: FinalizeRecoveredReci }) as RecipeRunOutput) runRecord = await args.runRegistry.update(args.runRecord.runId, { result: runtimeRunResultFromRecipeSummary(output.result!) }) output.run = runRecord - await args.artifactPointer.update({ commandStatus: "failed", ...(args.runtime ? { runtime: args.runtime } : {}), ...(args.artifacts ? { artifacts: args.artifacts } : {}), failure: args.serializedError, phases: args.phaseEvidence, stepFailures: args.output.stepFailures, browserEvidence: args.browserEvidence, diagnosticArtifacts: args.diagnosticArtifacts, result: output.result }) + await args.artifactPointer.update({ commandStatus: "failed", ...(args.runtime ? { runtime: args.runtime } : {}), ...(args.artifacts ? { artifacts: args.artifacts } : {}), failure: args.serializedError, phases: args.phaseEvidence, stepFailures: args.output.stepFailures, browserEvidence: args.browserEvidence, diagnosticArtifacts: args.diagnosticArtifacts, managedRuntimeServices: args.output.managedRuntimeServices, result: output.result }) return output } -export async function runRecipeCleanup(runRegistry: RuntimeRunRegistry, runRecord: RuntimeRunRecord, cleanup: () => Promise): Promise { +export async function runRecipeCleanup(runRegistry: RuntimeRunRegistry, runRecord: RuntimeRunRecord, cleanup: () => Promise, finalMetadata?: () => Record): Promise { const startedAtMs = Date.now() await runRegistry.update(runRecord.runId, { cleanup: { status: "running" } }) try { await cleanup() - const updatedRunRecord = await runRegistry.update(runRecord.runId, { cleanup: { status: "succeeded" } }) + const updatedRunRecord = await runRegistry.update(runRecord.runId, { cleanup: { status: "succeeded" }, ...(finalMetadata ? { metadata: finalMetadata() } : {}) }) return cleanupEvidenceFromRunRecord(updatedRunRecord, Date.now() - startedAtMs) } catch (error) { - const updatedRunRecord = await runRegistry.update(runRecord.runId, { cleanup: { status: "failed", error: serializeError(error) } }) + const updatedRunRecord = await runRegistry.update(runRecord.runId, { cleanup: { status: "failed", error: serializeError(error) }, ...(finalMetadata ? { metadata: finalMetadata() } : {}) }) const cleanupError = serializeRecipeRunError(error) throw new RunResourceCleanupError( cleanupEvidenceFromRunRecord(updatedRunRecord, Date.now() - startedAtMs, cleanupError), diff --git a/packages/cli/src/commands/recipe-run-types.ts b/packages/cli/src/commands/recipe-run-types.ts index dcb67fe8..f7317630 100644 --- a/packages/cli/src/commands/recipe-run-types.ts +++ b/packages/cli/src/commands/recipe-run-types.ts @@ -5,6 +5,7 @@ import type { RecipeExternalServiceBoundaryHostCorrelation } from "../recipe-ext import type { RecipeValidationIssue, RecipeWorkflowPhase } from "../recipe-validation.js" import type { RunOutput } from "../runtime-command-wrappers.js" import type { RecipeAdversarialCampaignOutput } from "../adversarial-recipe.js" +import type { RuntimeServiceEvidence } from "../runtime-services.js" export interface RecipeRunOptions { recipePath: string @@ -83,6 +84,7 @@ export interface RecipeRunOutput { fuzzRun?: RecipeFuzzRunResult adversarialCampaigns?: RecipeAdversarialCampaignOutput[] provenance?: RecipeRunProvenance + managedRuntimeServices?: RuntimeServiceEvidence[] result?: RecipeRunSummary artifacts?: ArtifactBundle run?: RuntimeRunRecord @@ -263,6 +265,7 @@ export interface RecipeArtifactPointerState { browserEvidence?: RecipeBrowserEvidence[] diagnosticArtifacts?: RecipeDiagnosticArtifactRef[] result?: RecipeRunSummary + managedRuntimeServices?: RuntimeServiceEvidence[] } export interface RecipeEffectiveRecipeArtifact { diff --git a/packages/cli/src/commands/recipe-run.ts b/packages/cli/src/commands/recipe-run.ts index 2d1244f5..e6118334 100644 --- a/packages/cli/src/commands/recipe-run.ts +++ b/packages/cli/src/commands/recipe-run.ts @@ -445,7 +445,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe browserEvidence, replayStatus: evidence.replayStatus ? recipeReplayStatusOutput(evidence.replayStatus) : undefined, failure: recipeFailure, - output: { ...completedRecipeOutputFields({ executions, componentContracts: componentContractResults(recipe, extraPlugins, phaseTracker.list(), executions), stagedFiles: stagedFiles.map(recipeRunStagedFile), fixtureDatabases, siteSeeds, distributionSetupArtifacts, distributionStartupProbes, probes, declaredArtifacts, stepFailures, phaseEvidence: phaseTracker.list(), advisoryFailures, browserEvidence, benchResultsList, fuzzRun: fuzzRunResult, evidence }), ...(adversarialCampaigns.length > 0 ? { adversarialCampaigns } : {}), provenance: recipeRunProvenance(recipe, recipePath) }, + output: { ...completedRecipeOutputFields({ executions, componentContracts: componentContractResults(recipe, extraPlugins, phaseTracker.list(), executions), stagedFiles: stagedFiles.map(recipeRunStagedFile), fixtureDatabases, siteSeeds, distributionSetupArtifacts, distributionStartupProbes, probes, declaredArtifacts, stepFailures, phaseEvidence: phaseTracker.list(), advisoryFailures, browserEvidence, benchResultsList, fuzzRun: fuzzRunResult, evidence }), ...(adversarialCampaigns.length > 0 ? { adversarialCampaigns } : {}), provenance: recipeRunProvenance(recipe, recipePath), managedRuntimeServices: serviceEvidence }, }) } @@ -464,7 +464,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe phaseEvidence: phaseTracker.list(), browserEvidence, replayStatus: evidence.replayStatus ? recipeReplayStatusOutput(evidence.replayStatus) : undefined, - output: { ...completedRecipeOutputFields({ executions, componentContracts: componentContractResults(recipe, extraPlugins, phaseTracker.list(), executions), stagedFiles: stagedFiles.map(recipeRunStagedFile), fixtureDatabases, siteSeeds, distributionSetupArtifacts, distributionStartupProbes, probes, declaredArtifacts, stepFailures, phaseEvidence: phaseTracker.list(), advisoryFailures, browserEvidence, benchResultsList, fuzzRun: fuzzRunResult, evidence }), ...(adversarialCampaigns.length > 0 ? { adversarialCampaigns } : {}), provenance: recipeRunProvenance(recipe, recipePath) }, + output: { ...completedRecipeOutputFields({ executions, componentContracts: componentContractResults(recipe, extraPlugins, phaseTracker.list(), executions), stagedFiles: stagedFiles.map(recipeRunStagedFile), fixtureDatabases, siteSeeds, distributionSetupArtifacts, distributionStartupProbes, probes, declaredArtifacts, stepFailures, phaseEvidence: phaseTracker.list(), advisoryFailures, browserEvidence, benchResultsList, fuzzRun: fuzzRunResult, evidence }), ...(adversarialCampaigns.length > 0 ? { adversarialCampaigns } : {}), provenance: recipeRunProvenance(recipe, recipePath), managedRuntimeServices: serviceEvidence }, }) } catch (error) { const failedServiceEvidence = runtimeServiceEvidenceFromError(error) @@ -592,6 +592,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe ...(adversarialCampaigns.length > 0 ? { adversarialCampaigns } : {}), diagnostics: recipeRuntimeDiagnostics(recipe, executions, error), provenance: recipeRunProvenance(recipe, recipePath, diagnosticArtifacts), + managedRuntimeServices: serviceEvidence, }, }) } finally { @@ -611,16 +612,12 @@ export async function runManagedServiceCleanup( cleanup: () => Promise, ): Promise { try { - return await runRecipeCleanup(runRegistry, runRecord, cleanup) + return await runRecipeCleanup(runRegistry, runRecord, cleanup, () => ({ managedRuntimeServices: serviceEvidence })) } catch (error) { if (preservePrimaryFailure && error instanceof RunResourceCleanupError) { return error.evidence } throw error - } finally { - await runRegistry.update(runRecord.runId, { - metadata: { managedRuntimeServices: serviceEvidence }, - }) } } diff --git a/packages/cli/src/recipe-validation.ts b/packages/cli/src/recipe-validation.ts index 089dcd5f..1e3cb44f 100644 --- a/packages/cli/src/recipe-validation.ts +++ b/packages/cli/src/recipe-validation.ts @@ -760,6 +760,7 @@ export async function validateWorkspaceRecipeSemantics(recipe: WorkspaceRecipe, function validateRecipeRuntimeServices(recipe: WorkspaceRecipe, addIssue: (code: string, path: string, message: string) => void): void { const ids = new Set() const services = recipe.inputs?.services ?? [] + if (services.filter((service) => service.configuration?.provider === "native").length > 2) addIssue("native-runtime-service-budget-exceeded", "$.inputs.services", "Recipes may declare at most two native runtime services.") const exposedEnvironment = new Set([ ...Object.keys(recipe.distribution?.env ?? {}), ...Object.keys(recipe.inputs?.runtimeEnv ?? {}), @@ -805,6 +806,13 @@ function validateRecipeRuntimeServices(recipe: WorkspaceRecipe, addIssue: (code: if (service.configuration[field] !== undefined) addIssue("unsupported-external-runtime-service-option", `${path}.configuration.${field}`, `External MySQL services do not support ${field}.`) } } + if (service.configuration?.provider === "native") { + if (service.kind !== "mysql") addIssue("unsupported-runtime-service-provider", `${path}.configuration.provider`, "The native provider supports only MySQL-compatible services.") + if (service.configuration.engine !== "mariadb") addIssue("unsupported-native-runtime-service-engine", `${path}.configuration.engine`, "The native provider requires engine=mariadb.") + for (const field of ["externalService", "hostEnv", "portEnv", "usernameEnv", "passwordEnv", "image", "rootAuthentication", "foreignKeyTargetPolicy", "responseStatus", "responseBody"] as const) { + if (service.configuration[field] !== undefined) addIssue("unsupported-native-runtime-service-option", `${path}.configuration.${field}`, `Native MariaDB services do not accept ${field}.`) + } + } const supportedOutputs: Record = { mysql: /^(host|port|username|password|database)$/, redis: /^(host|port|url)$/, diff --git a/packages/cli/src/runtime-services.ts b/packages/cli/src/runtime-services.ts index c627ad4a..b218c540 100644 --- a/packages/cli/src/runtime-services.ts +++ b/packages/cli/src/runtime-services.ts @@ -1,11 +1,16 @@ -import { spawn } from "node:child_process" +import { spawn, type ChildProcess } from "node:child_process" import { randomBytes } from "node:crypto" -import { createConnection } from "node:net" +import { access, chmod, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rm, stat, statfs, writeFile } from "node:fs/promises" +import { constants as fsConstants } from "node:fs" +import { createConnection, createServer } from "node:net" +import { tmpdir } from "node:os" +import { dirname, join, resolve } from "node:path" import type { RuntimePolicy, WorkspaceRecipeExternalServiceBoundary, WorkspaceRecipeRuntimeService } from "@automattic/wp-codebox-core" const MYSQL_IMAGES = { mysql: "mysql:8.4", mariadb: "mariadb:11.4" } as const const SERVICE_IMAGES = { redis: "redis:7.4-alpine", smtp: "axllent/mailpit:v1.27", http: "hashicorp/http-echo:1.0" } as const const DEFAULT_PROCESS_OUTPUT_LIMIT_BYTES = 1024 * 1024 +const MAX_NATIVE_RUNTIME_SERVICES = 2 export type RuntimeServiceControlAction = "stop" | "start" | "pause" | "resume" | "restart" | "disconnect" | "reconnect" | "flush" | "read-only" | "read-write" | "latency" @@ -27,11 +32,12 @@ export interface RuntimeServiceEvidence { teardown?: "completed" | "failed" diagnostic?: { code: "readiness-failed" | "provision-failed" | "teardown-failed" | "interrupted" } controls?: RuntimeServiceControlResult[] + memory?: { budgetMiB: number; observedRssMiB?: number } } export class RuntimeServiceProvisionError extends Error { - constructor(message: string, readonly evidence: RuntimeServiceEvidence[]) { - super(message) + constructor(message: string, readonly evidence: RuntimeServiceEvidence[], options?: ErrorOptions) { + super(message, options) this.name = "RuntimeServiceProvisionError" } } @@ -61,6 +67,11 @@ export interface RuntimeServiceDependencies { waitForReady(host: string, port: number, timeoutMs: number, signal?: AbortSignal): Promise randomBytes(size: number): Buffer environment?: Record + nativeBinaryDirectories?: readonly string[] + allocateNativePort?: () => Promise + removeNativeRoot?: (root: string) => Promise + signalNativeProcess?: (child: ChildProcess, signal: NodeJS.Signals) => boolean + verifyNativeFilesystem?: (root: string, datadir: string) => Promise } export interface RuntimeServiceProvider { @@ -114,6 +125,7 @@ export async function provisionRuntimeServices(services: WorkspaceRecipeRuntimeS externalServiceWritesApproved: options.externalServiceWritesApproved ?? false, } try { + if (services.filter((service) => service.configuration?.provider === "native").length > MAX_NATIVE_RUNTIME_SERVICES) throw new Error(`Managed runtime services exceed the native service budget of ${MAX_NATIVE_RUNTIME_SERVICES}`) validateDeclaredRuntimeServiceSecretTargets(services, options.reservedEnvNames ?? []) for (const service of services) { const managed = await runtimeServiceProvider(service).provision(service, dependencies, context, evidence) @@ -203,6 +215,14 @@ const mysqlExternalProvider: RuntimeServiceProvider = { provision: provisionMysqlExternalService, } +const mysqlNativeProvider: RuntimeServiceProvider = { + name: "native", + kind: "mysql", + version: () => "mariadb:native", + secretEnvTargets: mysqlRuntimeServiceSecretTargets, + provision: provisionMysqlNativeService, +} + const redisDockerProvider: RuntimeServiceProvider = { name: "docker", kind: "redis", version: (service) => service.configuration?.image ?? SERVICE_IMAGES.redis, secretEnvTargets: () => ({}), provision: provisionRedisDockerService } const smtpDockerProvider: RuntimeServiceProvider = { name: "docker", kind: "smtp", version: (service) => service.configuration?.image ?? SERVICE_IMAGES.smtp, secretEnvTargets: () => ({}), provision: provisionSmtpDockerService } const httpDockerProvider: RuntimeServiceProvider = { name: "docker", kind: "http", version: (service) => service.configuration?.image ?? SERVICE_IMAGES.http, secretEnvTargets: () => ({}), provision: provisionHttpDockerService } @@ -216,8 +236,12 @@ function mysqlRuntimeServiceSecretTargets(service: WorkspaceRecipeRuntimeService } function runtimeServiceProvider(service: WorkspaceRecipeRuntimeService): RuntimeServiceProvider { - if (service.kind === mysqlDockerProvider.kind) return service.configuration?.provider === "external" ? mysqlExternalProvider : mysqlDockerProvider - if (service.configuration?.provider === "external") throw new Error(`Managed runtime service kind does not support the external provider: ${service.kind}`) + if (service.kind === mysqlDockerProvider.kind) { + if (service.configuration?.provider === "external") return mysqlExternalProvider + if (service.configuration?.provider === "native") return mysqlNativeProvider + return mysqlDockerProvider + } + if (service.configuration?.provider === "external" || service.configuration?.provider === "native") throw new Error(`Managed runtime service kind does not support the ${service.configuration.provider} provider: ${service.kind}`) if (service.kind === redisDockerProvider.kind) return redisDockerProvider if (service.kind === smtpDockerProvider.kind) return smtpDockerProvider if (service.kind === httpDockerProvider.kind) return httpDockerProvider @@ -282,6 +306,758 @@ async function provisionMysqlDockerService(service: WorkspaceRecipeRuntimeServic } } +const NATIVE_MARIADB_ROOT_PREFIX = "wp-codebox-mariadb-" +const NATIVE_MARIADB_START_ATTEMPTS = 5 +const NATIVE_MARIADB_START_TIMEOUT_MS = 30_000 +const NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES = 256 * 1024 +const NATIVE_MARIADB_ADDRESS_SPACE_BYTES = 2 * 1024 * 1024 * 1024 +const NATIVE_MARIADB_FILE_SIZE_BYTES = 128 * 1024 * 1024 +const NATIVE_MARIADB_DATADIR_BYTES = 256 * 1024 * 1024 +const NATIVE_MARIADB_DATADIR_INODES = 4_096 +const NATIVE_MARIADB_CPU_SECONDS = 300 +const NATIVE_MARIADB_OPEN_FILES = 512 +const NATIVE_MARIADB_PROCESSES = 512 +const nativeOwnedProcesses = new Set() + +interface NativeMariaDbBinaries { + server: string + initialize: string + client: string + limiter: string + truncate: string + mkfs: string + fuse: string + unmount: string + version: string +} + +interface NativeMariaDbStorage { + process: OwnedNativeProcess + datadir: string +} + +interface OwnedNativeProcess { + child: ChildProcess + groupId?: number + exited: boolean + exitCode: number | null + signalCode: NodeJS.Signals | null + exit: Promise + identity?: string + identityReady: Promise +} + +interface OwnedNativeRoot { + path: string + device: number + inode: number +} + +async function provisionMysqlNativeService(service: WorkspaceRecipeRuntimeService, dependencies: RuntimeServiceDependencies, context: RuntimeServiceProvisionContext, evidenceList: RuntimeServiceEvidence[]): Promise { + const { signal } = context + const evidence: RuntimeServiceEvidence = { id: service.id, kind: service.kind, provider: "native", version: "mariadb:native", readiness: "pending", lifecycle: "provisioning", controls: [] } + evidenceList.push(evidence) + let root: OwnedNativeRoot | undefined + let processState: OwnedNativeProcess | undefined + let binariesForCleanup: NativeMariaDbBinaries | undefined + let storage: NativeMariaDbStorage | undefined + let adminSocket: string | undefined + let cleaned = false + let cleanupPromise: Promise | undefined + + const performCleanup = async (): Promise => { + if (cleaned || evidence.teardown === "completed") return + try { + if (processState) await stopOwnedNativeMariaDb(processState, root, dependencies, binariesForCleanup, adminSocket) + if (storage && binariesForCleanup) await stopNativeMariaDbStorage(storage, binariesForCleanup, dependencies, root) + if (root) await removeOwnedNativeRoot(root, dependencies) + cleaned = true + evidence.lifecycle = "released" + evidence.teardown = "completed" + delete evidence.diagnostic + } catch (error) { + evidence.lifecycle = "failed" + evidence.teardown = "failed" + evidence.diagnostic = { code: "teardown-failed" } + throw new Error(`Managed runtime service teardown failed: ${service.id}`, { cause: error }) + } + } + const cleanup = (): Promise => { + if (cleanupPromise) return cleanupPromise + cleanupPromise = performCleanup().catch((error) => { + cleanupPromise = undefined + throw error + }) + return cleanupPromise + } + + try { + assertNativeMariaDbConfiguration(service) + assertNativeMariaDbUnprivilegedHost() + throwIfAborted(signal) + const binaries = await resolveNativeMariaDbBinaries(dependencies, signal) + binariesForCleanup = binaries + evidence.version = binaries.version + root = await createOwnedNativeRoot() + const storageRoot = ownedNativePath(root, "storage") + await mkdir(storageRoot, { mode: 0o700 }) + const storageEnvironment = nativeMariaDbEnvironment(root.path) + const userArgument: string[] = [] + storage = await provisionNativeMariaDbStorage(binaries, root, storageRoot, dependencies, storageEnvironment, signal) + const datadir = join(storageRoot, "database") + const runtimeDirectory = join(storageRoot, "runtime") + const temporaryDirectory = join(storageRoot, "tmp") + const pluginDirectory = join(storageRoot, "plugins") + const secureFileDirectory = join(storageRoot, "files") + for (const directory of [datadir, runtimeDirectory, temporaryDirectory, pluginDirectory, secureFileDirectory]) await mkdir(directory, { mode: 0o700 }) + const socket = join(runtimeDirectory, "server.sock") + adminSocket = socket + const pidFile = join(runtimeDirectory, "server.pid") + const logFile = join(runtimeDirectory, "server.log") + const childEnvironment = nativeMariaDbEnvironment(temporaryDirectory) + + await executeOwnedNativeLimited(dependencies, binaries, binaries.initialize, [ + "--no-defaults", + `--datadir=${datadir}`, + "--auth-root-authentication-method=normal", + "--skip-test-db", + "--skip-name-resolve", + "--innodb-buffer-pool-size=32M", + "--innodb-buffer-pool-size-max=32M", + "--key-buffer-size=8M", + "--aria-pagecache-buffer-size=8M", + "--thread-handling=pool-of-threads", + "--thread-pool-size=1", + "--innodb-file-per-table=OFF", + "--innodb-data-file-path=ibdata1:32M:autoextend:max:96M", + "--innodb-temp-data-file-path=ibtmp1:16M:autoextend:max:32M", + "--innodb-log-file-size=32M", + `--open-files-limit=${NATIVE_MARIADB_OPEN_FILES}`, + ...userArgument, + ], { env: childEnvironment, signal, timeout: 60_000, maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }) + await assertOwnedNativeRoot(root) + throwIfAborted(signal) + + let port = 0 + for (let attempt = 0; attempt < NATIVE_MARIADB_START_ATTEMPTS; attempt += 1) { + port = await (dependencies.allocateNativePort ?? allocateLoopbackPort)() + await writeFile(logFile, "", { mode: 0o600 }) + processState = spawnOwnedNativeMariaDb(binaries.limiter, [...nativeMariaDbLimitArguments(), "--", binaries.server, ...nativeMariaDbDaemonArguments({ datadir, socket, pidFile, logFile, temporaryDirectory, pluginDirectory, secureFileDirectory, port, userArgument })], childEnvironment, false, temporaryDirectory) + try { + await waitForNativeMariaDbReady(binaries, socket, processState, root, dependencies, signal) + break + } catch (error) { + await stopOwnedNativeMariaDb(processState, root, dependencies, binaries, socket) + const bindCollision = await nativeMariaDbBindCollision(logFile, processState) + if (!processState.exited) throw new Error("Native MariaDB process exit was not proven") + processState = undefined + if (!bindCollision || attempt === NATIVE_MARIADB_START_ATTEMPTS - 1) throw error + } + } + if (!processState || port === 0) throw new Error("Native MariaDB startup did not establish an owned process") + + const password = dependencies.randomBytes(24).toString("base64url") + const adminArgs = nativeMariaDbSocketArguments(socket) + const executeAdminSql = async (sql: string): Promise<{ stdout: string }> => { + if (!root) throw new Error("Native MariaDB root is unavailable") + await assertOwnedNativeRoot(root) + return await executeNativeLimited(dependencies, binaries, binaries.client, adminArgs, { env: childEnvironment, signal, timeout: 10_000, stdin: sql, maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }) + } + await executeAdminSql([ + "CREATE DATABASE `runtime` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;", + `CREATE USER 'runtime'@'127.0.0.1' IDENTIFIED BY ${quoteMysqlValue(password)};`, + "GRANT ALL PRIVILEGES ON `runtime`.* TO 'runtime'@'127.0.0.1';", + "FLUSH PRIVILEGES;", + "", + ].join("\n")) + const engines = await executeAdminSql("SHOW ENGINES;\n") + assertNativeMariaDbEngines(engines.stdout) + if ((await readdir(pluginDirectory)).length !== 0) throw new Error("Native MariaDB plugin isolation cannot be proven") + await waitForNativeMariaDbRuntimeAccount(binaries, port, password, dependencies, childEnvironment, signal) + throwIfAborted(signal) + + evidence.readiness = "ready" + evidence.lifecycle = "provisioned" + const observedRssMiB = await ownedProcessRssMiB(processState) + evidence.memory = { budgetMiB: NATIVE_MARIADB_ADDRESS_SPACE_BYTES / 1024 / 1024, ...(observedRssMiB !== undefined ? { observedRssMiB } : {}) } + const values: Record = { host: "127.0.0.1", port: String(port), username: "runtime", password, database: "runtime" } + return { + env: runtimeServiceOutputEnvironment(service, values, new Set(["password"])), + secretEnv: service.outputs.password ? { [service.outputs.password]: password } : {}, + secretEnvTargets: mysqlRuntimeServiceSecretTargets(service), + evidence, + async control(action) { + const result: RuntimeServiceControlResult = { serviceId: service.id, action, status: "unsupported", fidelity: "unsupported", reason: `The native ${service.kind} provider does not support ${action}.` } + evidence.controls?.push(result) + return result + }, + release: cleanup, + } + } catch (error) { + evidence.readiness = "failed" + evidence.lifecycle = "failed" + const diagnostic = { code: signal?.aborted ? "interrupted" as const : processState ? "readiness-failed" as const : "provision-failed" as const } + let cleanupError: unknown + try { await cleanup() } catch (caught) { cleanupError = caught } + evidence.readiness = "failed" + evidence.lifecycle = "failed" + evidence.diagnostic = evidence.teardown === "failed" ? { code: "teardown-failed" } : diagnostic + throw new RuntimeServiceProvisionError(`Managed runtime service failed: ${service.id}`, evidenceList, { cause: cleanupError ?? error }) + } +} + +export function assertNativeMariaDbEngines(output: string): void { + const safe = new Set(["aria", "csv", "innodb", "memory", "mrg_myisam", "myisam", "performance_schema", "sequence"]) + const enabled = new Set() + for (const line of output.trim().split(/\r?\n/).filter(Boolean)) { + const [engine, support] = line.split(/\t|\s{2,}/).map((value) => value.trim().toLowerCase()) + if (engine && (support === "yes" || support === "default")) { + enabled.add(engine) + if (!safe.has(engine)) throw new Error("Native MariaDB exposes an outbound-capable or unknown storage engine") + } + } + if (!enabled.has("innodb")) throw new Error("Native MariaDB engine isolation cannot be proven") +} + +export function assertNativeMariaDbUnprivilegedHost(uid = typeof process.getuid === "function" ? process.getuid() : undefined): void { + if (uid === undefined || uid === 0) throw new Error("Native MariaDB requires a provably unprivileged host identity") +} + +export async function nativeMariaDbHostReadiness(dependencies: RuntimeServiceDependencies = defaultDependencies): Promise<{ status: "ready" | "unavailable"; reason?: string }> { + try { + assertNativeMariaDbUnprivilegedHost() + } catch { + return { status: "unavailable", reason: "unprivileged-host-required" } + } + let binaries: NativeMariaDbBinaries + try { + binaries = await resolveNativeMariaDbBinaries(dependencies) + } catch { + return { status: "unavailable", reason: "trusted-containment-tools-unavailable" } + } + let root: OwnedNativeRoot | undefined + let storage: NativeMariaDbStorage | undefined + try { + root = await createOwnedNativeRoot() + const mountpoint = ownedNativePath(root, "storage") + await mkdir(mountpoint, { mode: 0o700 }) + storage = await provisionNativeMariaDbStorage(binaries, root, mountpoint, dependencies, nativeMariaDbEnvironment(root.path)) + await writeFile(join(mountpoint, ".readiness"), "ready", { mode: 0o600 }) + await stopNativeMariaDbStorage(storage, binaries, dependencies, root) + storage = undefined + await removeOwnedNativeRoot(root, dependencies) + root = undefined + return { status: "ready" } + } catch { + try { + if (storage && root) await stopNativeMariaDbStorage(storage, binaries, dependencies, root) + if (root) await removeOwnedNativeRoot(root, dependencies) + } catch { + return { status: "unavailable", reason: "containment-probe-cleanup-failed" } + } + return { status: "unavailable", reason: "bounded-filesystem-unavailable" } + } +} + +function assertNativeMariaDbConfiguration(service: WorkspaceRecipeRuntimeService): void { + const configuration = service.configuration + if (configuration?.provider !== "native" || configuration.engine !== "mariadb") throw new Error("Native MySQL-compatible services require engine=mariadb") + for (const field of ["externalService", "hostEnv", "portEnv", "usernameEnv", "passwordEnv", "image", "rootAuthentication", "foreignKeyTargetPolicy", "responseStatus", "responseBody"] as const) { + if (configuration[field] !== undefined) throw new Error(`Native MariaDB does not accept ${field}`) + } +} + +async function resolveNativeMariaDbBinaries(dependencies: RuntimeServiceDependencies, signal?: AbortSignal): Promise { + const directories = dependencies.nativeBinaryDirectories ?? nativeExecutableDirectories() + const requireTrusted = dependencies.nativeBinaryDirectories === undefined + const server = await findNativeExecutable("mariadbd", directories, requireTrusted) + const initialize = await findNativeExecutable("mariadb-install-db", directories, requireTrusted) + const client = await findNativeExecutable("mariadb", directories, requireTrusted) + const limiter = await findNativeExecutable("prlimit", directories, requireTrusted) + const truncate = await findNativeExecutable("truncate", directories, requireTrusted) + const mkfs = await findNativeExecutable("mkfs.ext4", directories, requireTrusted) + const fuse = await findNativeExecutable("fuse2fs", directories, requireTrusted) + const unmount = await findNativeExecutable("fusermount3", directories, requireTrusted) + if (!server || !initialize || !client || !limiter || !truncate || !mkfs || !fuse || !unmount) throw new Error("Compatible native MariaDB containment tools are unavailable") + const environment = nativeMariaDbEnvironment(tmpdir()) + const [serverVersion, initializerHelp, clientVersion, limiterVersionResult] = await Promise.all([ + dependencies.execute(server, ["--no-defaults", "--version"], { env: environment, signal, timeout: 10_000, maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }), + nativeMariaDbInitializerHelp(initialize, dependencies, environment, signal), + dependencies.execute(client, ["--no-defaults", "--version"], { env: environment, signal, timeout: 10_000, maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }), + dependencies.execute(limiter, ["--version"], { env: environment, signal, timeout: 10_000, maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }), + ]) + const combinedVersions = `${serverVersion.stdout}\n${clientVersion.stdout}` + if (!/MariaDB/i.test(combinedVersions) || !/--auth-root-authentication-method/.test(initializerHelp) || !/--skip-test-db/.test(initializerHelp)) { + throw new Error("Native MariaDB tools do not satisfy the required isolation capabilities") + } + const version = combinedVersions.match(/\b(\d+\.\d+(?:\.\d+)?)-MariaDB\b/i)?.[1] + if (!version) throw new Error("Native MariaDB version identity cannot be proven") + const limiterVersion = limiterVersionResult.stdout + if (!/prlimit from util-linux/i.test(limiterVersion)) throw new Error("Native MariaDB hard resource limiter identity cannot be proven") + await dependencies.execute(limiter, [...nativeMariaDbLimitArguments(), "--", client, "--no-defaults", "--version"], { env: environment, signal, timeout: 10_000, maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }) + await Promise.all([ + dependencies.execute(truncate, ["--version"], { env: environment, signal, timeout: 10_000, maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }), + dependencies.execute(mkfs, ["-V"], { env: environment, signal, timeout: 10_000, maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }), + dependencies.execute(fuse, ["-V"], { env: environment, signal, timeout: 10_000, maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }), + dependencies.execute(unmount, ["--version"], { env: environment, signal, timeout: 10_000, maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }), + ]) + return { server, initialize, client, limiter, truncate, mkfs, fuse, unmount, version: `mariadb:${version}` } +} + +function nativeMariaDbLimitArguments(fileSizeBytes = NATIVE_MARIADB_FILE_SIZE_BYTES): string[] { + return [ + `--as=${NATIVE_MARIADB_ADDRESS_SPACE_BYTES}`, + `--cpu=${NATIVE_MARIADB_CPU_SECONDS}`, + `--fsize=${fileSizeBytes}`, + `--nofile=${NATIVE_MARIADB_OPEN_FILES}`, + `--nproc=${NATIVE_MARIADB_PROCESSES}`, + "--core=0", + "--memlock=0", + ] +} + +async function executeNativeLimited(dependencies: RuntimeServiceDependencies, binaries: NativeMariaDbBinaries, command: string, args: string[], options: Parameters[2]): Promise<{ stdout: string }> { + return await dependencies.execute(binaries.limiter, [...nativeMariaDbLimitArguments(), "--", command, ...args], options) +} + +async function executeOwnedNativeLimited(dependencies: RuntimeServiceDependencies, binaries: NativeMariaDbBinaries, command: string, args: string[], options: Parameters[2]): Promise<{ stdout: string }> { + const state = spawnOwnedNativeMariaDb(binaries.limiter, [...nativeMariaDbLimitArguments(), "--", command, ...args], options.env ?? nativeMariaDbEnvironment("/nonexistent"), true, options.env?.TMPDIR) + const limit = options.maxOutputBytes ?? NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES + const stdout: Buffer[] = [] + const stderr: Buffer[] = [] + let outputBytes = 0 + let overflow = false + const collect = (target: Buffer[], chunk: Buffer): void => { + const remaining = Math.max(0, limit - outputBytes) + if (remaining > 0) target.push(chunk.subarray(0, remaining)) + if (chunk.length > remaining) overflow = true + outputBytes += Math.min(chunk.length, remaining) + } + state.child.stdout?.on("data", (chunk: Buffer) => collect(stdout, chunk)) + state.child.stderr?.on("data", (chunk: Buffer) => collect(stderr, chunk)) + state.child.stdin?.end(options.stdin) + let rejectStop: (error: unknown) => void = () => undefined + const stopFailure = new Promise((_resolve, reject) => { rejectStop = reject }) + let stopPromise: Promise | undefined + const requestStop = (): void => { + if (!stopPromise) { + stopPromise = stopOwnedNativeProcess(state, dependencies) + void stopPromise.catch(rejectStop) + } + } + const abort = () => requestStop() + options.signal?.addEventListener("abort", abort, { once: true }) + let timedOut = false + const timer = setTimeout(() => { timedOut = true; requestStop() }, options.timeout) + try { + await Promise.race([state.exit, stopFailure]) + if (stopPromise) await stopPromise + if (!await waitForOwnedProcessGroupExit(state, 100)) await stopOwnedNativeProcess(state, dependencies) + else nativeOwnedProcesses.delete(state) + const boundedStdout = Buffer.concat(stdout).toString("utf8") + const boundedStderr = Buffer.concat(stderr).toString("utf8") + if (options.signal?.aborted) throw new Error("Managed runtime service provisioning interrupted") + if (timedOut) throw new Error("Native MariaDB initialization timed out") + if (overflow) throw new Error(`Runtime service process output exceeded ${limit} bytes`) + if (state.exitCode !== 0) { + const error = new Error(`Command failed with exit code ${state.exitCode ?? "unknown"}`) as Error & { stdout: string; stderr: string } + error.stdout = boundedStdout + error.stderr = boundedStderr + throw error + } + return { stdout: boundedStdout } + } finally { + clearTimeout(timer) + options.signal?.removeEventListener("abort", abort) + } +} + +async function provisionNativeMariaDbStorage(binaries: NativeMariaDbBinaries, root: OwnedNativeRoot, datadir: string, dependencies: RuntimeServiceDependencies, environment: NodeJS.ProcessEnv, signal?: AbortSignal): Promise { + const image = ownedNativePath(root, "datadir.ext4") + const storageLimits = nativeMariaDbLimitArguments(NATIVE_MARIADB_DATADIR_BYTES) + const uid = process.getuid?.() + const gid = process.getgid?.() + if (uid === undefined || gid === undefined || uid === 0) throw new Error("Native MariaDB bounded filesystem requires an unprivileged POSIX identity") + await dependencies.execute(binaries.limiter, [...storageLimits, "--", binaries.truncate, "--size", String(NATIVE_MARIADB_DATADIR_BYTES), image], { env: environment, signal, timeout: 10_000, maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }) + await dependencies.execute(binaries.limiter, [...storageLimits, "--", binaries.mkfs, "-q", "-F", "-N", String(NATIVE_MARIADB_DATADIR_INODES), "-E", `root_owner=${uid}:${gid}`, image], { env: environment, signal, timeout: 30_000, maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }) + const processState = spawnOwnedNativeMariaDb(binaries.limiter, [...storageLimits, "--", binaries.fuse, "-f", "-o", "rw,nosuid,nodev,noexec", image, datadir], environment) + try { + if (dependencies.verifyNativeFilesystem) await dependencies.verifyNativeFilesystem(root.path, datadir) + else await waitForNativeFilesystemContainment(root, datadir, processState, signal) + return { process: processState, datadir } + } catch (error) { + try { + await unmountNativeMariaDbStorage(datadir, binaries, dependencies, root).catch((unmountError) => { + const stderr = unmountError instanceof Error && "stderr" in unmountError && typeof unmountError.stderr === "string" ? unmountError.stderr : "" + if (!/not mounted|not found|no such file/i.test(stderr)) throw unmountError + }) + await stopOwnedNativeProcess(processState, dependencies) + } catch (stopError) { + throw new Error("Native MariaDB bounded filesystem stop failed", { cause: stopError }) + } + throw error + } +} + +async function waitForNativeFilesystemContainment(root: OwnedNativeRoot, datadir: string, state: OwnedNativeProcess, signal?: AbortSignal): Promise { + const deadline = Date.now() + 10_000 + while (Date.now() < deadline) { + throwIfAborted(signal) + if (state.exited) throw new Error("Native MariaDB bounded filesystem exited before readiness") + try { + const [directory, filesystem] = await Promise.all([stat(datadir), statfs(datadir)]) + const bytes = filesystem.blocks * filesystem.bsize + assertNativeMariaDbFilesystemGeometry(directory.dev, root.device, bytes, filesystem.files) + return + } catch (error) { + if (signal?.aborted) throw error + } + await abortableDelay(25, signal) + } + throw new Error("Native MariaDB bounded filesystem could not be proven") +} + +export function assertNativeMariaDbFilesystemGeometry(device: number, rootDevice: number, bytes: number, inodes: number): void { + if (device === rootDevice || bytes > NATIVE_MARIADB_DATADIR_BYTES || inodes > NATIVE_MARIADB_DATADIR_INODES || bytes < NATIVE_MARIADB_FILE_SIZE_BYTES || inodes < 1_024) { + throw new Error("Native MariaDB bounded filesystem geometry is unavailable") + } +} + +async function stopNativeMariaDbStorage(storage: NativeMariaDbStorage, binaries: NativeMariaDbBinaries, dependencies: RuntimeServiceDependencies, root: OwnedNativeRoot | undefined): Promise { + if (!storage.process.exited) { + if (!root) throw new Error("Native MariaDB storage root is unavailable") + await assertOwnedNativeRoot(root) + await unmountNativeMariaDbStorage(storage.datadir, binaries, dependencies, root) + } + await stopOwnedNativeProcess(storage.process, dependencies) +} + +async function unmountNativeMariaDbStorage(datadir: string, binaries: NativeMariaDbBinaries, dependencies: RuntimeServiceDependencies, root: OwnedNativeRoot): Promise { + for (let attempt = 0; attempt < 20; attempt += 1) { + try { + await executeNativeLimited(dependencies, binaries, binaries.unmount, ["--unmount", datadir], { env: nativeMariaDbEnvironment(root.path), timeout: 10_000, maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }) + return + } catch (error) { + const stderr = error instanceof Error && "stderr" in error && typeof error.stderr === "string" ? error.stderr : "" + if (!/device or resource busy/i.test(stderr) || attempt === 19) throw error + await abortableDelay(50) + } + } +} + +async function stopOwnedNativeProcess(state: OwnedNativeProcess, dependencies: RuntimeServiceDependencies): Promise { + if (!await ownedProcessGroupExists(state)) { nativeOwnedProcesses.delete(state); return } + if (!state.exited) await waitForOwnedProcessExit(state, 5_000) + if (await waitForOwnedProcessGroupExit(state, 100)) { nativeOwnedProcesses.delete(state); return } + await signalOwnedNativeProcess(state, "SIGTERM", dependencies) + if (await waitForOwnedProcessGroupExit(state, 5_000)) { nativeOwnedProcesses.delete(state); return } + await signalOwnedNativeProcess(state, "SIGKILL", dependencies) + if (!await waitForOwnedProcessGroupExit(state, 5_000)) throw new Error("Owned native containment process group did not exit") + nativeOwnedProcesses.delete(state) +} + +async function nativeMariaDbInitializerHelp(initialize: string, dependencies: RuntimeServiceDependencies, environment: NodeJS.ProcessEnv, signal?: AbortSignal): Promise { + try { + return (await dependencies.execute(initialize, ["--no-defaults", "--help"], { env: environment, signal, timeout: 10_000, maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES })).stdout + } catch (error) { + const stdout = error instanceof Error && "stdout" in error && typeof error.stdout === "string" ? error.stdout : "" + if (stdout.startsWith("Usage:") && stdout.includes("mariadb-install-db")) return stdout + throw error + } +} + +function nativeExecutableDirectories(): string[] { + return ["/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"] +} + +async function findNativeExecutable(name: string, directories: readonly string[], requireTrusted = false): Promise { + for (const directory of directories) { + const candidate = resolve(directory, name) + try { + await access(candidate, fsConstants.X_OK) + const canonical = await realpath(candidate) + const metadata = await stat(canonical) + if (metadata.isFile() && (!requireTrusted || await nativeExecutableIsTrusted(canonical))) return canonical + } catch { + // Missing and incompatible candidates are ignored; all required tools must resolve. + } + } + return undefined +} + +async function nativeExecutableIsTrusted(path: string): Promise { + let current = path + while (true) { + const metadata = await stat(current) + if (metadata.uid !== 0 || (metadata.mode & 0o022) !== 0) return false + if (current === "/") return true + current = dirname(current) + } +} + +async function createOwnedNativeRoot(): Promise { + const path = await mkdtemp(join(tmpdir(), NATIVE_MARIADB_ROOT_PREFIX)) + await chmod(path, 0o700) + const canonical = await realpath(path) + if (canonical !== resolve(path)) throw new Error("Native MariaDB root ownership cannot be proven") + const metadata = await lstat(path) + if (!metadata.isDirectory() || metadata.isSymbolicLink()) throw new Error("Native MariaDB root is not a private directory") + return { path, device: metadata.dev, inode: metadata.ino } +} + +function ownedNativePath(root: OwnedNativeRoot, name: string): string { + const path = resolve(root.path, name) + if (!path.startsWith(`${resolve(root.path)}/`)) throw new Error("Native MariaDB path escaped its private root") + return path +} + +async function assertOwnedNativeRoot(root: OwnedNativeRoot): Promise { + const metadata = await lstat(root.path) + if (!metadata.isDirectory() || metadata.isSymbolicLink() || metadata.dev !== root.device || metadata.ino !== root.inode) { + throw new Error("Native MariaDB root ownership changed") + } +} + +function nativeMariaDbEnvironment(privateTemporaryDirectory: string): NodeJS.ProcessEnv { + return { + PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + LANG: "C", + LC_ALL: "C", + HOME: privateTemporaryDirectory, + TMPDIR: privateTemporaryDirectory, + } +} + +function nativeMariaDbDaemonArguments(options: { datadir: string; socket: string; pidFile: string; logFile: string; temporaryDirectory: string; pluginDirectory: string; secureFileDirectory: string; port: number; userArgument: string[] }): string[] { + return [ + "--no-defaults", + `--datadir=${options.datadir}`, + `--socket=${options.socket}`, + `--pid-file=${options.pidFile}`, + `--log-error=${options.logFile}`, + `--tmpdir=${options.temporaryDirectory}`, + `--plugin-dir=${options.pluginDirectory}`, + `--secure-file-priv=${options.secureFileDirectory}`, + "--bind-address=127.0.0.1", + `--port=${options.port}`, + "--skip-name-resolve", + "--skip-log-bin", + "--skip-host-cache", + "--skip-slave-start", + "--skip-symbolic-links", + "--local-infile=OFF", + "--performance-schema=OFF", + "--skip-feedback", + "--innodb-buffer-pool-size=32M", + "--innodb-buffer-pool-size-max=32M", + "--innodb-log-buffer-size=4M", + "--key-buffer-size=8M", + "--aria-pagecache-buffer-size=8M", + "--thread-handling=pool-of-threads", + "--thread-pool-size=1", + "--aria-log-file-size=16M", + "--innodb-file-per-table=OFF", + "--innodb-data-file-path=ibdata1:32M:autoextend:max:96M", + "--innodb-temp-data-file-path=ibtmp1:16M:autoextend:max:32M", + "--innodb-log-file-size=32M", + "--max-connections=10", + "--max-prepared-stmt-count=256", + "--max-session-mem-used=32M", + `--open-files-limit=${NATIVE_MARIADB_OPEN_FILES}`, + "--thread-cache-size=0", + "--table-open-cache=128", + "--table-definition-cache=128", + "--tmp-table-size=8M", + "--max-heap-table-size=8M", + ...options.userArgument, + ] +} + +function nativeMariaDbSocketArguments(socket: string): string[] { + return ["--no-defaults", "--batch", "--skip-column-names", "--protocol=SOCKET", `--socket=${socket}`, "--user=root"] +} + +function spawnOwnedNativeMariaDb(command: string, args: string[], environment: NodeJS.ProcessEnv, captureOutput = false, cwd?: string): OwnedNativeProcess { + const child = spawn(command, args, { cwd, detached: process.platform !== "win32", env: environment, stdio: captureOutput ? ["pipe", "pipe", "pipe"] : "ignore", windowsHide: true }) + const state: OwnedNativeProcess = { child, groupId: child.pid, exited: false, exitCode: null, signalCode: null, exit: Promise.resolve(), identityReady: Promise.resolve() } + nativeOwnedProcesses.add(state) + state.identityReady = captureOwnedProcessIdentity(state) + state.exit = new Promise((resolveExit) => { + child.once("error", () => { state.exited = true; resolveExit() }) + child.once("exit", (code, signal) => { + state.exited = true + state.exitCode = code + state.signalCode = signal + resolveExit() + }) + }) + return state +} + +async function waitForNativeMariaDbReady(binaries: NativeMariaDbBinaries, socket: string, state: OwnedNativeProcess, root: OwnedNativeRoot, dependencies: RuntimeServiceDependencies, signal?: AbortSignal): Promise { + const deadline = Date.now() + NATIVE_MARIADB_START_TIMEOUT_MS + while (Date.now() < deadline) { + throwIfAborted(signal) + await assertOwnedNativeRoot(root) + if (state.exited) throw new Error("Native MariaDB exited before readiness") + try { + await executeNativeLimited(dependencies, binaries, binaries.client, nativeMariaDbSocketArguments(socket), { env: nativeMariaDbEnvironment(root.path), signal, timeout: 2_000, stdin: "SELECT 1;\n", maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }) + if (state.exited) throw new Error("Native MariaDB exited during readiness") + return + } catch (error) { + if (signal?.aborted) throw error + await abortableDelay(100, signal) + } + } + throw new Error("Native MariaDB readiness timed out") +} + +async function waitForNativeMariaDbRuntimeAccount(binaries: NativeMariaDbBinaries, port: number, password: string, dependencies: RuntimeServiceDependencies, environment: NodeJS.ProcessEnv, signal?: AbortSignal): Promise { + const deadline = Date.now() + 10_000 + const args = ["--no-defaults", ...mysqlConnectionArgs("127.0.0.1", port, "runtime"), "--database", "runtime"] + while (Date.now() < deadline) { + throwIfAborted(signal) + try { + await executeNativeLimited(dependencies, binaries, binaries.client, args, { env: { ...environment, MYSQL_PWD: password }, signal, timeout: 2_000, stdin: "SELECT 1;\n", maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }) + return + } catch (error) { + if (signal?.aborted) throw error + await abortableDelay(100, signal) + } + } + throw new Error("Native MariaDB runtime account readiness timed out") +} + +async function nativeMariaDbBindCollision(logFile: string, state: OwnedNativeProcess): Promise { + if (!state.exited) return false + try { + const log = await readFile(logFile, "utf8") + return /address already in use|can't start server.*bind|bind\(\).*error/i.test(log.slice(-64 * 1024)) + } catch { + return false + } +} + +async function allocateLoopbackPort(): Promise { + const server = createServer() + await new Promise((resolveListen, reject) => { + server.once("error", reject) + server.listen(0, "127.0.0.1", resolveListen) + }) + const address = server.address() + if (!address || typeof address === "string") { + server.close() + throw new Error("Unable to allocate a loopback port") + } + await new Promise((resolveClose, reject) => server.close((error) => error ? reject(error) : resolveClose())) + return address.port +} + +async function stopOwnedNativeMariaDb(state: OwnedNativeProcess, root: OwnedNativeRoot | undefined, dependencies: RuntimeServiceDependencies, binaries?: NativeMariaDbBinaries, socket?: string): Promise { + if (!state.exited && root) { + try { + await assertOwnedNativeRoot(root) + if (binaries && socket) await dependencies.execute(binaries.limiter, [...nativeMariaDbLimitArguments(), "--", binaries.client, ...nativeMariaDbSocketArguments(socket)], { env: nativeMariaDbEnvironment(dirname(socket)), timeout: 5_000, stdin: "SHUTDOWN;\n", maxOutputBytes: NATIVE_MARIADB_PROCESS_OUTPUT_LIMIT_BYTES }) + } catch { + // The owned child handle remains the authority when socket shutdown fails. + } + } + await stopOwnedNativeProcess(state, dependencies) +} + +async function captureOwnedProcessIdentity(state: OwnedNativeProcess): Promise { + if (process.platform !== "linux" || !state.child.pid) return + for (let attempt = 0; attempt < 20 && !state.exited; attempt += 1) { + try { + state.identity = await linuxProcessStartIdentity(state.child.pid) + return + } catch { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 5)) + } + } +} + +async function signalOwnedNativeProcess(state: OwnedNativeProcess, signal: NodeJS.Signals, dependencies: RuntimeServiceDependencies): Promise { + await state.identityReady + if (!state.exited && process.platform === "linux") { + if (!state.child.pid || !state.identity || await linuxProcessStartIdentity(state.child.pid) !== state.identity) { + throw new Error("Owned native MariaDB process identity changed") + } + } + let signaled: boolean + if (dependencies.signalNativeProcess) signaled = dependencies.signalNativeProcess(state.child, signal) + else if (process.platform !== "win32" && state.groupId) { + try { process.kill(-state.groupId, signal); signaled = true } catch (error) { signaled = (error as NodeJS.ErrnoException).code === "ESRCH" } + } else signaled = state.child.kill(signal) + if (!signaled && await ownedProcessGroupExists(state)) throw new Error("Owned native MariaDB process group could not be signaled") +} + +async function linuxProcessStartIdentity(pid: number): Promise { + const processStat = await readFile(`/proc/${pid}/stat`, "utf8") + const commandEnd = processStat.lastIndexOf(")") + const fieldsAfterCommand = commandEnd >= 0 ? processStat.slice(commandEnd + 2).trim().split(/\s+/) : [] + const startTime = fieldsAfterCommand[19] + if (!startTime || !/^\d+$/.test(startTime)) throw new Error("Owned process start identity is unavailable") + return `${pid}:${startTime}` +} + +async function waitForOwnedProcessExit(state: OwnedNativeProcess, timeoutMs: number): Promise { + if (state.exited) return true + let timer: NodeJS.Timeout | undefined + await Promise.race([ + state.exit, + new Promise((resolveTimeout) => { timer = setTimeout(resolveTimeout, timeoutMs) }), + ]) + if (timer) clearTimeout(timer) + return state.exited +} + +async function ownedProcessGroupExists(state: OwnedNativeProcess): Promise { + if (process.platform === "win32" || !state.groupId) return !state.exited + try { process.kill(-state.groupId, 0); return true } catch (error) { return (error as NodeJS.ErrnoException).code !== "ESRCH" } +} + +async function waitForOwnedProcessGroupExit(state: OwnedNativeProcess, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (!await ownedProcessGroupExists(state)) return true + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)) + } + return !await ownedProcessGroupExists(state) +} + +async function ownedProcessRssMiB(state: OwnedNativeProcess): Promise { + if (state.exited || !state.child.pid || process.platform !== "linux") return undefined + try { + const status = await readFile(`/proc/${state.child.pid}/status`, "utf8") + const kibibytes = Number(status.match(/^VmRSS:\s+(\d+)\s+kB$/m)?.[1]) + return Number.isFinite(kibibytes) ? Math.ceil(kibibytes / 1024) : undefined + } catch { + return undefined + } +} + +async function removeOwnedNativeRoot(root: OwnedNativeRoot, dependencies: RuntimeServiceDependencies): Promise { + await assertOwnedNativeRoot(root) + await assertNoSymlinks(root.path) + await (dependencies.removeNativeRoot ?? (async (path) => await rm(path, { recursive: true, force: false })))(root.path) + try { + await lstat(root.path) + throw new Error("Native MariaDB private root still exists") + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + } +} + +async function assertNoSymlinks(directory: string): Promise { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name) + const metadata = await lstat(path) + if (metadata.isSymbolicLink()) throw new Error("Native MariaDB private root contains a symlink") + if (metadata.isDirectory()) await assertNoSymlinks(path) + } +} + async function provisionMysqlExternalService(service: WorkspaceRecipeRuntimeService, dependencies: RuntimeServiceDependencies, context: RuntimeServiceProvisionContext, evidenceList: RuntimeServiceEvidence[]): Promise { const { signal } = context const engine = service.configuration?.engine ?? "mysql" diff --git a/packages/runtime-core/src/recipe-run-summary.ts b/packages/runtime-core/src/recipe-run-summary.ts index 43bba4fe..1e167017 100644 --- a/packages/runtime-core/src/recipe-run-summary.ts +++ b/packages/runtime-core/src/recipe-run-summary.ts @@ -110,6 +110,7 @@ export function normalizeRecipeRunSummary(raw: unknown, options: RecipeRunSummar artifact_directory: stringValue(objectValue(result.artifacts).directory) || stringValue(result.artifacts), recipe_path: stringValue(result.recipePath), package_provenance: objectOrUndefined(objectValue(result.provenance).packages), + managed_runtime_services: result.managedRuntimeServices === undefined ? undefined : arrayObjects(result.managedRuntimeServices), }), }) as RecipeRunSummary } diff --git a/packages/runtime-core/src/recipe-schema.ts b/packages/runtime-core/src/recipe-schema.ts index 2a025be8..83e96c2e 100644 --- a/packages/runtime-core/src/recipe-schema.ts +++ b/packages/runtime-core/src/recipe-schema.ts @@ -180,6 +180,11 @@ export function createWorkspaceRecipeJsonSchema(options: WorkspaceRecipeJsonSche type: "array", description: "Disposable host services provisioned before runtime creation. Outputs must be explicitly mapped into runtime environment variable names.", items: { $ref: "#/$defs/runtimeService" }, + allOf: [{ + contains: { type: "object", properties: { configuration: { type: "object", properties: { provider: { const: "native" } }, required: ["provider"] } }, required: ["configuration"] }, + minContains: 0, + maxContains: 2, + }], }, externalServices: { type: "array", @@ -937,7 +942,7 @@ export function createWorkspaceRecipeJsonSchema(options: WorkspaceRecipeJsonSche type: "object", additionalProperties: false, properties: { - provider: { enum: ["docker", "external"] }, + provider: { enum: ["docker", "external", "native"] }, externalService: { type: "string", pattern: "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, engine: { enum: ["mysql", "mariadb"] }, rootAuthentication: { enum: ["generated-password", "empty-password"] }, @@ -953,6 +958,9 @@ export function createWorkspaceRecipeJsonSchema(options: WorkspaceRecipeJsonSche allOf: [{ if: { properties: { provider: { const: "external" } }, required: ["provider"] }, then: { required: ["externalService", "hostEnv", "usernameEnv", "passwordEnv"] }, + }, { + if: { properties: { provider: { const: "native" } }, required: ["provider"] }, + then: { required: ["engine"], properties: { engine: { const: "mariadb" } } }, }], }, outputs: { diff --git a/packages/runtime-core/src/runtime-contract-manifest.ts b/packages/runtime-core/src/runtime-contract-manifest.ts index afe6cd8b..b0334294 100644 --- a/packages/runtime-core/src/runtime-contract-manifest.ts +++ b/packages/runtime-core/src/runtime-contract-manifest.ts @@ -48,6 +48,8 @@ export const CODEBOX_RUN_WORDPRESS_WORKLOAD_ABILITY = "wp-codebox/run-wordpress- export const CODEBOX_RUN_FUZZ_SUITE_ABILITY = "wp-codebox/run-fuzz-suite" as const export const CODEBOX_RESOLVE_RUNTIME_REQUIREMENTS_ABILITY = "wp-codebox/resolve-runtime-requirements" as const export const RUNTIME_DESCRIPTOR_SCHEMA = "wp-codebox/runtime-descriptor/v1" as const +export const RUNTIME_SERVICE_CAPABILITIES_SCHEMA = "wp-codebox/runtime-service-capabilities/v1" as const +export const NATIVE_MARIADB_RUNTIME_SERVICE_CAPABILITY = "runtime-service:mysql:native:mariadb" as const export const CODEBOX_PUBLIC_RUNTIME_CAPABILITIES = [ "agent-task:run", @@ -61,6 +63,11 @@ export const CODEBOX_PUBLIC_RUNTIME_CAPABILITIES = [ "contract-manifest:read", ] as const +export const CODEBOX_PUBLIC_RUNTIME_PACKAGE_CAPABILITIES = [ + ...CODEBOX_PUBLIC_RUNTIME_CAPABILITIES, + NATIVE_MARIADB_RUNTIME_SERVICE_CAPABILITY, +] as const + export const CODEBOX_PUBLIC_RUNTIME_ABILITIES = { agentTask: { run: CODEBOX_RUN_AGENT_TASK_ABILITY, @@ -87,6 +94,10 @@ export const CODEBOX_PUBLIC_RUNTIME_COMMANDS = { } as const export const CODEBOX_PUBLIC_RUNTIME_WORDPRESS_CAPABILITIES = { + runtimeServices: { + schema: RUNTIME_SERVICE_CAPABILITIES_SCHEMA, + packageCapabilities: [NATIVE_MARIADB_RUNTIME_SERVICE_CAPABILITY], + }, wordpressRuntime: { commands: CODEBOX_PUBLIC_RUNTIME_COMMANDS.wordpressRuntime, capabilities: RUNTIME_BACKED_FUZZ_SUITE_RUNNER_CAPABILITIES, @@ -130,6 +141,9 @@ export const RUNTIME_CONTRACT_SCHEMAS = { browserPreviewBootConfig: BROWSER_PREVIEW_BOOT_CONFIG_SCHEMA, runtimeAccess: RUNTIME_ACCESS_SCHEMA, }, + runtimeService: { + capabilities: RUNTIME_SERVICE_CAPABILITIES_SCHEMA, + }, browserSession: { productDto: BROWSER_SESSION_PRODUCT_DTO_SCHEMA, containedSiteStatus: BROWSER_CONTAINED_SITE_STATUS_SCHEMA, @@ -272,7 +286,15 @@ export interface RuntimeDescriptor { publicApi: true contractManifest: true } - capabilities: typeof CODEBOX_PUBLIC_RUNTIME_CAPABILITIES + capabilities: readonly string[] + packageCapabilities: typeof CODEBOX_PUBLIC_RUNTIME_PACKAGE_CAPABILITIES + runtimeServices: { + nativeMariaDb: { + capability: typeof NATIVE_MARIADB_RUNTIME_SERVICE_CAPABILITY + status: "ready" | "unavailable" | "unknown" + reason?: string + } + } abilities: typeof CODEBOX_PUBLIC_RUNTIME_ABILITIES wordpressFuzzRuntimeContract: WordPressFuzzRuntimeContract contractManifest: RuntimeContractManifest @@ -291,7 +313,8 @@ export function runtimeContractManifest(): RuntimeContractManifest { } } -export function runtimeDescriptor(): RuntimeDescriptor { +export function runtimeDescriptor(options: { nativeMariaDb?: { status: "ready" | "unavailable" | "unknown"; reason?: string } } = {}): RuntimeDescriptor { + const nativeMariaDb = options.nativeMariaDb ?? { status: "unknown" as const } return { schema: RUNTIME_DESCRIPTOR_SCHEMA, version: 1, @@ -304,7 +327,11 @@ export function runtimeDescriptor(): RuntimeDescriptor { publicApi: true, contractManifest: true, }, - capabilities: CODEBOX_PUBLIC_RUNTIME_CAPABILITIES, + capabilities: nativeMariaDb.status === "ready" ? [...CODEBOX_PUBLIC_RUNTIME_CAPABILITIES, NATIVE_MARIADB_RUNTIME_SERVICE_CAPABILITY] : CODEBOX_PUBLIC_RUNTIME_CAPABILITIES, + packageCapabilities: CODEBOX_PUBLIC_RUNTIME_PACKAGE_CAPABILITIES, + runtimeServices: { + nativeMariaDb: { capability: NATIVE_MARIADB_RUNTIME_SERVICE_CAPABILITY, ...nativeMariaDb }, + }, abilities: CODEBOX_PUBLIC_RUNTIME_ABILITIES, wordpressFuzzRuntimeContract: wordpressFuzzRuntimeContract(), contractManifest: runtimeContractManifest(), diff --git a/packages/runtime-core/src/runtime-contracts.ts b/packages/runtime-core/src/runtime-contracts.ts index cf5a6609..a93e9b82 100644 --- a/packages/runtime-core/src/runtime-contracts.ts +++ b/packages/runtime-core/src/runtime-contracts.ts @@ -140,7 +140,7 @@ export interface WorkspaceRecipeRuntimeService { id: string kind: "mysql" | "redis" | "smtp" | "http" | (string & {}) configuration?: { - provider?: "docker" | "external" + provider?: "docker" | "external" | "native" externalService?: string engine?: "mysql" | "mariadb" rootAuthentication?: "generated-password" | "empty-password" diff --git a/tests/native-mariadb-runtime-service.integration.test.ts b/tests/native-mariadb-runtime-service.integration.test.ts new file mode 100644 index 00000000..648aca01 --- /dev/null +++ b/tests/native-mariadb-runtime-service.integration.test.ts @@ -0,0 +1,103 @@ +import assert from "node:assert/strict" +import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises" +import { createServer } from "node:net" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { runRecipe } from "../packages/cli/src/commands/recipe-run.ts" +import { executeRuntimeServiceProcess, nativeMariaDbHostReadiness, provisionRuntimeServices } from "../packages/cli/src/runtime-services.ts" +import type { WorkspaceRecipeRuntimeService } from "../packages/runtime-core/src/index.ts" + +const service = (id: string, prefix = "DB"): WorkspaceRecipeRuntimeService => ({ + id, + kind: "mysql", + configuration: { provider: "native", engine: "mariadb" }, + outputs: { host: `${prefix}_HOST`, port: `${prefix}_PORT`, username: `${prefix}_USER`, password: `${prefix}_SECRET`, database: `${prefix}_NAME` }, +}) +const readiness = await nativeMariaDbHostReadiness() +if (readiness.status !== "ready") { + console.log(`native MariaDB runtime service integration skipped: ${readiness.reason ?? "host-unavailable"}`) +} else { +const rootsBefore = new Set((await readdir(tmpdir())).filter((name) => name.startsWith("wp-codebox-mariadb-"))) +const concurrentProvisioning = await Promise.allSettled([provisionRuntimeServices([service("native-one")]), provisionRuntimeServices([service("native-two", "SECOND_DB")])]) +const successfulPeers = concurrentProvisioning.filter((result): result is PromiseFulfilledResult>> => result.status === "fulfilled").map((result) => result.value) +const rejectedPeer = concurrentProvisioning.find((result): result is PromiseRejectedResult => result.status === "rejected") +if (rejectedPeer) { + await Promise.allSettled(successfulPeers.map(async (peer) => await peer.release())) + throw rejectedPeer.reason +} +const [first, second] = successfulPeers +assert.ok(first && second) +try { + assert.notEqual(first.env.DB_PORT, second.env.SECOND_DB_PORT) + const password = first.secretEnv.DB_SECRET ?? "" + assert.ok(password) + const args = ["--no-defaults", "--batch", "--skip-column-names", "--protocol=TCP", "--host=127.0.0.1", `--port=${first.env.DB_PORT}`, "--user=runtime", "--database=runtime"] + const environment = { ...process.env, MYSQL_PWD: password } + const sql = [ + "CREATE TABLE wp_options (option_id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, option_name VARCHAR(191) NOT NULL UNIQUE, option_value LONGTEXT NOT NULL) ENGINE=InnoDB;", + "CREATE TABLE wp_2_options LIKE wp_options;", + "START TRANSACTION; INSERT INTO wp_options(option_name, option_value) VALUES ('rolled_back', 'yes'); ROLLBACK;", + "SELECT COUNT(*) FROM wp_options WHERE option_name='rolled_back';", + "INSERT INTO wp_2_options(option_name, option_value) VALUES ('siteurl', 'https://example.test/network/site-2');", + "SELECT option_value FROM wp_2_options WHERE option_name='siteurl';", + "", + ].join("\n") + const result = await executeRuntimeServiceProcess("mariadb", args, { env: environment, timeout: 20_000, stdin: sql }) + assert.deepEqual(result.stdout.trim().split("\n"), ["0", "https://example.test/network/site-2"]) + + const lockHolder = executeRuntimeServiceProcess("mariadb", args, { env: environment, timeout: 20_000, stdin: "SELECT GET_LOCK('wp_codebox_native_lock', 0); SELECT SLEEP(2);\n" }) + await new Promise((resolve) => setTimeout(resolve, 250)) + const contender = await executeRuntimeServiceProcess("mariadb", args, { env: environment, timeout: 10_000, stdin: "SELECT GET_LOCK('wp_codebox_native_lock', 0);\n" }) + assert.equal(contender.stdout.trim(), "0", "independent sessions preserve MariaDB advisory-lock behavior") + assert.equal((await lockHolder).stdout.trim().split("\n")[0], "1") + + let outboundConnections = 0 + const sentinel = createServer((socket) => { outboundConnections += 1; socket.destroy() }) + await new Promise((resolveListen) => sentinel.listen(0, "127.0.0.1", resolveListen)) + const sentinelAddress = sentinel.address() + assert.ok(sentinelAddress && typeof sentinelAddress !== "string") + const hostileSql = [ + "INSTALL SONAME 'ha_federated';", + `CREATE TABLE hostile_federated (id INT) ENGINE=FEDERATED CONNECTION='mysql://runtime:${password}@127.0.0.1:${sentinelAddress.port}/runtime/source';`, + `CREATE TABLE hostile_connect (id INT) ENGINE=CONNECT TABLE_TYPE=MYSQL CONNECTION='mysql://runtime:${password}@127.0.0.1:${sentinelAddress.port}/runtime/source';`, + "CREATE TABLE hostile_spider (id INT) ENGINE=SPIDER;", + "CREATE TABLE hostile_s3 (id INT) ENGINE=S3;", + ] + for (const statement of hostileSql) await assert.rejects(executeRuntimeServiceProcess("mariadb", args, { env: environment, timeout: 5_000, stdin: `${statement}\n` })) + await new Promise((resolveDelay) => setTimeout(resolveDelay, 100)) + assert.equal(outboundConnections, 0, "disabled engines and plugin loading cannot contact an external SQL sentinel") + await new Promise((resolveClose, rejectClose) => sentinel.close((error) => error ? rejectClose(error) : resolveClose())) + assert.ok((first.evidence[0]?.memory?.observedRssMiB ?? 0) > 0) + assert.ok((first.evidence[0]?.memory?.observedRssMiB ?? 129) <= 128, JSON.stringify(first.evidence[0]?.memory)) + + const recipeRoot = await mkdtemp(join(tmpdir(), "wp-codebox-native-mariadb-recipe-")) + try { + const recipePath = join(recipeRoot, "recipe.json") + const code = "mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT); $db = mysqli_init(); mysqli_real_connect($db, getenv('DB_HOST'), getenv('DB_USER'), getenv('DB_PASSWORD'), getenv('DB_NAME'), (int) getenv('DB_PORT')); $row = mysqli_fetch_assoc(mysqli_query($db, 'SELECT VERSION() AS version')); echo str_contains($row['version'], 'MariaDB') ? 'native-mariadb' : 'wrong-engine';" + await writeFile(recipePath, JSON.stringify({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [service("recipe-native")] }, workflow: { steps: [{ command: "wordpress.run-php", args: [`code=${code}`] }] } })) + const artifactsDirectory = join(recipeRoot, "artifacts") + const recipeResult = await runRecipe({ recipePath, artifactsDirectory, previewHoldBlocking: false, previewLeaseRequested: false, previewLeaseChild: false, timeoutMs: 180_000, json: true, summary: false, dryRun: false }) + assert.equal(recipeResult.success, true, JSON.stringify(recipeResult)) + assert.equal(recipeResult.executions.at(-1)?.stdout.trim(), "native-mariadb") + assert.equal(recipeResult.managedRuntimeServices?.[0]?.lifecycle, "released") + assert.equal(recipeResult.managedRuntimeServices?.[0]?.teardown, "completed") + const resultServices = recipeResult.result?.metadata.managed_runtime_services as Array<{ lifecycle: string; teardown: string }> + assert.equal(resultServices[0]?.lifecycle, "released") + assert.equal(resultServices[0]?.teardown, "completed") + assert.equal((recipeResult.run?.metadata?.managedRuntimeServices as Array<{ lifecycle: string }>)[0]?.lifecycle, "released") + const pointer = JSON.parse(await import("node:fs/promises").then(async ({ readFile }) => await readFile(join(artifactsDirectory, "latest-runtime.json"), "utf8"))) + assert.equal(pointer.managedRuntimeServices[0]?.lifecycle, "released") + assert.equal(pointer.managedRuntimeServices[0]?.teardown, "completed") + assert.equal(JSON.stringify(pointer).includes(password), false) + } finally { + await rm(recipeRoot, { recursive: true, force: true }) + } +} finally { + const releases = await Promise.allSettled([first.release(), second.release()]) + const releaseFailure = releases.find((result): result is PromiseRejectedResult => result.status === "rejected") + if (releaseFailure) throw releaseFailure.reason +} +const leakedRoots = (await readdir(tmpdir())).filter((name) => name.startsWith("wp-codebox-mariadb-") && !rootsBefore.has(name)) +assert.deepEqual(leakedRoots, [], "real native MariaDB integration recursively removes every private root") +console.log("native MariaDB runtime service integration passed") +} diff --git a/tests/native-mariadb-runtime-service.test.ts b/tests/native-mariadb-runtime-service.test.ts new file mode 100644 index 00000000..bcbad063 --- /dev/null +++ b/tests/native-mariadb-runtime-service.test.ts @@ -0,0 +1,264 @@ +import assert from "node:assert/strict" +import { chmod, mkdtemp, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises" +import { createServer } from "node:net" +import { tmpdir } from "node:os" +import { basename, dirname, join } from "node:path" +import { assertNativeMariaDbEngines, assertNativeMariaDbFilesystemGeometry, assertNativeMariaDbUnprivilegedHost, nativeMariaDbHostReadiness, provisionRuntimeServices, RuntimeServiceProvisionError, runtimeServicePlan, type RuntimeServiceDependencies } from "../packages/cli/src/runtime-services.ts" +import { validateWorkspaceRecipeSemantics } from "../packages/cli/src/recipe-validation.ts" +import { validateWorkspaceRecipeJsonSchema, type WorkspaceRecipeRuntimeService } from "../packages/runtime-core/src/index.ts" + +const service: WorkspaceRecipeRuntimeService = { + id: "native-db", + kind: "mysql", + configuration: { provider: "native", engine: "mariadb" }, + outputs: { host: "DB_HOST", port: "DB_PORT", username: "DB_USER", password: "NATIVE_DB_PASSWORD", database: "DB_NAME" }, +} + +assert.equal(validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [service] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }).valid, true) +assert.deepEqual(runtimeServicePlan([service]), [{ id: "native-db", kind: "mysql", provider: "native", version: "mariadb:native", bind: "loopback", port: "ephemeral", persistentVolume: false, configuration: service.configuration, outputs: service.outputs }]) +assert.equal(validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [{ ...service, configuration: { provider: "native", engine: "mysql" } }] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }).valid, false) +const forbiddenConfiguration = { ...service, configuration: { ...service.configuration, hostEnv: "PRODUCTION_DB_HOST", image: "mariadb:latest" } } +const forbiddenIssues = await validateWorkspaceRecipeSemantics({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [forbiddenConfiguration] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }, "recipe.json") +assert.equal(forbiddenIssues.filter((issue) => issue.code === "unsupported-native-runtime-service-option").length, 2) +assert.throws(() => assertNativeMariaDbUnprivilegedHost(0), /unprivileged/) +assert.doesNotThrow(() => assertNativeMariaDbUnprivilegedHost(1000)) +assert.throws(() => assertNativeMariaDbFilesystemGeometry(1, 1, 256 * 1024 * 1024, 4_096), /geometry/) +assert.throws(() => assertNativeMariaDbFilesystemGeometry(2, 1, 257 * 1024 * 1024, 4_096), /geometry/) +assert.throws(() => assertNativeMariaDbFilesystemGeometry(2, 1, 256 * 1024 * 1024, 4_097), /geometry/) +assert.doesNotThrow(() => assertNativeMariaDbFilesystemGeometry(2, 1, 256 * 1024 * 1024, 4_096)) +assert.doesNotThrow(() => assertNativeMariaDbEngines("InnoDB\tDEFAULT\tTransactional\nFEDERATED\tNO\tOutbound\nMEMORY\tYES\tMemory\n")) +assert.throws(() => assertNativeMariaDbEngines("InnoDB\tDEFAULT\tTransactional\nFEDERATED\tYES\tOutbound\n"), /outbound-capable/) +assert.throws(() => assertNativeMariaDbEngines("InnoDB\tNO\tTransactional\n"), /cannot be proven/) +assert.equal(validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [{ ...service, id: "one" }, { ...service, id: "two" }, { ...service, id: "three" }] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }).valid, false) +const budgetIssues = await validateWorkspaceRecipeSemantics({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [{ ...service, id: "one" }, { ...service, id: "two" }, { ...service, id: "three" }] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }, "recipe.json") +assert.equal(budgetIssues.some((issue) => issue.code === "native-runtime-service-budget-exceeded"), true) + +const fixture = await mkdtemp(join(tmpdir(), "wp-codebox-native-provider-fake-")) +const serverScript = `#!/usr/bin/env node +const fs = require('node:fs'); const net = require('node:net'); +if (process.argv.includes('--version')) { console.log('mariadbd Ver 10.11.14-MariaDB'); process.exit(0); } +const value = (name) => process.argv.find((arg) => arg.startsWith(name + '='))?.slice(name.length + 1); +const port = Number(value('--port')); const log = value('--log-error'); const socket = value('--socket'); +fs.writeFileSync(value('--pid-file'), '1'); +fs.writeFileSync(value('--pid-file') + '.actual', String(process.pid)); +const server = net.createServer(); server.on('error', (error) => { fs.appendFileSync(log, error.code === 'EADDRINUSE' ? 'address already in use' : 'daemon crash: ' + error.code); process.exit(1); }); +server.listen(port, '127.0.0.1'); const timer = setInterval(() => { if (fs.existsSync(socket + '.shutdown')) { clearInterval(timer); server.close(() => process.exit(0)); } }, 10); +process.on('SIGTERM', () => server.close(() => process.exit(0))); +` +const limiterScript = `#!/bin/sh +if [ "$1" = "--version" ]; then echo "prlimit from util-linux 2.39.3"; exit 0; fi +for arg in "$@"; do case "$arg" in */mariadbd) printf '%s\n' "$@" > "$HOME/daemon-prlimit-args";; esac; done +while [ "$1" != "--" ] && [ "$#" -gt 0 ]; do shift; done +shift +exec "$@" +` +const fuseScript = `#!/usr/bin/env node +const fs = require('node:fs'); const mount = process.argv.at(-1); +fs.writeFileSync(mount + '/.fuse.actual', String(process.pid)); +const timer = setInterval(() => { if (fs.existsSync(mount + '/.unmount')) { clearInterval(timer); process.exit(0); } }, 10); +process.on('SIGTERM', () => process.exit(0)); +` +const initializerScript = `#!/usr/bin/env node +const fs = require('node:fs'); +if (process.argv.includes('--help')) { console.log('Usage: mariadb-install-db --auth-root-authentication-method --skip-test-db'); process.exit(0); } +fs.writeFileSync(process.env.HOME + '/initializer-args', process.argv.slice(2).join('\\n')); +` +try { + for (const name of ["mariadbd", "mariadb-install-db", "mariadb", "prlimit", "truncate", "mkfs.ext4", "fuse2fs", "fusermount3"]) { + const path = join(fixture, name) + await writeFile(path, name === "mariadbd" ? serverScript : name === "prlimit" ? limiterScript : name === "fuse2fs" ? fuseScript : name === "mariadb-install-db" ? initializerScript : "#!/usr/bin/env node\nsetInterval(() => {}, 1000)\n") + await chmod(path, 0o700) + } + + const calls: Array<{ command: string; args: string[]; stdin?: string; env?: NodeJS.ProcessEnv }> = [] + const dependencies: RuntimeServiceDependencies = { + nativeBinaryDirectories: [fixture], + async verifyNativeFilesystem() {}, + randomBytes: (size) => Buffer.alloc(size, 0x5a), + async waitForReady() {}, + async execute(command, args, options) { + calls.push({ command, args, stdin: options.stdin, env: options.env }) + if (basename(command) === "prlimit" && args[0] === "--version") return { stdout: "prlimit from util-linux 2.39.3\n" } + const separator = basename(command) === "prlimit" ? args.indexOf("--") : -1 + const effectiveCommand = separator >= 0 ? args[separator + 1] ?? command : command + const effectiveArgs = separator >= 0 ? args.slice(separator + 2) : args + if (basename(effectiveCommand) === "mariadbd" && effectiveArgs.includes("--version")) return { stdout: "mariadbd Ver 10.11.14-MariaDB\n" } + if (basename(effectiveCommand) === "mariadb-install-db" && effectiveArgs.includes("--help")) return { stdout: "Usage: mariadb-install-db --auth-root-authentication-method --skip-test-db\n" } + if (basename(effectiveCommand) === "mariadb" && effectiveArgs.includes("--version")) return { stdout: "mariadb Distrib 10.11.14-MariaDB\n" } + if (["truncate", "mkfs.ext4", "fuse2fs", "fusermount3"].includes(basename(effectiveCommand)) && effectiveArgs.some((arg) => arg === "--version" || arg === "-V")) return { stdout: `${basename(effectiveCommand)} fixture\n` } + if (basename(effectiveCommand) === "fusermount3" && effectiveArgs.includes("--unmount")) await writeFile(join(effectiveArgs.at(-1) ?? "", ".unmount"), "") + if (options.stdin === "SHUTDOWN;\n") { + const socket = args.find((arg) => arg.startsWith("--socket="))?.slice("--socket=".length) + assert.ok(socket) + await writeFile(`${socket}.shutdown`, "") + } + if (basename(effectiveCommand) === "mariadb" && options.stdin === "SELECT 1;\n") await new Promise((resolve) => setTimeout(resolve, 200)) + if (options.stdin === "SHOW ENGINES;\n") return { stdout: "InnoDB\tDEFAULT\tTransactional\nFEDERATED\tNO\tOutbound\nMEMORY\tYES\tMemory\n" } + return { stdout: options.stdin === "SELECT 1;\n" ? "1\n" : "" } + }, + } + + assert.deepEqual(await nativeMariaDbHostReadiness(dependencies), { status: "ready" }) + assert.equal(calls.every((call) => call.env?.PATH === "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"), true, "native commands ignore the caller PATH") + + const before = new Set((await readdir(tmpdir())).filter((name) => name.startsWith("wp-codebox-mariadb-"))) + const provisioned = await provisionRuntimeServices([service], { dependencies }) + const password = Buffer.alloc(24, 0x5a).toString("base64url") + assert.equal(provisioned.env.DB_HOST, "127.0.0.1") + assert.equal(provisioned.env.DB_USER, "runtime") + assert.equal(provisioned.env.DB_NAME, "runtime") + assert.equal(provisioned.env.NATIVE_DB_PASSWORD, undefined) + assert.equal(provisioned.secretEnv.NATIVE_DB_PASSWORD, password) + assert.deepEqual(provisioned.secretEnvTargets, { DB_PASSWORD: "NATIVE_DB_PASSWORD" }) + assert.equal(JSON.stringify(provisioned.evidence).includes(password), false) + assert.equal(provisioned.evidence[0]?.provider, "native") + assert.equal(provisioned.evidence[0]?.memory?.budgetMiB, 2048) + const daemonInvocation = await readFile(join((await readdir(tmpdir())).filter((name) => name.startsWith("wp-codebox-mariadb-") && !before.has(name)).map((name) => join(tmpdir(), name))[0] ?? "", "storage", "tmp", "daemon-prlimit-args"), "utf8") + const daemonArgs = daemonInvocation.trim().split("\n") + const datadir = daemonArgs.find((arg) => arg.startsWith("--datadir="))?.slice("--datadir=".length) + assert.ok(datadir) + const initializerArgs = (await readFile(join(dirname(datadir), "tmp", "initializer-args"), "utf8")).trim().split("\n") + assert.ok(initializerArgs.includes("--no-defaults")) + assert.ok(initializerArgs.some((arg) => arg === `--datadir=${datadir}`)) + assert.ok(calls.some((call) => call.args.some((arg) => arg.endsWith("truncate")) && call.args.includes("268435456")), "the provider creates a fixed 256 MiB backing image") + assert.ok(calls.some((call) => call.args.some((arg) => arg.endsWith("mkfs.ext4")) && call.args.includes("-N") && call.args.includes("4096")), "the provider creates a fixed 4096-inode filesystem") + assert.ok(daemonArgs.includes("--as=2147483648")) + assert.ok(daemonArgs.includes("--cpu=300")) + assert.ok(daemonArgs.includes("--fsize=134217728")) + assert.ok(daemonArgs.includes("--nofile=512")) + assert.ok(daemonArgs.includes("--nproc=512")) + assert.ok(daemonArgs.includes(`--plugin-dir=${join(dirname(datadir), "plugins")}`)) + assert.ok(daemonArgs.includes(`--secure-file-priv=${join(dirname(datadir), "files")}`)) + assert.ok(daemonArgs.every((arg) => !arg.startsWith("--socket=") || arg.startsWith(`--socket=${dirname(datadir)}/runtime/`))) + assert.equal(daemonArgs.includes(password), false) + const createUser = calls.find((call) => call.stdin?.includes("CREATE USER")) + assert.ok(createUser?.stdin?.includes(password), "the generated credential is delivered only over client stdin") + assert.equal(createUser?.args.includes(password), false) + await provisioned.release() + await provisioned.release() + const after = (await readdir(tmpdir())).filter((name) => name.startsWith("wp-codebox-mariadb-") && !before.has(name)) + assert.deepEqual(after, [], "successful release recursively removes every provider-owned root") + + const descendantInitializerScript = `#!/usr/bin/env node +const fs = require('node:fs'); const { spawn } = require('node:child_process'); +if (process.argv.includes('--help')) { console.log('Usage: mariadb-install-db --auth-root-authentication-method --skip-test-db'); process.exit(0); } +const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' }); +fs.writeFileSync(process.env.HOME + '/initializer-child-pid', String(child.pid)); child.unref(); +` + await writeFile(join(fixture, "mariadb-install-db"), descendantInitializerScript) + await chmod(join(fixture, "mariadb-install-db"), 0o700) + const descendantProvisioned = await provisionRuntimeServices([{ ...service, id: "initializer-descendant" }], { dependencies }) + const descendantRoot = (await readdir(tmpdir())).find((name) => name.startsWith("wp-codebox-mariadb-") && !before.has(name)) + assert.ok(descendantRoot) + const descendantPid = Number(await readFile(join(tmpdir(), descendantRoot, "storage", "tmp", "initializer-child-pid"), "utf8")) + assert.throws(() => process.kill(descendantPid, 0), (error: unknown) => (error as NodeJS.ErrnoException).code === "ESRCH", "initializer descendants are gone before provisioning continues") + await descendantProvisioned.release() + await writeFile(join(fixture, "mariadb-install-db"), initializerScript) + await chmod(join(fixture, "mariadb-install-db"), 0o700) + + const occupied = createServer() + await new Promise((resolve) => occupied.listen(0, "127.0.0.1", resolve)) + const address = occupied.address() + assert.ok(address && typeof address !== "string") + let allocations = 0 + const collision = await provisionRuntimeServices([service], { dependencies: { ...dependencies, allocateNativePort: async () => ++allocations === 1 ? address.port : 45127 } }) + assert.equal(allocations, 2, "a loopback bind collision allocates a fresh port") + await collision.release() + await new Promise((resolve) => occupied.close(() => resolve())) + + const staleLogPort = createServer() + await new Promise((resolve) => staleLogPort.listen(0, "127.0.0.1", resolve)) + const staleAddress = staleLogPort.address() + assert.ok(staleAddress && typeof staleAddress !== "string") + let staleAllocations = 0 + await assert.rejects(provisionRuntimeServices([service], { dependencies: { ...dependencies, allocateNativePort: async () => ++staleAllocations === 1 ? staleAddress.port : 70_000 } }), RuntimeServiceProvisionError) + assert.equal(staleAllocations, 2, "a stale bind error cannot classify the next daemon crash as retryable") + await new Promise((resolve) => staleLogPort.close(() => resolve())) + + const concurrent = await Promise.all([provisionRuntimeServices([{ ...service, id: "one" }], { dependencies }), provisionRuntimeServices([{ ...service, id: "two" }], { dependencies })]) + assert.notEqual(concurrent[0].env.DB_PORT, concurrent[1].env.DB_PORT) + await Promise.all(concurrent.map(async (managed) => await managed.release())) + + for (const phase of ["preflight", "readiness", "administration", "runtime-account"] as const) { + const phaseCancellation = new AbortController() + if (phase === "preflight") phaseCancellation.abort() + const phaseDependencies: RuntimeServiceDependencies = { ...dependencies, async execute(command, args, options) { + const isSocketReadiness = options.stdin === "SELECT 1;\n" && args.some((arg) => arg === "--protocol=SOCKET") + const isAdministration = options.stdin?.includes("CREATE DATABASE") === true + const isRuntimeAccount = options.stdin === "SELECT 1;\n" && args.some((arg) => arg === "--protocol=TCP") + if ((phase === "readiness" && isSocketReadiness) || (phase === "administration" && isAdministration) || (phase === "runtime-account" && isRuntimeAccount)) phaseCancellation.abort() + return await dependencies.execute(command, args, options) + } } + await assert.rejects(provisionRuntimeServices([{ ...service, id: `cancel-${phase}` }], { dependencies: phaseDependencies, signal: phaseCancellation.signal }), (error: unknown) => error instanceof RuntimeServiceProvisionError && error.evidence[0]?.diagnostic?.code === "interrupted" && error.evidence[0]?.teardown === "completed", `cancellation during ${phase} cleans the private root`) + } + + const missing: RuntimeServiceDependencies = { ...dependencies, nativeBinaryDirectories: [join(fixture, "missing")] } + await assert.rejects(provisionRuntimeServices([service], { dependencies: missing }), RuntimeServiceProvisionError) + assert.deepEqual(await nativeMariaDbHostReadiness({ ...dependencies, nativeBinaryDirectories: [join(fixture, "missing")] }), { status: "unavailable", reason: "trusted-containment-tools-unavailable" }) + + const partialConcurrency = await Promise.allSettled([ + provisionRuntimeServices([{ ...service, id: "partial-success" }], { dependencies }), + provisionRuntimeServices([{ ...service, id: "partial-failure" }], { dependencies: { ...dependencies, nativeBinaryDirectories: [join(fixture, "missing")] } }), + ]) + const partialSuccesses = partialConcurrency.filter((result): result is PromiseFulfilledResult>> => result.status === "fulfilled").map((result) => result.value) + assert.equal(partialSuccesses.length, 1) + await Promise.allSettled(partialSuccesses.map(async (peer) => await peer.release())) + assert.equal(partialConcurrency.some((result) => result.status === "rejected"), true) + + let removalAttempts = 0 + const cleanupFailure = await provisionRuntimeServices([service], { dependencies: { ...dependencies, removeNativeRoot: async (root) => { + removalAttempts += 1 + if (removalAttempts === 1) throw new Error("remove denied") + await rm(root, { recursive: true, force: false }) + } } }) + const concurrentCleanup = await Promise.allSettled([cleanupFailure.release(), cleanupFailure.release()]) + assert.equal(concurrentCleanup.every((result) => result.status === "rejected"), true) + assert.equal(removalAttempts, 1, "concurrent cleanup callers share one teardown attempt") + assert.equal(cleanupFailure.evidence[0]?.teardown, "failed") + await cleanupFailure.release() + assert.equal(removalAttempts, 2) + assert.equal(cleanupFailure.evidence[0]?.teardown, "completed") + assert.equal(cleanupFailure.evidence[0]?.lifecycle, "released") + assert.equal(cleanupFailure.evidence[0]?.diagnostic, undefined) + + let stopRoot: string | undefined + const stopFailureDependencies: RuntimeServiceDependencies = { ...dependencies, signalNativeProcess: () => false, async execute(command, args, options) { + const stopSocket = args.find((arg) => arg.startsWith("--socket="))?.slice("--socket=".length) + if (stopSocket) stopRoot = dirname(dirname(dirname(stopSocket))) + if (options.stdin === "SHUTDOWN;\n") throw new Error("socket shutdown failed") + return await dependencies.execute(command, args, options) + } } + const stopFailure = await provisionRuntimeServices([{ ...service, id: "stop-failure" }], { dependencies: stopFailureDependencies }) + await assert.rejects(stopFailure.release(), /teardown failed/) + assert.equal(stopFailure.evidence[0]?.teardown, "failed") + assert.ok(stopRoot) + const actualPid = Number(await readFile(join(stopRoot, "storage", "runtime", "server.pid.actual"), "utf8")) + const fusePid = Number(await readFile(join(stopRoot, "storage", ".fuse.actual"), "utf8")) + assert.doesNotThrow(() => process.kill(actualPid, 0), "failed stop retains a live owned process instead of clearing its handle") + process.kill(actualPid, "SIGKILL") + process.kill(fusePid, "SIGKILL") + await new Promise((resolve) => setTimeout(resolve, 50)) + await rm(stopRoot, { recursive: true, force: true }) + + let attackedRoot: string | undefined + const outside = join(fixture, "outside-sentinel") + await writeFile(outside, "must survive") + const symlinkDependencies: RuntimeServiceDependencies = { ...dependencies, async execute(command, args, options) { + const result = await dependencies.execute(command, args, options) + const socket = args.find((arg) => arg.startsWith("--socket="))?.slice("--socket=".length) + if (options.stdin?.includes("CREATE DATABASE") && socket) { + const storageRoot = dirname(dirname(socket)) + attackedRoot = dirname(storageRoot) + await symlink(outside, join(storageRoot, "database", "host-path-attack")) + } + return result + } } + const attacked = await provisionRuntimeServices([service], { dependencies: symlinkDependencies }) + await assert.rejects(attacked.release(), /teardown failed/, "cleanup refuses a symlinked private datadir") + assert.equal(await import("node:fs/promises").then(async ({ readFile }) => await readFile(outside, "utf8")), "must survive") + if (attackedRoot) await rm(attackedRoot, { recursive: true, force: true }) +} finally { + await rm(fixture, { recursive: true, force: true }) +} + +console.log("native MariaDB runtime service fake tests passed") diff --git a/tests/release-package-coverage.test.ts b/tests/release-package-coverage.test.ts index 67185a7e..6017d181 100644 --- a/tests/release-package-coverage.test.ts +++ b/tests/release-package-coverage.test.ts @@ -100,6 +100,12 @@ try { const { stdout: version } = await execFileAsync(process.execPath, [cliEntrypoint, "--version"]) assert.match(version, /^\d+\.\d+\.\d+\s*$/) await execFileAsync(process.execPath, [cliEntrypoint, "commands"]) + const descriptorModule = await import(pathToFileURL(join(root, "packages", "runtime-core", "dist", "runtime-contract-manifest.js")).href) as { runtimeDescriptor(): { capabilities: string[]; packageCapabilities: string[]; runtimeServices: { nativeMariaDb: { status: string } }; contractManifest: { capabilities: { runtimeServices: { packageCapabilities: string[] } } } } } + const descriptor = descriptorModule.runtimeDescriptor() + assert.equal(descriptor.capabilities.includes("runtime-service:mysql:native:mariadb"), false, "package support must not claim unknown host readiness") + assert.ok(descriptor.packageCapabilities.includes("runtime-service:mysql:native:mariadb"), "packaged descriptor must declare native MariaDB package support") + assert.equal(descriptor.runtimeServices.nativeMariaDb.status, "unknown") + assert.deepEqual(descriptor.contractManifest.capabilities.runtimeServices.packageCapabilities, ["runtime-service:mysql:native:mariadb"]) const wrapper = join(root, "bin", "wp-codebox") const { stdout: wrapperVersion } = await execFileAsync(wrapper, ["--version"], { diff --git a/tests/runtime-contract-manifest.test.ts b/tests/runtime-contract-manifest.test.ts index 87c29b34..7d588673 100644 --- a/tests/runtime-contract-manifest.test.ts +++ b/tests/runtime-contract-manifest.test.ts @@ -22,6 +22,7 @@ import { BROWSER_PREVIEW_BOOT_CONFIG_SCHEMA, BROWSER_SESSION_PRODUCT_DTO_SCHEMA, CODEBOX_PUBLIC_RUNTIME_CAPABILITIES, + CODEBOX_PUBLIC_RUNTIME_PACKAGE_CAPABILITIES, CODEBOX_PUBLIC_RUNTIME_ABILITIES, CODEBOX_PUBLIC_RUNTIME_COMMANDS, CODEBOX_PUBLIC_RUNTIME_READINESS, @@ -60,6 +61,8 @@ import { RUNTIME_PACKAGE_OUTPUT_PROJECTION_SCHEMA, RUNTIME_PROFILE_SCHEMA, RUNTIME_DESCRIPTOR_SCHEMA, + RUNTIME_SERVICE_CAPABILITIES_SCHEMA, + NATIVE_MARIADB_RUNTIME_SERVICE_CAPABILITY, RUNTIME_RUN_RESULT_SCHEMA, SANDBOX_ISOLATION_PROOF_SCHEMA, TYPED_ARTIFACT_SCHEMA, @@ -94,6 +97,7 @@ assert.equal(manifest.schemas.agentTask.runRequest, AGENT_TASK_RUN_REQUEST_SCHEM assert.equal(manifest.schemas.agentTask.runResult, AGENT_TASK_RUN_RESULT_SCHEMA) assert.equal(manifest.schemas.runtimeBoundary.profile, RUNTIME_PROFILE_SCHEMA) assert.equal(manifest.schemas.runtimeBoundary.previewLease, PREVIEW_LEASE_SCHEMA) +assert.equal(manifest.schemas.runtimeService.capabilities, RUNTIME_SERVICE_CAPABILITIES_SCHEMA) assert.equal(manifest.schemas.runtimeBoundary.browserSessionProductDto, BROWSER_SESSION_PRODUCT_DTO_SCHEMA) assert.equal(manifest.schemas.browserSession.productDto, BROWSER_SESSION_PRODUCT_DTO_SCHEMA) assert.equal(manifest.schemas.browserSession.containedSiteStatus, BROWSER_CONTAINED_SITE_STATUS_SCHEMA) @@ -164,6 +168,7 @@ assert.equal(manifest.capabilities.wordpressRuntime.commands.runWorkload, "run-w assert.equal(manifest.capabilities.wordpressRuntime.commands.runFuzzSuite, "run-fuzz-suite") assert.equal(manifest.capabilities.wordpressRuntime.capabilities.commands?.includes("wordpress.run-workload"), true) assert.equal(manifest.capabilities.wordpressRuntime.runner_modes["runtime-backed"], true) +assert.deepEqual(manifest.capabilities.runtimeServices, { schema: RUNTIME_SERVICE_CAPABILITIES_SCHEMA, packageCapabilities: [NATIVE_MARIADB_RUNTIME_SERVICE_CAPABILITY] }) assert.equal(manifest.readiness.wordpressRuntime.schema, FUZZ_RUNNER_READINESS_SCHEMA) assert.equal(manifest.readiness.wordpressRuntime.status, "ready") assert.equal(manifest.readiness.wordpressRuntime.mode, "runtime-backed") @@ -195,8 +200,14 @@ assert.equal(descriptor.readiness.status, "available") assert.equal(descriptor.readiness.publicApi, true) assert.equal(descriptor.readiness.contractManifest, true) assert.deepEqual(descriptor.capabilities, CODEBOX_PUBLIC_RUNTIME_CAPABILITIES) +assert.deepEqual(descriptor.packageCapabilities, CODEBOX_PUBLIC_RUNTIME_PACKAGE_CAPABILITIES) assert.ok(descriptor.capabilities.includes("runtime-requirements:resolve")) assert.ok(descriptor.capabilities.includes("wordpress-runtime:sandbox-isolation-proof")) +assert.equal(descriptor.capabilities.includes(NATIVE_MARIADB_RUNTIME_SERVICE_CAPABILITY), false) +assert.deepEqual(descriptor.runtimeServices.nativeMariaDb, { capability: NATIVE_MARIADB_RUNTIME_SERVICE_CAPABILITY, status: "unknown" }) +const readyDescriptor = runtimeDescriptor({ nativeMariaDb: { status: "ready" } }) +assert.equal(readyDescriptor.capabilities.includes(NATIVE_MARIADB_RUNTIME_SERVICE_CAPABILITY), true) +assert.deepEqual(readyDescriptor.runtimeServices.nativeMariaDb, { capability: NATIVE_MARIADB_RUNTIME_SERVICE_CAPABILITY, status: "ready" }) assert.deepEqual(descriptor.abilities, CODEBOX_PUBLIC_RUNTIME_ABILITIES) assert.deepEqual(descriptor.contractManifest, manifest) diff --git a/tests/runtime-services-lifecycle.test.ts b/tests/runtime-services-lifecycle.test.ts index 982d35ba..9f708356 100644 --- a/tests/runtime-services-lifecycle.test.ts +++ b/tests/runtime-services-lifecycle.test.ts @@ -13,12 +13,20 @@ try { const succeeded = await registry.create({ runId: "service-success", status: "running", metadata: {} }) const succeededEvidence: RuntimeServiceEvidence[] = [{ id: "mysql", kind: "mysql", provider: "test", version: "test", readiness: "ready", lifecycle: "provisioned" }] + const succeededUpdates: Array<{ cleanup?: string; lifecycle?: string }> = [] + const originalUpdate = registry.update.bind(registry) + registry.update = async (runId, update) => { + succeededUpdates.push({ cleanup: update.cleanup?.status, lifecycle: (update.metadata?.managedRuntimeServices as RuntimeServiceEvidence[] | undefined)?.[0]?.lifecycle }) + return await originalUpdate(runId, update) + } const cleanup = await runManagedServiceCleanup(registry, succeeded, succeededEvidence, false, async () => { succeededEvidence[0]!.lifecycle = "released" succeededEvidence[0]!.teardown = "completed" }) assert.equal(cleanup.state, "completed") assert.deepEqual((await registry.read(succeeded.runId)).metadata.managedRuntimeServices, succeededEvidence) + assert.equal(succeededUpdates.some((update) => update.lifecycle === "provisioned"), false, "final service evidence is never persisted before teardown") + assert.ok(succeededUpdates.some((update) => update.cleanup === "succeeded" && update.lifecycle === "released"), "cleanup completion and final service evidence share one registry update") const failed = await registry.create({ runId: "service-failure", status: "running", metadata: {} }) const failedEvidence: RuntimeServiceEvidence[] = [{ id: "mysql", kind: "mysql", provider: "test", version: "test", readiness: "ready", lifecycle: "provisioned" }] @@ -30,6 +38,7 @@ try { }) assert.equal(preserved.state, "failed", "a primary recipe failure keeps structured cleanup evidence") assert.deepEqual((await registry.read(failed.runId)).metadata.managedRuntimeServices, failedEvidence) + assert.ok(succeededUpdates.some((update) => update.cleanup === "failed" && update.lifecycle === "failed"), "cleanup failure and final service evidence share one registry update") const terminal = await registry.create({ runId: "service-terminal-cleanup-failure", status: "running", metadata: {} }) await assert.rejects(