diff --git a/.gitignore b/.gitignore index 058b468d..3096237c 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ grimoires/ .windsurf/ .claude/ +.vscode/ # Superpowers SDD scratch workspace (briefs, reports, review packages, ledger) .superpowers/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..5232632b --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,185 @@ +# AGENTS.md + +## Purpose + +This repository packages the MuleSoft DataWeave runtime as GraalVM native artifacts. +The language/runtime itself is supplied by pinned `org.mule.weave:*` dependencies; +language implementation changes generally belong in the upstream DataWeave repository. + +- `native-cli`: native `dw` executable. Java/picocli parses arguments; Scala implements commands. +- `native-lib`: native `dwlib` shared library, C ABI, Python binding, and Node binding. +- `native-cli-integration-tests`: downloaded suites executed against the compiled native CLI. +- `benchmarks/`: shared corpus/schema and CLI, JVM, Node, and Python benchmark runners. + +Key entry points are `DWCLI.java`, `NativeRuntime.scala`, `NativeLib.java`, and +`ScriptRuntime.java`. Read the nearest module README before broad changes. + +## Toolchain + +- Use the checked-in `./gradlew` wrapper. +- Native builds require GraalVM Community Java 24 with `native-image`. +- Set `GRAALVM_HOME` and `JAVA_HOME`; native-image tasks need about 6 GB heap. +- Node bindings require Node.js 18+ and a C compiler/node-gyp toolchain. +- Python bindings support Python 3.9+. +- Java source/target compatibility is 17; Scala is 2.12. + +## Build Commands + +Run from the repository root unless a command says otherwise. + +```bash +./gradlew native-cli:test +./gradlew native-lib:test -PskipNodeTests=true -PskipPythonTests=true +./gradlew native-cli:nativeCompile +./gradlew native-lib:nativeCompile +./gradlew build -PskipNodeTests=true +./gradlew native-cli:distro +./gradlew native-lib:buildPythonWheel +./gradlew native-lib:buildNodePackage +./gradlew clean +``` + +Native outputs are `native-cli/build/native/nativeCompile/dw` and +`native-lib/build/native/nativeCompile/dwlib.{dylib,so,dll}`. Useful flags: +`-PskipNodeTests=true`, `-PskipPythonTests=true`, `-PskipStripDebug=true`, and +`-PpythonExe=/path/to/python`. + +## Test Commands + +```bash +./gradlew native-cli:test +# One ScalaTest suite (preferred single-test granularity) +./gradlew native-cli:test --tests "org.mule.weave.dwnative.cli.DataWeaveCLITest" +# One native-lib JUnit class or method; skip unrelated binding lanes +./gradlew native-lib:test --tests "org.mule.weave.lib.ScriptRuntimeTest" \ + -PskipNodeTests=true -PskipPythonTests=true +./gradlew native-lib:test --tests "org.mule.weave.lib.ScriptRuntimeTest.runSimpleScript" \ + -PskipNodeTests=true -PskipPythonTests=true +# Native CLI integration/TCK suite (builds and runs the native binary) +./gradlew -PweaveTestSuiteVersion=2.10.0 -DweaveSuiteVersion=2.10.0 \ + native-cli-integration-tests:test +./gradlew native-cli-integration-tests:test \ + --tests "org.mule.weave.clinative.NativeCliTest" +# Dependency-free benchmark JavaScript tests +./gradlew native-lib:benchmarkJsUnitTest +cd benchmarks && node --test lib/stats.test.mjs +cd benchmarks && node --test --test-name-pattern="computeStats returns" lib/stats.test.mjs +``` + +Node/Vitest commands (run after native staging/build when using integration tests): + +```bash +cd native-lib/node +npm install +npm run build +npm test +npm run test:unit +npm run test:integration +npm test -- tests/unit/result.test.ts +npm test -- tests/integration/dataweave.test.ts -t "basic arithmetic" +``` + +Python binding tests are a custom executable script, not a configured pytest suite: + +```bash +./gradlew native-lib:pythonTest +cd native-lib/python && python3 tests/test_dataweave_module.py +cd benchmarks/runners/python +python3 -m unittest test_bench.TestStats.test_matches_lib_stats_on_1_to_100 +``` + +Node TCK is separate from normal `nodeTest`: run `./gradlew native-lib:stageTckSuites`, +then `cd native-lib/node && npm run test:tck`. + +## Lint and Formatting + +There is no repository-wide lint or formatter command: no Scalafmt, Checkstyle, +Spotless, ESLint, Prettier, Black, or Ruff configuration is checked in. Do not claim a +lint step exists or introduce formatting churn. Match neighboring code. For TypeScript, +`cd native-lib/node && npm run build:ts` is the configured strict type check. + +No `.cursorrules`, `.cursor/rules/`, or `.github/copilot-instructions.md` rules exist. + +## Code Style + +- Preserve local import grouping; use explicit imports and avoid new wildcards. +- Java: four spaces, braces on the declaration line, braced control flow, explicit + types, `PascalCase` classes, `camelCase` members, `UPPER_SNAKE_CASE` constants. +- Scala: two spaces, prefer `val`, idiomatic `foreach { x => ... }`, explicit public + return types, case classes/sealed traits for domain variants, no semicolons. +- TypeScript/ESM: two spaces, double quotes, semicolons, trailing commas in multiline + constructs, `camelCase` values, `PascalCase` types/classes. +- TypeScript is strict and emits CommonJS. Use `node:` built-in imports and `import type`; + local `.ts` imports omit extensions. +- Benchmark `.mjs` is ESM: include `.mjs` extensions and use `import.meta.url`, never + `require` or CommonJS-only assumptions. +- Python: PEP 8 naming, four spaces, type hints on public/boundary APIs, dataclasses for + result/input models, and snake_case public names translated to camelCase wire fields. +- Gradle formatting varies by file; preserve the file's indentation and quote style. +- Add comments/Javadoc/TSDoc for public APIs, ABI contracts, ownership, concurrency, or + non-obvious algorithms. Avoid comments that merely restate code. + +## Types and Errors + +- Model domain outcomes explicitly: Scala case classes/traits, TypeScript interfaces, + Python dataclasses. Use loose JSON/maps only at serialization or FFI boundaries. +- Preserve wire fields exactly: `success`, `result`, `error`, `mimeType`, `charset`, + `binary`, `streamHandle`, `content`, and `properties`. +- Script failures normally return an unsuccessful result envelope. Binding/lifecycle + failures throw `DataWeaveError`; opt-in modes may promote script failures to + `DataWeaveScriptError`. CLI failures need context and a nonzero exit code. +- Include operation context in errors, avoid null-only messages, and escape every string + crossing a manually constructed JSON boundary. +- Use `try/finally` for native allocations, stream registrations, isolate attachment, + and cleanup. Suppress cleanup errors only when they must not mask an earlier failure. +- Never let JavaScript or Python exceptions unwind across C callbacks; translate them to + the documented callback status (`0` success, nonzero/-1 error, `0` read means EOF). + +## Native and FFI Rules + +- Treat exported C names, argument order, callback semantics, nullability, ownership, + and result JSON as a stable ABI. Update Java, C addon, TypeScript, Python, tests, and + documentation together when changing it. +- Every OS thread calling Graal must attach its own isolate thread and detach afterward. + Never reuse an `IsolateThread` from another OS thread. +- Do not capture Graal `Word`/pointer values in Java lambdas; convert to raw addresses and + reconstruct pointers inside explicit workers. +- Copy callback/native buffers before returning; never retain borrowed pointers or use a + Graal-owned C string after `free_cstring`. +- Preserve bounded queues/backpressure and chunk remainders; chunks can exceed the native + 8 KiB callback buffer and do not align with logical records. +- Node worker threads must use N-API thread-safe functions; do not call V8 directly. +- Native-image `buildArgs` are load-bearing; do not simplify initialization, charset, + locale, HTTP, reflection, or resources without native build/runtime coverage. +- Preserve SPI descriptors for `DataFormat` and `ModuleLoader`; `native-lib` deliberately + re-materializes them when creating its fat JAR. Missing entries often appear as + runtime "unknown mime type" failures, not build errors. + +## Tests and Generated Files + +- Scala CLI tests use `AnyFreeSpec`/`Matchers`; exercise picocli parsing and assert exit + code, stdout, and stderr. +- Java library tests use JUnit 5. Node uses Vitest projects (`unit`, `integration`, `tck`). + Benchmark JavaScript uses `node:test`; its Gradle task enumerates files explicitly, so + wire new test files into `native-lib/build.gradle` when they belong in normal testing. +- Streaming changes need chunk integrity, terminal metadata, large-chunk, failure, and + cleanup coverage. Shared benchmark math/schema changes need JS/Python/Scala parity. +- Never edit generated `build/genresource/.../ComponentVersion.scala`, + `build/genjava/.../BenchmarkMode.java`, Graal headers, staged `dwlib.*`, downloaded TCK + suites, benchmark results, or generated benchmark inputs. Change generators/sources. +- Gradle stages native libraries into Python and Node packages; do not commit staged or + packaged artifacts (`node_modules`, `dist`, `native`, wheels, tarballs, results). + +## Benchmarks and CI + +Benchmarks are opt-in: `./gradlew benchmarkCompare -Pbenchmark=true`. Runner tasks carry +`ext.benchmarkRunner = true`; the root aggregator auto-discovers them. New runners must +use the shared corpus/schema and write `benchmarks/results/-.json`. +Do not add benchmark execution to normal `build` or `test`. The CLI benchmark requires a +binary built with `-Pbenchmark=true`; `DW_BENCH_BIN` may select a prebuilt one. + +CI builds Ubuntu and Windows with GraalVM 24. Native CLI regression suites and Node TCK +run only on `master`. Run the smallest relevant suite, then the nearest module test; use +a native build for FFI, SPI, resources, initialization, packaging, or native-image changes. +Keep PRs focused, add regression tests, minimize dependencies, and report vulnerabilities +through the Salesforce portal named in `SECURITY.md`, never through public issues. diff --git a/benchmarks/README.md b/benchmarks/README.md index 67ec233d..3b1dfe3d 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -13,6 +13,12 @@ Language-agnostic benchmark harness for the DataWeave native-lib wrappers. (Scala/Gradle subproject `:benchmarks-engine`, depends on `org.mule.weave:runtime` at the same `weaveVersion` the native image is built from). `runners/python/` is the Python runner (stdlib scripts under `native-lib`, wrapping the same staged `dwlib` as Node). + `runners/cli/` is the CLI runner: a Node parent that spawns the `dw` native + binary (built with `-Pbenchmark=true`, which compiles in an in-binary + benchmark harness gated by `BenchmarkMode.ENABLED` and dispatched via the + `DW_BENCH` env var — the shipped `dw` contains none of it). It emits + `cold-start`, `first-run`, and `warm`; it does **not** emit `streaming` + (the `dw run` path has no chunked-input FFI like the library's). - `report/report.mjs` — joins result files against the manifest and prints a comparison table. - `results/` — gitignored per-run output. @@ -39,6 +45,10 @@ and `JAVA_HOME` set to it (see the root README / `CLAUDE.md`). The pinned build `graalvmVersion` in `gradle.properties`. The **engine runner alone** drives the JVM `DataWeaveScriptingEngine` and runs on any JDK — no native image required. +The **CLI runner** requires the bench-enabled binary +(`./gradlew native-cli:nativeCompile -Pbenchmark=true`); set `DW_BENCH_BIN` to +point at a prebuilt one. Like the library runners it needs the GraalVM toolchain. + ## Running The one-shot cross-runner comparison — runs **every** registered runner and prints the table: @@ -50,6 +60,7 @@ Single-runner options: ./gradlew native-lib:benchmark -Pbenchmark=true # Node only: build wrapper, run, report ./gradlew benchmarks-engine:benchmarkEngine -Pbenchmark=true # engine (JVM) only: writes results/engine-.json ./gradlew native-lib:benchmarkPython -Pbenchmark=true # Python only: writes results/python-.json + ./gradlew native-cli:benchmarkCli -Pbenchmark=true # CLI only: writes results/cli-.json Or directly, once the wrapper is built (`./gradlew native-lib:buildNodePackage`): diff --git a/benchmarks/runners/cli/coldstart.mjs b/benchmarks/runners/cli/coldstart.mjs new file mode 100644 index 00000000..caca5d6e --- /dev/null +++ b/benchmarks/runners/cli/coldstart.mjs @@ -0,0 +1,100 @@ +import { spawn } from "node:child_process"; +import { join } from "node:path"; +import { casesForMetric } from "../../lib/manifest.mjs"; +import { computeStats } from "../../lib/stats.mjs"; +import { locateBinary } from "./locate.mjs"; + +/** Build `--input=name=file\tmime\tcharset` args for a case (absolute paths). */ +function inputArgs(manifest, c) { + const args = []; + for (const [name, inp] of Object.entries(c.inputs ?? {})) { + const file = join(manifest.corpusDir, inp.file); + const charset = inp.charset ?? "utf-8"; + args.push(`--input=${name}=${file}\t${inp.mimeType}\t${charset}`); + } + return args; +} + +/** + * Spawn one fresh dw process in coldfirst mode. Cold-start = wall-clock from just + * before spawn to the child's "READY" marker (process launch + native image load + + * NativeRuntime init). first-run is timed in-process by the child. Rejects on a + * non-zero exit or a missing READY/JSON line so a failed sample never records a + * bogus timing. + */ +function sampleOnce(bin, manifest, c) { + const scriptPath = join(manifest.corpusDir, c.script); + const args = ["--bench-mode=coldfirst", `--script=${scriptPath}`, ...inputArgs(manifest, c)]; + return new Promise((resolve, reject) => { + const t0 = process.hrtime.bigint(); + const child = spawn(bin, args, { + stdio: ["ignore", "pipe", "pipe"], + env: { ...process.env, DW_BENCH: "1" }, + }); + let coldStartMs; + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf-8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + if (coldStartMs === undefined && stdout.includes("READY\n")) { + coldStartMs = Number(process.hrtime.bigint() - t0) / 1e6; + } + }); + child.stderr.setEncoding("utf-8"); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.on("error", reject); + child.on("close", (code) => { + if (code !== 0) { + reject(new Error(`cli coldfirst failed for '${c.id}' (exit ${code})\n${stderr}`)); + return; + } + if (coldStartMs === undefined) { + reject(new Error(`cli coldfirst for '${c.id}' never printed READY\n${stderr}`)); + return; + } + const jsonLine = stdout.split("\n").filter((l) => l && l !== "READY").pop(); + if (!jsonLine) { + reject(new Error(`cli coldfirst for '${c.id}' printed no result line\n${stderr}`)); + return; + } + let firstRunMs; + try { + ({ firstRunMs } = JSON.parse(jsonLine)); + } catch (error) { + reject(new Error(`cli coldfirst for '${c.id}' printed invalid JSON: ${jsonLine}`, { cause: error })); + return; + } + resolve({ coldStartMs, firstRunMs }); + }); + }); +} + +/** @returns {Promise>} */ +export async function runColdStartAndFirstRun(manifest, { samplesOverride } = {}) { + const bin = locateBinary(); + const rows = []; + const ids = new Set([ + ...casesForMetric(manifest, "cold-start").map((c) => c.id), + ...casesForMetric(manifest, "first-run").map((c) => c.id), + ]); + + for (const id of ids) { + const c = manifest.cases.find((x) => x.id === id); + const n = samplesOverride ?? c.iterations?.samples ?? 20; + const colds = []; + const firsts = []; + for (let i = 0; i < n; i++) { + const { coldStartMs, firstRunMs } = await sampleOnce(bin, manifest, c); + colds.push(coldStartMs); + firsts.push(firstRunMs); + } + if (c.metrics.includes("cold-start")) { + rows.push({ id, metric: "cold-start", unit: "ms", stats: computeStats(colds), iterations: n }); + } + if (c.metrics.includes("first-run")) { + rows.push({ id, metric: "first-run", unit: "ms", stats: computeStats(firsts), iterations: n }); + } + } + return rows; +} diff --git a/benchmarks/runners/cli/emit.mjs b/benchmarks/runners/cli/emit.mjs new file mode 100644 index 00000000..699667d0 --- /dev/null +++ b/benchmarks/runners/cli/emit.mjs @@ -0,0 +1,63 @@ +import { writeFileSync, mkdirSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { join, dirname } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { loadManifest, validateResultIds } from "../../lib/manifest.mjs"; +import { gatherEnv } from "../../lib/env.mjs"; +import { locateBinary } from "./locate.mjs"; +import { runColdStartAndFirstRun } from "./coldstart.mjs"; +import { runWarm } from "./warm.mjs"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const CORPUS = join(__dirname, "..", "..", "corpus"); +const RESULTS_DIR = join(__dirname, "..", "..", "results"); + +/** Assemble the full schema object (identical contract to the Node runner). */ +export function buildResult(env, cases) { + return { + schemaVersion: "1.0", + runner: env.runner, + env, + timestamp: new Date().toISOString(), + cases, + }; +} + +/** Best-effort `dw --version` first line; falls back to "dw". */ +function probeVersion(bin) { + try { + const out = execFileSync(bin, ["--version"], { encoding: "utf-8" }); + const line = out.split("\n").map((l) => l.trim()).filter(Boolean)[0]; + return line ? `dw ${line}` : "dw"; + } catch { + return "dw"; + } +} + +export async function main() { + const manifest = loadManifest(CORPUS); + const bin = locateBinary(); + const env = gatherEnv({ runner: "cli", runtimeVersion: probeVersion(bin) }); + // The CLI is a native binary, not the staged dwlib — override the lib fingerprint. + env.dwlibBuildId = "n/a-cli"; + + const coldRows = await runColdStartAndFirstRun(manifest); + const warmRows = await runWarm(manifest); + + const cases = [...coldRows, ...warmRows]; + validateResultIds(manifest, cases); + + mkdirSync(RESULTS_DIR, { recursive: true }); + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const outPath = join(RESULTS_DIR, `cli-${stamp}.json`); + writeFileSync(outPath, JSON.stringify(buildResult(env, cases), null, 2)); + console.log(`wrote ${outPath} (${cases.length} rows)`); + return outPath; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((e) => { + console.error(e.message); + process.exit(1); + }); +} diff --git a/benchmarks/runners/cli/emit.test.mjs b/benchmarks/runners/cli/emit.test.mjs new file mode 100644 index 00000000..88d15870 --- /dev/null +++ b/benchmarks/runners/cli/emit.test.mjs @@ -0,0 +1,27 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { loadManifest, validateResultIds } from "../../lib/manifest.mjs"; +import { buildResult } from "./emit.mjs"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const CORPUS = join(__dirname, "..", "..", "corpus"); + +test("buildResult produces a schema-shaped object with runner 'cli'", () => { + const env = { + runner: "cli", os: "x", cpu: "y", runtimeVersion: "dw vX", + weaveVersion: "2.12.0-x", commit: "abc", dwlibBuildId: "n/a-cli", + }; + const cases = [{ id: "trivial", metric: "cold-start", unit: "ms", stats: { median: 1 }, iterations: 10 }]; + const r = buildResult(env, cases); + assert.equal(r.schemaVersion, "1.0"); + assert.equal(r.runner, "cli"); + assert.ok(typeof r.timestamp === "string"); + assert.deepEqual(r.cases, cases); +}); + +test("orphan ids are rejected before writing", () => { + const manifest = loadManifest(CORPUS); + assert.throws(() => validateResultIds(manifest, [{ id: "totally-made-up" }]), /orphan id/); +}); diff --git a/benchmarks/runners/cli/locate.mjs b/benchmarks/runners/cli/locate.mjs new file mode 100644 index 00000000..d8112077 --- /dev/null +++ b/benchmarks/runners/cli/locate.mjs @@ -0,0 +1,26 @@ +import { existsSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +// benchmarks/runners/cli -> benchmarks/runners -> benchmarks -> repo root +const REPO_ROOT = join(__dirname, "..", "..", ".."); +const BIN_NAME = process.platform === "win32" ? "dw.exe" : "dw"; +const DEFAULT_BIN = join(REPO_ROOT, "native-cli", "build", "native", "nativeCompile", BIN_NAME); + +/** + * Resolve the benchmark-enabled `dw` native binary. Honors DW_BENCH_BIN (absolute + * path to a bench-built dw); otherwise the default nativeCompile output. The binary + * must be built with -Pbenchmark=true so BenchmarkHarness is reachable. + */ +export function locateBinary() { + const candidate = process.env.DW_BENCH_BIN || DEFAULT_BIN; + if (!existsSync(candidate)) { + throw new Error( + `dw benchmark binary not found at ${candidate}. ` + + `Build it with: ./gradlew native-cli:nativeCompile -Pbenchmark=true ` + + `(or set DW_BENCH_BIN to a bench-enabled dw).` + ); + } + return candidate; +} diff --git a/benchmarks/runners/cli/locate.test.mjs b/benchmarks/runners/cli/locate.test.mjs new file mode 100644 index 00000000..8bc8c3fc --- /dev/null +++ b/benchmarks/runners/cli/locate.test.mjs @@ -0,0 +1,23 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { locateBinary } from "./locate.mjs"; + +test("DW_BENCH_BIN override is returned as-is when it exists", () => { + // Point at a file guaranteed to exist: this test file itself. + const self = new URL(import.meta.url).pathname; + process.env.DW_BENCH_BIN = self; + try { + assert.equal(locateBinary(), self); + } finally { + delete process.env.DW_BENCH_BIN; + } +}); + +test("throws an actionable error when the binary is absent", () => { + process.env.DW_BENCH_BIN = "/nonexistent/dw-binary-xyz"; + try { + assert.throws(() => locateBinary(), /nativeCompile|not found|build/i); + } finally { + delete process.env.DW_BENCH_BIN; + } +}); diff --git a/benchmarks/runners/cli/warm.mjs b/benchmarks/runners/cli/warm.mjs new file mode 100644 index 00000000..7de1af9e --- /dev/null +++ b/benchmarks/runners/cli/warm.mjs @@ -0,0 +1,76 @@ +import { spawn } from "node:child_process"; +import { join } from "node:path"; +import { casesForMetric } from "../../lib/manifest.mjs"; +import { computeStats } from "../../lib/stats.mjs"; +import { locateBinary } from "./locate.mjs"; + +function inputArgs(manifest, c) { + const args = []; + for (const [name, inp] of Object.entries(c.inputs ?? {})) { + const file = join(manifest.corpusDir, inp.file); + const charset = inp.charset ?? "utf-8"; + args.push(`--input=${name}=${file}\t${inp.mimeType}\t${charset}`); + } + return args; +} + +/** Spawn dw once in warm mode; resolve the parsed warmMs[] sample array. */ +function warmSamples(bin, manifest, c) { + const scriptPath = join(manifest.corpusDir, c.script); + const warmup = c.iterations?.warmup ?? 10; + const iters = c.iterations?.warm ?? 100; + const args = [ + "--bench-mode=warm", + `--script=${scriptPath}`, + `--warmup=${warmup}`, + `--iters=${iters}`, + ...inputArgs(manifest, c), + ]; + return new Promise((resolve, reject) => { + const child = spawn(bin, args, { + stdio: ["ignore", "pipe", "pipe"], + env: { ...process.env, DW_BENCH: "1" }, + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf-8"); + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.setEncoding("utf-8"); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.on("error", reject); + child.on("close", (code) => { + if (code !== 0) { + reject(new Error(`cli warm failed for '${c.id}' (exit ${code})\n${stderr}`)); + return; + } + const jsonLine = stdout.split("\n").filter((l) => l && l !== "READY").pop(); + if (!jsonLine) { + reject(new Error(`cli warm for '${c.id}' printed no result line\n${stderr}`)); + return; + } + let warmMs; + try { + ({ warmMs } = JSON.parse(jsonLine)); + } catch (error) { + reject(new Error(`cli warm for '${c.id}' printed invalid JSON: ${jsonLine}`, { cause: error })); + return; + } + if (!Array.isArray(warmMs) || warmMs.length === 0) { + reject(new Error(`cli warm for '${c.id}' returned no samples\n${stderr}`)); + return; + } + resolve({ warmMs, iters }); + }); + }); +} + +/** @returns {Promise>} */ +export async function runWarm(manifest) { + const bin = locateBinary(); + const rows = []; + for (const c of casesForMetric(manifest, "warm")) { + const { warmMs, iters } = await warmSamples(bin, manifest, c); + rows.push({ id: c.id, metric: "warm", unit: "ms", stats: computeStats(warmMs), iterations: iters }); + } + return rows; +} diff --git a/benchmarks/runners/node/coldstart.mjs b/benchmarks/runners/node/coldstart.mjs index f6f7354b..9e1a28c4 100644 --- a/benchmarks/runners/node/coldstart.mjs +++ b/benchmarks/runners/node/coldstart.mjs @@ -49,7 +49,13 @@ function sampleOnce(corpusDir, caseId) { reject(new Error(`coldstart child for '${caseId}' printed no result line\n${stderr}`)); return; } - const { firstRunMs } = JSON.parse(jsonLine); + let firstRunMs; + try { + ({ firstRunMs } = JSON.parse(jsonLine)); + } catch (error) { + reject(new Error(`coldstart child for '${caseId}' printed invalid JSON: ${jsonLine}`, { cause: error })); + return; + } resolve({ coldStartMs, firstRunMs }); }); }); diff --git a/docs/superpowers/specs/2026-07-27-cli-benchmark-runner-design.md b/docs/superpowers/specs/2026-07-27-cli-benchmark-runner-design.md new file mode 100644 index 00000000..eb2c615f --- /dev/null +++ b/docs/superpowers/specs/2026-07-27-cli-benchmark-runner-design.md @@ -0,0 +1,239 @@ +# CLI benchmark runner — design + +_2026-07-27_ + +## Goal + +Add a fourth runner to the `benchmarks/` harness that measures the **`dw` native +CLI**. Today the harness compares three runners over a shared corpus: the Node and +Python `native-lib` wrappers (native shared library via FFI) and a JVM engine +baseline (`DataWeaveScriptingEngine`). The CLI runner completes the picture with +the shipped native executable, driving `NativeRuntime` — the CLI's own engine +wrapper — through a GraalVM native image. + +The runner integrates through the harness's existing extension contract: emit a +result file conforming to `schema/result.schema.json` and tag a Gradle task +`ext.benchmarkRunner = true`. The root `benchmarkCompare` auto-discovers it; no +edit to that aggregator, `report.mjs`, or the schema is required. + +## Why a CLI runner is distinct + +The three quadrants of the DataWeave runtime are: + +- **engine** — JVM library (`DataWeaveScriptingEngine` directly). +- **node / python** — native shared library (`dwlib`) via FFI. +- **cli** — native standalone executable (`dw`), driving `NativeRuntime`. + +Because `NativeRuntime` is the CLI's wrapper over the same scripting engine the +`engine` runner drives raw, the **engine-vs-cli delta isolates native-image vs +JVM** for the same logical runtime work — the same reason the README calls the +cross-runner cold-start comparison meaningful (the native image has no JVM to +boot). + +## Key decisions + +These were settled during brainstorming; recorded here so the rationale is not lost. + +1. **Measure the native binary, not a JVM entrypoint.** A JVM entrypoint runner + would heavily overlap the existing `engine` runner and would not measure the + shipped artifact. The native binary is what users actually run. + +2. **The binary emits a `READY` marker in benchmark mode.** Rather than treat the + binary as a black box (which would force one whole-process number per + invocation and lose the init/exec split), we add a benchmark harness *inside* + `native-cli` that prints `READY` the instant `NativeRuntime` is constructed. + This reuses the exact parent/child protocol the Node/Python/engine runners + already use, yielding a true `cold-start` **and** a like-for-like `first-run` + **and** a `warm` steady-state loop — all from the real native binary. + +3. **Build-time gate, stripped from production.** The benchmark harness must not + ship in the production `dw`. This is a security-sensitive CLI (`--untrusted`, + `--privileges`); an always-reachable alternate entrypoint is both a footgun (a + stray `DW_BENCH` env var could hijack a normal invocation) and unwanted + surface. A generated `BenchmarkMode.ENABLED` constant is `false` in normal + builds and `true` only under `-Pbenchmark=true`. Native-image reachability + analysis folds the harness away as dead code when `ENABLED` is a compile-time + `false`, so the shipped `dw` contains no benchmark code and no consumer can + reach it. Cost: the benchmark build performs its own `nativeCompile`. + +4. **Corpus only.** The runner measures the shared corpus, exactly like the other + three runners — no ad-hoc user-script mode, no arbitrary-subcommand timing. + (Arbitrary subcommands like `dw spell`/`dw validate` cannot reuse the + cold/first split anyway: init is entangled inside each command, so there is no + single honest "runtime ready" boundary to mark. Timing those would be a + separate whole-process tool with a non-comparable metric — explicitly out of + scope.) + +5. **No `streaming` metric.** The `dw run` path writes a whole output stream and + has no chunked-input FFI like the library's `runTransform`. The CLI emits + `cold-start`, `first-run`, and `warm` only. The report renders absent cells as + `—`, so this needs zero report or schema changes — it is a documented, + honest gap, matching how each runner declares only the metrics it can produce. + +## Architecture + +Parent/child split mirroring the Node runner, but **the child is the `dw` binary +itself** running in a build-gated benchmark mode. + +### Child — benchmark harness inside `native-cli` + +- `DWCLI.main` checks, before any picocli parsing or banner output: + `if (BenchmarkMode.ENABLED && ) → BenchmarkHarness.main(args)`. + In a production build `BenchmarkMode.ENABLED` is a compile-time `false`, so the + entire branch (and `BenchmarkHarness`) is unreachable and tree-shaken out of the + native image. +- `BenchmarkHarness` is **corpus-agnostic**: it knows nothing about + `manifest.json`. The parent passes it a script file, input files with explicit + mime/charset, a mode, and iteration counts. This keeps `native-cli` free of + benchmark-corpus knowledge and keeps the (gated) footprint minimal. +- The harness constructs **one** `NativeRuntime` and drives it through + `NativeRuntime.run(script, "bench", inputs, out)`. It reuses the Scala building + blocks already present in the engine runner package where practical + (`CountingOutputStream`, the "READY then single JSON line" pattern). + +Invocation shape (args produced by the parent): + +``` +DW_BENCH=1 dw --bench-mode=coldfirst \ + --script= \ + --input=payload=:application/json:utf-8 \ + [--warmup=N --iters=M] +``` + +(The exact env-var name and flag spelling are an implementation detail; a +specific, collision-unlikely name is used. `ENABLED` is what actually gates +reachability — the env var only selects mode within an already-bench-enabled +binary.) + +#### Mode `coldfirst` — one fresh process per sample + +The parent spawns this N times; each process yields one cold-start + one first-run. + +1. Read script + input files into memory. +2. Construct one `NativeRuntime` (the init being measured from outside). +3. Print `READY\n` and **flush immediately**, before any banner/logging can + interleave. The parent stamps `cold-start` = wall-clock from spawn to this + marker (process launch + image load + runtime init). +4. `nowNs()`; `runtime.run(script, "bench", inputs, CountingOutputStream)`; + measure `first-run` in-process. +5. Assert success. On failure: non-zero exit + stderr, so a bad sample never + records a bogus timing (same contract as `runners/node/coldstart.mjs`). +6. Print one JSON line: `{"firstRunMs": }`. + +#### Mode `warm` — single process, in-process loop + +Mirrors `runners/node/warm-bench.mjs`. + +1. Construct one `NativeRuntime`, print `READY`. +2. `warmup` unmeasured `run()` calls, then `iters` measured `run()` calls. +3. Print `{"warmMs": []}`. The parent computes stats via the shared + `computeStats`, so the stats path is identical across runners. + +#### Output discipline + +`dw` prints a banner and can log to stdout. The harness runs with logging silenced +and writes transformation output to a `CountingOutputStream` (never real stdout), +so the only stdout is `READY` + the JSON line. The parent reads the **last** JSON +line and tolerates stray lines (the engine parent already does this). + +### Parent — `benchmarks/runners/cli/` (Node) + +Mirrors `runners/node/` file layout and reuses the shared libs +(`lib/manifest.mjs`, `lib/stats.mjs`, `lib/env.mjs`) verbatim. + +- **`locate.mjs`** — resolves the **benchmark-enabled** `dw` binary. Env override + `DW_BENCH_BIN=/abs/path/to/dw`, else the default native build output + `native-cli/build/native/nativeCompile/dw{,.exe}`. Fails fast with a + "build the bench binary (`./gradlew native-cli:nativeCompile -Pbenchmark=true`)" + message if absent (same shape as `runners/node/wrapper.mjs`). +- **`coldstart.mjs`** — spawns `dw` in `coldfirst` mode per sample; stamps + cold-start at spawn→`READY`, parses `firstRunMs`. Structurally + `runners/node/coldstart.mjs` with the child command swapped from + `node coldstart-child.mjs` to `dw --bench-mode=coldfirst …`. Same + reject-on-failure contract. +- **`warm.mjs`** — spawns `dw` once per warm case in `warm` mode, reads the + `warmMs[]` array, runs it through the shared `computeStats`. +- **`emit.mjs`** — entrypoint. `gatherEnv({ runner: "cli", runtimeVersion: + "dw " })`, run cold+first then warm, `validateResultIds`, write + `results/cli-.json`. Same structure as `runners/node/emit.mjs`. + +### Metrics emitted + +Per case, gated by that case's declared `metrics[]` in `corpus/manifest.json` +(no manifest changes needed). + +| metric | how measured | comparable to | +|---|---|---| +| `cold-start` | spawn → `READY` (launch + image load + `NativeRuntime` init) | node/python/engine cold-start — **headline: native binary vs JVM** | +| `first-run` | in-process compile+exec, first call | node/python/engine first-run — like-for-like (both exclude launch) | +| `warm` | in-process steady-state loop | node/python/engine warm | +| `streaming` | not emitted | — (documented gap) | + +Env fields: `runtimeVersion` from `dw --version`; `dwlibBuildId = "n/a-cli"` +(following the engine runner's `"n/a-engine"` convention — the CLI is a binary, +not the staged `dwlib`). + +## Gradle wiring + +In `native-cli/build.gradle`: + +- **`genBenchmarkMode`** — generates `BenchmarkMode.java` (constant `ENABLED`) + into `build/genresource/`, following the existing `genVersions` / + `ComponentVersion` pattern. `ENABLED = true` only when `-Pbenchmark=true`, else + `false`. Wired onto the compile classpath. +- The benchmark-enabled native image reuses `nativeCompile` invoked with + `-Pbenchmark=true` (the flag flows into `genBenchmarkMode`). Normal `build`/CI + never sets it, so the shipped `dw` is built with `ENABLED=false` and the harness + is tree-shaken out. +- **`benchmarkCli`** — `onlyIf { benchmark==true }`, `ext.benchmarkRunner = true`, + `dependsOn nativeCompile` and the shared `genBenchInputs`; runs + `node corpus/gen-inputs.mjs && node runners/cli/emit.mjs`. It does **not** + render the report. The root `benchmarkCompare` auto-discovers it via the + `benchmarkRunner` tag — no edit to that task, per the README "Adding a runner" + contract. + +## Error handling + +Same fail-fast contract as `coldstart.mjs` / `EngineChild`: + +- Child non-zero exit → parent rejects with stderr. +- Missing `READY` marker → reject. +- Missing JSON line → reject. +- `result.success == false` inside the harness → non-zero exit + stderr. + +A bad sample never records a bogus timing. `locate.mjs` fails with a clear +build-the-bench-binary message when the artifact is absent. + +## Testing + +Follows the repo's parity-guard split: dwlib/binary-free tests run in the normal +`test`; binary-dependent tests are benchmark-only. + +- **Scala** (`native-cli:test`, scalatest `AnyFreeSpec`/`Matchers`): + - `BenchmarkHarness` arg parsing (script / input / mode / iters). + - `coldfirst` and `warm` each emit exactly `READY` + one JSON line. + - Transformation output goes to the counting stream, not stdout. + - Failure path → non-zero exit + stderr. + - Guard: `BenchmarkMode.ENABLED` is `false` in a normal build. +- **JS** (`benchmarkJsUnitTest`, dwlib-free, always-on parity set): + - `locate.mjs` resolution + `DW_BENCH_BIN` override. + - `emit.mjs` result-object builder. + - Parent parsing of child stdout, using fixtures / a fake child — no real + binary. +- **Smoke** (benchmark-only, needs the bench binary): one real `coldfirst` spawn + on the `trivial` case asserting a positive cold-start and a `firstRunMs`. + +## Out of scope + +- Ad-hoc user-supplied script benchmarking. +- Timing arbitrary `dw` subcommands (`spell`, `validate`, `wizard`, `repl`). +- `streaming` metric for the CLI. +- Any change to `report.mjs`, `schema/result.schema.json`, `benchmarkCompare`, or + the corpus manifest. + +## Documentation + +Update `benchmarks/README.md`: add `runners/cli/` to the layout, note the +build-gated bench binary and `-Pbenchmark=true` requirement, and record that the +CLI emits `cold-start`/`first-run`/`warm` (no `streaming`). Add the single-runner +invocation (`./gradlew native-cli:benchmarkCli -Pbenchmark=true`). diff --git a/native-cli/build.gradle b/native-cli/build.gradle index 5ce3b5e1..68c3855d 100644 --- a/native-cli/build.gradle +++ b/native-cli/build.gradle @@ -9,6 +9,9 @@ sourceSets { scala { srcDirs = ['src/main/scala', 'build/genresource'] } + java { + srcDirs += 'build/genjava' + } } } @@ -66,11 +69,38 @@ task genVersions() { outputPrinter.close() } +def genJavaDirectory = new File("$project.buildDir/genjava") + +task genBenchmarkMode() { + def enabled = project.findProperty('benchmark')?.toString()?.toBoolean() == true + def benchmarkMode = new File(genJavaDirectory, "org/mule/weave/cli/BenchmarkMode.java") + def parentFile = benchmarkMode.getParentFile() + if (!parentFile.exists()) { + parentFile.mkdirs() + } + final PrintWriter outputPrinter = new PrintWriter(new FileWriter(benchmarkMode)) + outputPrinter.println("package org.mule.weave.cli;") + outputPrinter.println() + outputPrinter.println("// GENERATED by genBenchmarkMode — do not edit.") + outputPrinter.println("// ENABLED is true only when built with -Pbenchmark=true; native-image") + outputPrinter.println("// folds the benchmark branch away as dead code when this is false.") + outputPrinter.println("public final class BenchmarkMode {") + outputPrinter.println(" private BenchmarkMode() {}") + outputPrinter.println(" public static final boolean ENABLED = " + enabled + ";") + outputPrinter.println("}") + outputPrinter.close() +} + defaultTasks += genVersions compileScala { dependsOn genVersions + dependsOn genBenchmarkMode +} + +compileJava { + dependsOn genBenchmarkMode } // Merging Service Files @@ -157,4 +187,24 @@ tasks.compileJava.classpath += files(sourceSets.main.scala.classesDirectory) //tasks.named("nativeCompile") { // classpathJar = file("${buildDir}/libs/native-cli-" + nativeVersion + "-all.jar") -//} \ No newline at end of file +//} + +// The CLI runner as an aggregator-registered runner: emits its result file but +// does NOT render the report (the root :benchmarkCompare renders once over all +// runners). Tagged `benchmarkRunner` so :benchmarkCompare discovers it automatically. +// Requires the bench-enabled binary — nativeCompile must run with -Pbenchmark=true so +// BenchmarkMode.ENABLED is true and BenchmarkHarness is reachable in dw. +tasks.register('benchmarkCli', Exec) { + onlyIf { project.findProperty('benchmark')?.toString()?.toBoolean() == true } + ext.benchmarkRunner = true + + dependsOn tasks.named('nativeCompile') + workingDir("${rootDir}/benchmarks") + + def script = 'node corpus/gen-inputs.mjs && node runners/cli/emit.mjs' + if (System.getProperty('os.name').toLowerCase().contains('windows')) { + commandLine('cmd', '/c', script) + } else { + commandLine('bash', '-c', script) + } +} \ No newline at end of file diff --git a/native-cli/src/main/java/org/mule/weave/cli/DWCLI.java b/native-cli/src/main/java/org/mule/weave/cli/DWCLI.java index ee3bd421..008e8a68 100644 --- a/native-cli/src/main/java/org/mule/weave/cli/DWCLI.java +++ b/native-cli/src/main/java/org/mule/weave/cli/DWCLI.java @@ -30,6 +30,14 @@ public class DWCLI { public static void main(String[] args) { + // Benchmark dispatch: only reachable in a build made with -Pbenchmark=true. + // The outer compile-time constant lets javac remove this block from production. + if (BenchmarkMode.ENABLED) { + if (System.getenv("DW_BENCH") != null) { + org.mule.weave.dwnative.benchmark.BenchmarkHarness.main(args); + return; + } + } new DWCLI().run(args, DefaultConsole$.MODULE$); } diff --git a/native-cli/src/main/scala/org/mule/weave/dwnative/benchmark/BenchmarkHarness.scala b/native-cli/src/main/scala/org/mule/weave/dwnative/benchmark/BenchmarkHarness.scala new file mode 100644 index 00000000..7b5e9cba --- /dev/null +++ b/native-cli/src/main/scala/org/mule/weave/dwnative/benchmark/BenchmarkHarness.scala @@ -0,0 +1,127 @@ +package org.mule.weave.dwnative.benchmark + +import org.mule.weave.dwnative.NativeRuntime +import org.mule.weave.dwnative.WeaveExecutionResult +import org.mule.weave.dwnative.cli.DefaultConsole +import org.mule.weave.dwnative.utils.DataWeaveUtils +import org.mule.weave.v2.runtime.BindingValue +import org.mule.weave.v2.runtime.ScriptingBindings + +import java.io.{ File, OutputStream, PrintStream } +import java.nio.charset.Charset +import java.nio.file.Files + +final case class BenchInput(name: String, file: String, mimeType: String, charset: String) +final case class BenchArgs(mode: String, scriptFile: String, inputs: Seq[BenchInput], warmup: Int, iters: Int) + +/** Corpus-agnostic in-binary benchmark harness. Reachable only in a build made with + * -Pbenchmark=true (guarded by BenchmarkMode.ENABLED in DWCLI); native-image folds it + * out of a production dw. Prints "READY" the instant one NativeRuntime is constructed, + * then a single JSON line of timings. Parent (benchmarks/runners/cli) measures cold-start + * as spawn->READY wall-clock. */ +object BenchmarkHarness { + + /** Discards bytes; used as the transform write sink so we never touch real stdout. */ + private final class DiscardStream extends OutputStream { + override def write(b: Int): Unit = () + override def write(b: Array[Byte]): Unit = () + override def write(b: Array[Byte], off: Int, len: Int): Unit = () + } + + private def nowNs(): Long = System.nanoTime() + private def msSince(startNs: Long): Double = (System.nanoTime() - startNs) / 1e6 + + def parseArgs(args: Array[String]): BenchArgs = { + var mode = "" + var script = "" + val inputs = scala.collection.mutable.ArrayBuffer[BenchInput]() + var warmup = 0 + var iters = 100 + args.foreach { arg => + val eq = arg.indexOf('=') + val key = if (eq >= 0) arg.substring(0, eq) else arg + val value = if (eq >= 0) arg.substring(eq + 1) else "" + key match { + case "--bench-mode" => mode = value + case "--script" => script = value + case "--warmup" => warmup = value.toInt + case "--iters" => iters = value.toInt + case "--input" => + // value = =\t[\t] + val nameSep = value.indexOf('=') + val name = value.substring(0, nameSep) + val rest = value.substring(nameSep + 1) + val parts = rest.split("\t", 3) + val file = parts(0) + val mimeType = parts(1) + val charset = if (parts.length > 2 && parts(2).nonEmpty) parts(2) else "utf-8" + inputs += BenchInput(name, file, mimeType, charset) + case _ => throw new RuntimeException(s"unknown bench arg: $arg") + } + } + if (mode.isEmpty) throw new RuntimeException("--bench-mode is required") + if (script.isEmpty) throw new RuntimeException("--script is required") + BenchArgs(mode, script, inputs.toSeq, warmup, iters) + } + + private def newRuntime(): NativeRuntime = { + val console = DefaultConsole.enableSilent() + val utils = new DataWeaveUtils(console) + new NativeRuntime(utils.getLibPathHome(), Array.empty[File], console, None) + } + + private def readScript(a: BenchArgs): String = + new String(Files.readAllBytes(new File(a.scriptFile).toPath), java.nio.charset.StandardCharsets.UTF_8) + + private def bindings(a: BenchArgs): ScriptingBindings = { + val b = new ScriptingBindings() + a.inputs.foreach { in => + val bytes = Files.readAllBytes(new File(in.file).toPath) + val bv = new BindingValue(bytes, Some(in.mimeType), Map.empty[String, Any], Charset.forName(in.charset)) + b.addBinding(in.name, bv) + } + b + } + + private def assertOk(r: WeaveExecutionResult): Unit = + if (!r.success()) throw new RuntimeException("run failed: " + r.result()) + + def runColdFirst(a: BenchArgs, out: PrintStream, sink: OutputStream): Unit = { + val script = readScript(a) + val b = bindings(a) + val rt = newRuntime() // engine init — measured externally as cold-start + out.print("READY\n"); out.flush() + val start = nowNs() + assertOk(rt.run(script, "bench", b, sink, "application/json", None)) + val firstRunMs = msSince(start) + out.print("{\"firstRunMs\":" + firstRunMs + "}\n") + } + + def runWarm(a: BenchArgs, out: PrintStream, sink: OutputStream): Unit = { + val script = readScript(a) + val b = bindings(a) + val rt = newRuntime() + out.print("READY\n"); out.flush() + var i = 0 + while (i < a.warmup) { assertOk(rt.run(script, "bench", b, sink, "application/json", None)); i += 1 } + val samples = new Array[Double](a.iters) + i = 0 + while (i < a.iters) { + val start = nowNs() + assertOk(rt.run(script, "bench", b, sink, "application/json", None)) + samples(i) = msSince(start) + i += 1 + } + out.print("{\"warmMs\":[" + samples.mkString(",") + "]}\n") + } + + def main(args: Array[String]): Unit = { + val a = parseArgs(args) + val sink = new DiscardStream() + a.mode match { + case "coldfirst" => runColdFirst(a, System.out, sink) + case "warm" => runWarm(a, System.out, sink) + case other => throw new RuntimeException(s"unknown --bench-mode: $other") + } + } +} diff --git a/native-cli/src/test/scala/org/mule/weave/dwnative/benchmark/BenchmarkHarnessTest.scala b/native-cli/src/test/scala/org/mule/weave/dwnative/benchmark/BenchmarkHarnessTest.scala new file mode 100644 index 00000000..81342839 --- /dev/null +++ b/native-cli/src/test/scala/org/mule/weave/dwnative/benchmark/BenchmarkHarnessTest.scala @@ -0,0 +1,104 @@ +package org.mule.weave.dwnative.benchmark + +import org.scalatest.freespec.AnyFreeSpec +import org.scalatest.matchers.should.Matchers + +import java.io.{ ByteArrayOutputStream, File, PrintStream } +import java.nio.charset.StandardCharsets +import java.nio.file.Files + +class BenchmarkHarnessTest extends AnyFreeSpec with Matchers { + + private def tmp(suffix: String, content: String): File = { + val f = File.createTempFile("bench", suffix) + f.deleteOnExit() + Files.write(f.toPath, content.getBytes(StandardCharsets.UTF_8)) + f + } + + private def capture(fn: PrintStream => Unit): String = { + val buf = new ByteArrayOutputStream() + val ps = new PrintStream(buf, true, "UTF-8") + fn(ps) + new String(buf.toByteArray, StandardCharsets.UTF_8) + } + + "parseArgs" - { + "parses coldfirst mode with one input" in { + val a = BenchmarkHarness.parseArgs(Array( + "--bench-mode=coldfirst", + "--script=/tmp/x.dwl", + "--input=payload=/tmp/p.json\tapplication/json\tutf-8")) + a.mode shouldBe "coldfirst" + a.scriptFile shouldBe "/tmp/x.dwl" + a.inputs should have size 1 + a.inputs.head shouldBe BenchInput("payload", "/tmp/p.json", "application/json", "utf-8") + } + + "parses warm mode with warmup and iters" in { + val a = BenchmarkHarness.parseArgs(Array( + "--bench-mode=warm", "--script=/tmp/x.dwl", "--warmup=5", "--iters=30")) + a.mode shouldBe "warm" + a.warmup shouldBe 5 + a.iters shouldBe 30 + a.inputs shouldBe empty + } + + "handles a mimeType-only input (charset defaults to utf-8)" in { + val a = BenchmarkHarness.parseArgs(Array( + "--bench-mode=coldfirst", "--script=/tmp/x.dwl", + "--input=payload=/tmp/p.json\tapplication/json")) + a.inputs.head.charset shouldBe "utf-8" + } + } + + "runColdFirst" - { + "emits READY then a single firstRunMs JSON line, output not on the stream" in { + val script = tmp(".dwl", "output application/json --- payload.a + 1") + val input = tmp(".json", "{\"a\": 41}") + val a = BenchArgs("coldfirst", script.getAbsolutePath, + Seq(BenchInput("payload", input.getAbsolutePath, "application/json", "utf-8")), 0, 0) + val sink = new ByteArrayOutputStream() + val stdout = capture(ps => BenchmarkHarness.runColdFirst(a, ps, sink)) + val lines = stdout.split("\n").filter(_.nonEmpty) + lines.head shouldBe "READY" + lines.last should include ("firstRunMs") + lines.count(_.contains("firstRunMs")) shouldBe 1 + // The transformed "42" went to the sink, NOT to stdout. + new String(sink.toByteArray, StandardCharsets.UTF_8).trim shouldBe "42" + } + } + + "runWarm" - { + "emits READY then a warmMs array of length iters" in { + val script = tmp(".dwl", "output application/json --- payload.a + 1") + val input = tmp(".json", "{\"a\": 41}") + val a = BenchArgs("warm", script.getAbsolutePath, + Seq(BenchInput("payload", input.getAbsolutePath, "application/json", "utf-8")), 1, 3) + val stdout = capture(ps => BenchmarkHarness.runWarm(a, ps, new ByteArrayOutputStream())) + val json = stdout.split("\n").filter(_.contains("warmMs")).head + json should include ("warmMs") + // 3 comma-separated samples -> 2 commas inside the array + json.count(_ == ',') shouldBe 2 + } + } + + "a failing script throws (non-zero exit path)" in { + val script = tmp(".dwl", "output application/json --- 1 / 0") + val input = tmp(".json", "{}") + val a = BenchArgs("coldfirst", script.getAbsolutePath, + Seq(BenchInput("payload", input.getAbsolutePath, "application/json", "utf-8")), 0, 0) + an [RuntimeException] should be thrownBy + BenchmarkHarness.runColdFirst(a, capturePs(), new ByteArrayOutputStream()) + } + + private def capturePs(): PrintStream = new PrintStream(new ByteArrayOutputStream(), true, "UTF-8") + + "BenchmarkMode.ENABLED" - { + "is false in a normal (non -Pbenchmark) build" in { + // Tests run without -Pbenchmark, so the generated constant must be false — + // proving the harness is dead code / stripped from a production image. + org.mule.weave.cli.BenchmarkMode.ENABLED shouldBe false + } + } +} diff --git a/native-lib/build.gradle b/native-lib/build.gradle index 5fa389e7..95ed3af9 100644 --- a/native-lib/build.gradle +++ b/native-lib/build.gradle @@ -310,7 +310,8 @@ tasks.register('benchmarkJsUnitTest', Exec) { } workingDir("${rootDir}/benchmarks") def files = 'lib/stats.test.mjs lib/manifest.test.mjs lib/env.test.mjs ' + - 'report/report.test.mjs runners/node/emit.test.mjs' + 'report/report.test.mjs runners/node/emit.test.mjs ' + + 'runners/cli/locate.test.mjs runners/cli/emit.test.mjs' def script = 'node --test ' + files if (System.getProperty('os.name').toLowerCase().contains('windows')) { commandLine('cmd', '/c', script)