@W-23599590 Add CLI runner to DataWeave native benchmark harness

.gitignore

@@ -25,6 +25,7 @@ grimoires/
 
 .windsurf/
 .claude/
+.vscode/
 
 # Superpowers SDD scratch workspace (briefs, reports, review packages, ledger)
 .superpowers/

AGENTS.md

@@ -0,0 +1,185 @@
+# AGENTS.md
+
+## Purpose
+
+This repository packages the MuleSoft DataWeave runtime as GraalVM native artifacts.
+The language/runtime itself is supplied by pinned `org.mule.weave:*` dependencies;
+language implementation changes generally belong in the upstream DataWeave repository.
+
+- `native-cli`: native `dw` executable. Java/picocli parses arguments; Scala implements commands.
+- `native-lib`: native `dwlib` shared library, C ABI, Python binding, and Node binding.
+- `native-cli-integration-tests`: downloaded suites executed against the compiled native CLI.
+- `benchmarks/`: shared corpus/schema and CLI, JVM, Node, and Python benchmark runners.
+
+Key entry points are `DWCLI.java`, `NativeRuntime.scala`, `NativeLib.java`, and
+`ScriptRuntime.java`. Read the nearest module README before broad changes.
+
+## Toolchain
+
+- Use the checked-in `./gradlew` wrapper.
+- Native builds require GraalVM Community Java 24 with `native-image`.
+- Set `GRAALVM_HOME` and `JAVA_HOME`; native-image tasks need about 6 GB heap.
+- Node bindings require Node.js 18+ and a C compiler/node-gyp toolchain.
+- Python bindings support Python 3.9+.
+- Java source/target compatibility is 17; Scala is 2.12.
+
+## Build Commands
+
+Run from the repository root unless a command says otherwise.
+
+```bash
+./gradlew native-cli:test
+./gradlew native-lib:test -PskipNodeTests=true -PskipPythonTests=true
+./gradlew native-cli:nativeCompile
+./gradlew native-lib:nativeCompile
+./gradlew build -PskipNodeTests=true
+./gradlew native-cli:distro
+./gradlew native-lib:buildPythonWheel
+./gradlew native-lib:buildNodePackage
+./gradlew clean
+```
+
+Native outputs are `native-cli/build/native/nativeCompile/dw` and
+`native-lib/build/native/nativeCompile/dwlib.{dylib,so,dll}`. Useful flags:
+`-PskipNodeTests=true`, `-PskipPythonTests=true`, `-PskipStripDebug=true`, and
+`-PpythonExe=/path/to/python`.
+
+## Test Commands
+
+```bash
+./gradlew native-cli:test
+# One ScalaTest suite (preferred single-test granularity)
+./gradlew native-cli:test --tests "org.mule.weave.dwnative.cli.DataWeaveCLITest"
+# One native-lib JUnit class or method; skip unrelated binding lanes
+./gradlew native-lib:test --tests "org.mule.weave.lib.ScriptRuntimeTest" \
+  -PskipNodeTests=true -PskipPythonTests=true
+./gradlew native-lib:test --tests "org.mule.weave.lib.ScriptRuntimeTest.runSimpleScript" \
+  -PskipNodeTests=true -PskipPythonTests=true
+# Native CLI integration/TCK suite (builds and runs the native binary)
+./gradlew -PweaveTestSuiteVersion=2.10.0 -DweaveSuiteVersion=2.10.0 \
+  native-cli-integration-tests:test
+./gradlew native-cli-integration-tests:test \
+  --tests "org.mule.weave.clinative.NativeCliTest"
+# Dependency-free benchmark JavaScript tests
+./gradlew native-lib:benchmarkJsUnitTest
+cd benchmarks && node --test lib/stats.test.mjs
+cd benchmarks && node --test --test-name-pattern="computeStats returns" lib/stats.test.mjs
+```
+
+Node/Vitest commands (run after native staging/build when using integration tests):
+
+```bash
+cd native-lib/node
+npm install
+npm run build
+npm test
+npm run test:unit
+npm run test:integration
+npm test -- tests/unit/result.test.ts
+npm test -- tests/integration/dataweave.test.ts -t "basic arithmetic"
+```
+
+Python binding tests are a custom executable script, not a configured pytest suite:
+
+```bash
+./gradlew native-lib:pythonTest
+cd native-lib/python && python3 tests/test_dataweave_module.py
+cd benchmarks/runners/python
+python3 -m unittest test_bench.TestStats.test_matches_lib_stats_on_1_to_100
+```
+
+Node TCK is separate from normal `nodeTest`: run `./gradlew native-lib:stageTckSuites`,
+then `cd native-lib/node && npm run test:tck`.
+
+## Lint and Formatting
+
+There is no repository-wide lint or formatter command: no Scalafmt, Checkstyle,
+Spotless, ESLint, Prettier, Black, or Ruff configuration is checked in. Do not claim a
+lint step exists or introduce formatting churn. Match neighboring code. For TypeScript,
+`cd native-lib/node && npm run build:ts` is the configured strict type check.
+
+No `.cursorrules`, `.cursor/rules/`, or `.github/copilot-instructions.md` rules exist.
+
+## Code Style
+
+- Preserve local import grouping; use explicit imports and avoid new wildcards.
+- Java: four spaces, braces on the declaration line, braced control flow, explicit
+  types, `PascalCase` classes, `camelCase` members, `UPPER_SNAKE_CASE` constants.
+- Scala: two spaces, prefer `val`, idiomatic `foreach { x => ... }`, explicit public
+  return types, case classes/sealed traits for domain variants, no semicolons.
+- TypeScript/ESM: two spaces, double quotes, semicolons, trailing commas in multiline
+  constructs, `camelCase` values, `PascalCase` types/classes.
+- TypeScript is strict and emits CommonJS. Use `node:` built-in imports and `import type`;
+  local `.ts` imports omit extensions.
+- Benchmark `.mjs` is ESM: include `.mjs` extensions and use `import.meta.url`, never
+  `require` or CommonJS-only assumptions.
+- Python: PEP 8 naming, four spaces, type hints on public/boundary APIs, dataclasses for
+  result/input models, and snake_case public names translated to camelCase wire fields.
+- Gradle formatting varies by file; preserve the file's indentation and quote style.
+- Add comments/Javadoc/TSDoc for public APIs, ABI contracts, ownership, concurrency, or
+  non-obvious algorithms. Avoid comments that merely restate code.
+
+## Types and Errors
+
+- Model domain outcomes explicitly: Scala case classes/traits, TypeScript interfaces,
+  Python dataclasses. Use loose JSON/maps only at serialization or FFI boundaries.
+- Preserve wire fields exactly: `success`, `result`, `error`, `mimeType`, `charset`,
+  `binary`, `streamHandle`, `content`, and `properties`.
+- Script failures normally return an unsuccessful result envelope. Binding/lifecycle
+  failures throw `DataWeaveError`; opt-in modes may promote script failures to
+  `DataWeaveScriptError`. CLI failures need context and a nonzero exit code.
+- Include operation context in errors, avoid null-only messages, and escape every string
+  crossing a manually constructed JSON boundary.
+- Use `try/finally` for native allocations, stream registrations, isolate attachment,
+  and cleanup. Suppress cleanup errors only when they must not mask an earlier failure.
+- Never let JavaScript or Python exceptions unwind across C callbacks; translate them to
+  the documented callback status (`0` success, nonzero/-1 error, `0` read means EOF).
+
+## Native and FFI Rules
+
+- Treat exported C names, argument order, callback semantics, nullability, ownership,
+  and result JSON as a stable ABI. Update Java, C addon, TypeScript, Python, tests, and
+  documentation together when changing it.
+- Every OS thread calling Graal must attach its own isolate thread and detach afterward.
+  Never reuse an `IsolateThread` from another OS thread.
+- Do not capture Graal `Word`/pointer values in Java lambdas; convert to raw addresses and
+  reconstruct pointers inside explicit workers.
+- Copy callback/native buffers before returning; never retain borrowed pointers or use a
+  Graal-owned C string after `free_cstring`.
+- Preserve bounded queues/backpressure and chunk remainders; chunks can exceed the native
+  8 KiB callback buffer and do not align with logical records.
+- Node worker threads must use N-API thread-safe functions; do not call V8 directly.
+- Native-image `buildArgs` are load-bearing; do not simplify initialization, charset,
+  locale, HTTP, reflection, or resources without native build/runtime coverage.
+- Preserve SPI descriptors for `DataFormat` and `ModuleLoader`; `native-lib` deliberately
+  re-materializes them when creating its fat JAR. Missing entries often appear as
+  runtime "unknown mime type" failures, not build errors.
+
+## Tests and Generated Files
+
+- Scala CLI tests use `AnyFreeSpec`/`Matchers`; exercise picocli parsing and assert exit
+  code, stdout, and stderr.
+- Java library tests use JUnit 5. Node uses Vitest projects (`unit`, `integration`, `tck`).
+  Benchmark JavaScript uses `node:test`; its Gradle task enumerates files explicitly, so
+  wire new test files into `native-lib/build.gradle` when they belong in normal testing.
+- Streaming changes need chunk integrity, terminal metadata, large-chunk, failure, and
+  cleanup coverage. Shared benchmark math/schema changes need JS/Python/Scala parity.
+- Never edit generated `build/genresource/.../ComponentVersion.scala`,
+  `build/genjava/.../BenchmarkMode.java`, Graal headers, staged `dwlib.*`, downloaded TCK
+  suites, benchmark results, or generated benchmark inputs. Change generators/sources.
+- Gradle stages native libraries into Python and Node packages; do not commit staged or
+  packaged artifacts (`node_modules`, `dist`, `native`, wheels, tarballs, results).
+
+## Benchmarks and CI
+
+Benchmarks are opt-in: `./gradlew benchmarkCompare -Pbenchmark=true`. Runner tasks carry
+`ext.benchmarkRunner = true`; the root aggregator auto-discovers them. New runners must
+use the shared corpus/schema and write `benchmarks/results/<runner>-<timestamp>.json`.
+Do not add benchmark execution to normal `build` or `test`. The CLI benchmark requires a
+binary built with `-Pbenchmark=true`; `DW_BENCH_BIN` may select a prebuilt one.
+
+CI builds Ubuntu and Windows with GraalVM 24. Native CLI regression suites and Node TCK
+run only on `master`. Run the smallest relevant suite, then the nearest module test; use
+a native build for FFI, SPI, resources, initialization, packaging, or native-image changes.
+Keep PRs focused, add regression tests, minimize dependencies, and report vulnerabilities
+through the Salesforce portal named in `SECURITY.md`, never through public issues.

benchmarks/README.md

@@ -13,6 +13,12 @@ Language-agnostic benchmark harness for the DataWeave native-lib wrappers.
   (Scala/Gradle subproject `:benchmarks-engine`, depends on `org.mule.weave:runtime` at
   the same `weaveVersion` the native image is built from). `runners/python/` is the
   Python runner (stdlib scripts under `native-lib`, wrapping the same staged `dwlib` as Node).
+  `runners/cli/` is the CLI runner: a Node parent that spawns the `dw` native
+  binary (built with `-Pbenchmark=true`, which compiles in an in-binary
+  benchmark harness gated by `BenchmarkMode.ENABLED` and dispatched via the
+  `DW_BENCH` env var — the shipped `dw` contains none of it). It emits
+  `cold-start`, `first-run`, and `warm`; it does **not** emit `streaming`
+  (the `dw run` path has no chunked-input FFI like the library's).
 - `report/report.mjs` — joins result files against the manifest and prints a comparison table.
 - `results/` — gitignored per-run output.
 
@@ -39,6 +45,10 @@ and `JAVA_HOME` set to it (see the root README / `CLAUDE.md`). The pinned build
 `graalvmVersion` in `gradle.properties`. The **engine runner alone** drives the JVM
 `DataWeaveScriptingEngine` and runs on any JDK — no native image required.
 
+The **CLI runner** requires the bench-enabled binary
+(`./gradlew native-cli:nativeCompile -Pbenchmark=true`); set `DW_BENCH_BIN` to
+point at a prebuilt one. Like the library runners it needs the GraalVM toolchain.
+
 ## Running
 
 The one-shot cross-runner comparison — runs **every** registered runner and prints the table:
@@ -50,6 +60,7 @@ Single-runner options:
     ./gradlew native-lib:benchmark -Pbenchmark=true              # Node only: build wrapper, run, report
     ./gradlew benchmarks-engine:benchmarkEngine -Pbenchmark=true # engine (JVM) only: writes results/engine-<ts>.json
     ./gradlew native-lib:benchmarkPython -Pbenchmark=true        # Python only: writes results/python-<ts>.json
+    ./gradlew native-cli:benchmarkCli -Pbenchmark=true            # CLI only: writes results/cli-<ts>.json
 
 Or directly, once the wrapper is built (`./gradlew native-lib:buildNodePackage`):
 

benchmarks/runners/cli/coldstart.mjs

@@ -0,0 +1,100 @@
+import { spawn } from "node:child_process";
+import { join } from "node:path";
+import { casesForMetric } from "../../lib/manifest.mjs";
+import { computeStats } from "../../lib/stats.mjs";
+import { locateBinary } from "./locate.mjs";
+
+/** Build `--input=name=file\tmime\tcharset` args for a case (absolute paths). */
+function inputArgs(manifest, c) {
+  const args = [];
+  for (const [name, inp] of Object.entries(c.inputs ?? {})) {
+    const file = join(manifest.corpusDir, inp.file);
+    const charset = inp.charset ?? "utf-8";
+    args.push(`--input=${name}=${file}\t${inp.mimeType}\t${charset}`);
+  }
+  return args;
+}
+
+/**
+ * Spawn one fresh dw process in coldfirst mode. Cold-start = wall-clock from just
+ * before spawn to the child's "READY" marker (process launch + native image load +
+ * NativeRuntime init). first-run is timed in-process by the child. Rejects on a
+ * non-zero exit or a missing READY/JSON line so a failed sample never records a
+ * bogus timing.
+ */
+function sampleOnce(bin, manifest, c) {
+  const scriptPath = join(manifest.corpusDir, c.script);
+  const args = ["--bench-mode=coldfirst", `--script=${scriptPath}`, ...inputArgs(manifest, c)];
+  return new Promise((resolve, reject) => {
+    const t0 = process.hrtime.bigint();
+    const child = spawn(bin, args, {
+      stdio: ["ignore", "pipe", "pipe"],
+      env: { ...process.env, DW_BENCH: "1" },
+    });
+    let coldStartMs;
+    let stdout = "";
+    let stderr = "";
+    child.stdout.setEncoding("utf-8");
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk;
+      if (coldStartMs === undefined && stdout.includes("READY\n")) {
+        coldStartMs = Number(process.hrtime.bigint() - t0) / 1e6;
+      }
+    });
+    child.stderr.setEncoding("utf-8");
+    child.stderr.on("data", (chunk) => (stderr += chunk));
+    child.on("error", reject);
+    child.on("close", (code) => {
+      if (code !== 0) {
+        reject(new Error(`cli coldfirst failed for '${c.id}' (exit ${code})\n${stderr}`));
+        return;
+      }
+      if (coldStartMs === undefined) {
+        reject(new Error(`cli coldfirst for '${c.id}' never printed READY\n${stderr}`));
+        return;
+      }
+      const jsonLine = stdout.split("\n").filter((l) => l && l !== "READY").pop();
+      if (!jsonLine) {
+        reject(new Error(`cli coldfirst for '${c.id}' printed no result line\n${stderr}`));
+        return;
+      }
+      let firstRunMs;
+      try {
+        ({ firstRunMs } = JSON.parse(jsonLine));
+      } catch (error) {
+        reject(new Error(`cli coldfirst for '${c.id}' printed invalid JSON: ${jsonLine}`, { cause: error }));
+        return;
+      }
+      resolve({ coldStartMs, firstRunMs });
+    });
+  });
+}
+
+/** @returns {Promise<Array<{id,metric,unit,stats,iterations}>>} */
+export async function runColdStartAndFirstRun(manifest, { samplesOverride } = {}) {
+  const bin = locateBinary();
+  const rows = [];
+  const ids = new Set([
+    ...casesForMetric(manifest, "cold-start").map((c) => c.id),
+    ...casesForMetric(manifest, "first-run").map((c) => c.id),
+  ]);
+
+  for (const id of ids) {
+    const c = manifest.cases.find((x) => x.id === id);
+    const n = samplesOverride ?? c.iterations?.samples ?? 20;
+    const colds = [];
+    const firsts = [];
+    for (let i = 0; i < n; i++) {
+      const { coldStartMs, firstRunMs } = await sampleOnce(bin, manifest, c);
+      colds.push(coldStartMs);
+      firsts.push(firstRunMs);
+    }
+    if (c.metrics.includes("cold-start")) {
+      rows.push({ id, metric: "cold-start", unit: "ms", stats: computeStats(colds), iterations: n });
+    }
+    if (c.metrics.includes("first-run")) {
+      rows.push({ id, metric: "first-run", unit: "ms", stats: computeStats(firsts), iterations: n });
+    }
+  }
+  return rows;
+}

benchmarks/runners/cli/emit.mjs

@@ -0,0 +1,63 @@
+import { writeFileSync, mkdirSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { join, dirname } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { loadManifest, validateResultIds } from "../../lib/manifest.mjs";
+import { gatherEnv } from "../../lib/env.mjs";
+import { locateBinary } from "./locate.mjs";
+import { runColdStartAndFirstRun } from "./coldstart.mjs";
+import { runWarm } from "./warm.mjs";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const CORPUS = join(__dirname, "..", "..", "corpus");
+const RESULTS_DIR = join(__dirname, "..", "..", "results");
+
+/** Assemble the full schema object (identical contract to the Node runner). */
+export function buildResult(env, cases) {
+  return {
+    schemaVersion: "1.0",
+    runner: env.runner,
+    env,
+    timestamp: new Date().toISOString(),
+    cases,
+  };
+}
+
+/** Best-effort `dw --version` first line; falls back to "dw". */
+function probeVersion(bin) {
+  try {
+    const out = execFileSync(bin, ["--version"], { encoding: "utf-8" });
+    const line = out.split("\n").map((l) => l.trim()).filter(Boolean)[0];
+    return line ? `dw ${line}` : "dw";
+  } catch {
+    return "dw";
+  }
+}
+
+export async function main() {
+  const manifest = loadManifest(CORPUS);
+  const bin = locateBinary();
+  const env = gatherEnv({ runner: "cli", runtimeVersion: probeVersion(bin) });
+  // The CLI is a native binary, not the staged dwlib — override the lib fingerprint.
+  env.dwlibBuildId = "n/a-cli";
+
+  const coldRows = await runColdStartAndFirstRun(manifest);
+  const warmRows = await runWarm(manifest);
+
+  const cases = [...coldRows, ...warmRows];
+  validateResultIds(manifest, cases);
+
+  mkdirSync(RESULTS_DIR, { recursive: true });
+  const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+  const outPath = join(RESULTS_DIR, `cli-${stamp}.json`);
+  writeFileSync(outPath, JSON.stringify(buildResult(env, cases), null, 2));
+  console.log(`wrote ${outPath} (${cases.length} rows)`);
+  return outPath;
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  main().catch((e) => {
+    console.error(e.message);
+    process.exit(1);
+  });
+}