Add --smoke-tree-sitter flag and fail builds with empty embed

Freebuff 0.0.64 still crashed for users with the same wasm error even
though it was built from a commit that contained the base64 embed. The
runtime stack trace pointed at the path-resolution fallback in
init-node.ts:76, meaning the embed didn't reach the SDK bundle's
globalThis check at runtime — the binary fell through to fs.existsSync
which never works on Windows bunfs paths.

Two hardening passes so this can't ship silently again:

- cli/src/pre-init/tree-sitter-wasm.ts: hidden `--smoke-tree-sitter`
  flag, handled in the very first import. Calls Parser.init({ wasmBinary
  }) directly with the embedded base64 and exits 0/1. Lives here (not
  commander) on purpose — it tests *the embed*, not the broader init
  path that has a path-resolution fallback that would mask a broken
  embed by passing in dev mode.
- cli/scripts/build-binary.ts: post-bun-compile, scan the output binary
  for the wasm's base64 prefix. Build fails if the bytes didn't actually
  make it through bundling (e.g. bun dropping a huge string literal,
  bundle cache reading a stale empty stub). Always-on log of which path
  the wasm was resolved from so CI logs make the embed step diagnosable.
  More resilient resolve: search workspace root, cli/node_modules, and
  sdk/node_modules before falling back to createRequire — Windows CI's
  `bun install --cwd cli` lays out web-tree-sitter differently than
  a hoisted root install.
- packages/code-map/src/init-node.ts: accept bunfs paths
  (`/~BUN/root/...`) without an fs.existsSync check. fs.existsSync
  inconsistently returns false for bun --compile asset paths on Windows
  even though the runtime can read them, so the existing path-resolution
  fallback was permanently broken on Windows. Belt-and-braces: this
  makes the fallback work even if the embed step regresses.
- cli/scripts/smoke-binary.ts: run --smoke-tree-sitter as a deterministic
  pre-check before the long-window boot smoke. A broken embed fails fast
  with a clear "exit code 1, no boot ok marker" error instead of a 10s
  timeout that depends on render-loop timing.

Verified locally: build embeds 205KB wasm as 274KB base64, post-build
verification finds the prefix in the compiled binary, --smoke-tree-sitter
exits 0 with "tree-sitter smoke ok", full smoke passes.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: jahooma

cli/scripts/build-binary.ts

@@ -145,10 +145,10 @@ async function main() {
   patchOpenTuiAssetPaths()
   await ensureOpenTuiNativeBundle(targetInfo)
 
-  const restoreTreeSitterWasmStub = embedTreeSitterWasmAsBase64()
+  const treeSitterEmbed = embedTreeSitterWasmAsBase64()
   // Restore the stub even on build failure so a developer's git working
   // tree doesn't end up with a multi-megabyte modified file.
-  process.on('exit', restoreTreeSitterWasmStub)
+  process.on('exit', treeSitterEmbed.restore)
 
   const outputFilename =
     targetInfo.platform === 'win32' ? `${binaryName}.exe` : binaryName
@@ -194,7 +194,17 @@ async function main() {
   // Build done — restore the stub so a developer's working tree doesn't show
   // a multi-megabyte diff. (The exit handler above is a backstop for crashes;
   // the eager call here keeps a successful build clean.)
-  restoreTreeSitterWasmStub()
+  treeSitterEmbed.restore()
+
+  // Fail the build if the wasm bytes didn't actually make it into the
+  // compiled binary. Catches silent regressions (e.g. bun dropping a huge
+  // string literal, or some future bundler optimization) before we ship a
+  // broken artifact to users.
+  verifyTreeSitterWasmEmbedded(
+    outputFile,
+    treeSitterEmbed.wasmBase64Prefix,
+    treeSitterEmbed.wasmByteLength,
+  )
 
   if (targetInfo.platform !== 'win32') {
     chmodSync(outputFile, 0o755)
@@ -225,7 +235,11 @@ main().catch((error: unknown) => {
  * Returns a function that restores the stub. Always invoke it (success or
  * failure) so a developer's working tree doesn't show a multi-MB diff.
  */
-function embedTreeSitterWasmAsBase64(): () => void {
+function embedTreeSitterWasmAsBase64(): {
+  restore: () => void
+  wasmBase64Prefix: string
+  wasmByteLength: number
+} {
   const stubPath = join(cliRoot, 'src', 'pre-init', 'tree-sitter-wasm-bytes.ts')
   const originalStub = readFileSync(stubPath, 'utf8')
   let restored = false
@@ -239,11 +253,30 @@ function embedTreeSitterWasmAsBase64(): () => void {
     }
   }
 
-  // Resolve from the CLI workspace so monorepo hoisting differences don't
-  // matter — `web-tree-sitter` is an SDK dep, but the CLI imports it
-  // transitively and the bundler walks it from here.
-  const cliRequire = createRequire(join(cliRoot, 'package.json'))
-  const wasmPath = cliRequire.resolve('web-tree-sitter/tree-sitter.wasm')
+  // Try multiple candidate locations because bun's hoisting differs by
+  // platform and install command — Windows CI does `bun install --cwd cli`
+  // which can leave web-tree-sitter in cli/node_modules, while monorepo
+  // root installs hoist it to ../node_modules. Fall back to createRequire
+  // last so any failure surfaces with the full search trail.
+  const candidates = [
+    join(cliRoot, 'node_modules', 'web-tree-sitter', 'tree-sitter.wasm'),
+    join(cliRoot, '..', 'node_modules', 'web-tree-sitter', 'tree-sitter.wasm'),
+    join(cliRoot, '..', 'sdk', 'node_modules', 'web-tree-sitter', 'tree-sitter.wasm'),
+  ]
+  let wasmPath = candidates.find((p) => existsSync(p))
+  if (!wasmPath) {
+    try {
+      const cliRequire = createRequire(join(cliRoot, 'package.json'))
+      wasmPath = cliRequire.resolve('web-tree-sitter/tree-sitter.wasm')
+    } catch (err) {
+      throw new Error(
+        `Could not locate web-tree-sitter/tree-sitter.wasm. Searched:\n  - ` +
+          candidates.join('\n  - ') +
+          `\nAnd createRequire failed: ${err instanceof Error ? err.message : String(err)}`,
+      )
+    }
+  }
+
   const wasmBytes = readFileSync(wasmPath)
   const base64 = wasmBytes.toString('base64')
 
@@ -254,8 +287,47 @@ function embedTreeSitterWasmAsBase64(): () => void {
     `export const TREE_SITTER_WASM_BASE64 = ${JSON.stringify(base64)}\n`
 
   writeFileSync(stubPath, generated)
-  log(`Embedded tree-sitter.wasm (${wasmBytes.length} bytes → ${base64.length} chars base64)`)
-  return restore
+  // Always-on log (not behind VERBOSE) so CI shows which path was used and
+  // whether the embed succeeded — this is the single most useful breadcrumb
+  // when the runtime check fails on a user machine.
+  logAlways(
+    `Embedded tree-sitter.wasm from ${wasmPath} (${wasmBytes.length} bytes → ${base64.length} chars base64)`,
+  )
+  return {
+    restore,
+    wasmBase64Prefix: base64.slice(0, 40),
+    wasmByteLength: wasmBytes.length,
+  }
+}
+
+/**
+ * Sanity-check the compiled binary actually contains the embedded base64.
+ * If bun --compile ever silently drops a large string literal, or our embed
+ * step's file write didn't take effect before the bundle ran, we want the
+ * build to fail here instead of producing a binary that crashes for users.
+ */
+function verifyTreeSitterWasmEmbedded(
+  outputFile: string,
+  wasmBase64Prefix: string,
+  wasmByteLength: number,
+): void {
+  const binary = readFileSync(outputFile)
+  // Search as a Buffer so we don't have to load the whole binary as a UTF-8
+  // string (binaries are not valid UTF-8 and toString would corrupt bytes).
+  const needle = Buffer.from(wasmBase64Prefix, 'utf8')
+  const idx = binary.indexOf(needle)
+  if (idx === -1) {
+    throw new Error(
+      `Embedded tree-sitter wasm prefix not found in ${outputFile}.\n` +
+        `Expected base64 prefix (first 40 chars): ${wasmBase64Prefix}\n` +
+        `Original wasm size: ${wasmByteLength} bytes.\n` +
+        `This means the build-binary.ts embed step ran but bun --compile\n` +
+        `did not include the bytes in the output. The runtime smoke test\n` +
+        `would fall back to path-based wasm resolution, which is broken on\n` +
+        `Windows.`,
+    )
+  }
+  logAlways(`Verified embedded wasm prefix at offset ${idx} of compiled binary.`)
 }
 
 function patchOpenTuiAssetPaths() {

cli/scripts/smoke-binary.ts

@@ -81,6 +81,39 @@ const FATAL_PATTERNS = [
 // the renderer is up).
 const DEFAULT_RUN_SECONDS = 10
 
+function runTreeSitterSmoke(binary: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const proc = spawn(binary, ['--smoke-tree-sitter'], {
+      stdio: ['ignore', 'pipe', 'pipe'],
+      env: { ...process.env, NO_COLOR: '1', TERM: 'dumb' },
+    })
+
+    let captured = ''
+    const append = (chunk: Buffer): void => {
+      captured += chunk.toString('utf8')
+    }
+    proc.stdout?.on('data', append)
+    proc.stderr?.on('data', append)
+
+    proc.once('error', reject)
+    proc.once('exit', (code) => {
+      if (code === 0 && /tree-sitter smoke ok/.test(captured)) {
+        resolve()
+        return
+      }
+
+      reject(
+        new Error(
+          `tree-sitter smoke failed with exit code ${code}\n${captured.slice(
+            0,
+            8 * 1024,
+          )}`,
+        ),
+      )
+    })
+  })
+}
+
 async function main(): Promise<void> {
   const binary = process.argv[2]
   const runSeconds = Number(process.argv[3] ?? DEFAULT_RUN_SECONDS)
@@ -100,6 +133,9 @@ async function main(): Promise<void> {
 
   console.log(`smoke-binary: spawning ${binary} for ${runSeconds}s…`)
 
+  await runTreeSitterSmoke(binary)
+  console.log('smoke-binary: tree-sitter init OK.')
+
   const proc = spawn(binary, [], {
     stdio: ['ignore', 'pipe', 'pipe'],
     env: { ...process.env, NO_COLOR: '1', TERM: 'dumb' },

cli/src/pre-init/tree-sitter-wasm.ts

@@ -18,18 +18,54 @@
 
 import { TREE_SITTER_WASM_BASE64 } from './tree-sitter-wasm-bytes'
 
+let embeddedWasm: Uint8Array | undefined
 if (TREE_SITTER_WASM_BASE64.length > 0) {
   const buf = Buffer.from(TREE_SITTER_WASM_BASE64, 'base64')
+  embeddedWasm = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
   // globalThis is the only cross-bundle channel: the SDK pre-built bundle
   // inlines its own copy of `init-node.ts`, so a module-level variable in
   // the source package isn't visible to the singleton initialized via the
   // SDK. Slice into a fresh Uint8Array view instead of handing over the
   // Buffer's shared underlying ArrayBuffer.
   ;(
     globalThis as { __CODEBUFF_TREE_SITTER_WASM_BINARY__?: Uint8Array }
-  ).__CODEBUFF_TREE_SITTER_WASM_BINARY__ = new Uint8Array(
-    buf.buffer,
-    buf.byteOffset,
-    buf.byteLength,
-  )
+  ).__CODEBUFF_TREE_SITTER_WASM_BINARY__ = embeddedWasm
+}
+
+// Deterministic CI gate: `<binary> --smoke-tree-sitter` proves the embed
+// shipped end-to-end. Lives here, in the very first import, on purpose:
+//
+// - We're testing whether the *embed* works. Going through commander +
+//   initTreeSitterForNode would also pass via the path-resolution
+//   fallback when the embed is empty (e.g. dev mode), giving false
+//   positives that mask a broken production build.
+// - Failing here, before any other module loads, gives a sharp signal:
+//   the embed either worked or it didn't. No render-loop timing, no
+//   commander wiring, no SDK init order to debug.
+//
+// Async IIFE because Parser.init returns a promise; process.exit tears
+// the process down before parallel top-level imports can fire side
+// effects we'd have to clean up.
+if (process.argv.includes('--smoke-tree-sitter')) {
+  void (async () => {
+    try {
+      if (!embeddedWasm) {
+        console.error(
+          'tree-sitter smoke FAIL: TREE_SITTER_WASM_BASE64 stub is empty — ' +
+            'the build-binary.ts embed step did not run or did not write the file.',
+        )
+        process.exit(1)
+      }
+      const { Parser } = await import('web-tree-sitter')
+      await Parser.init({ wasmBinary: embeddedWasm })
+      // Marker grepped by cli/scripts/smoke-binary.ts — keep this exact text.
+      console.log(
+        `tree-sitter smoke ok (${embeddedWasm.byteLength} bytes wasm initialized)`,
+      )
+      process.exit(0)
+    } catch (err) {
+      console.error('tree-sitter smoke FAIL:', err)
+      process.exit(1)
+    }
+  })()
 }

packages/code-map/src/init-node.ts

@@ -30,14 +30,20 @@ function getEmbeddedWasmBinary(): Uint8Array | undefined {
   )[WASM_BINARY_GLOBAL_KEY]
 }
 
+function isBunEmbeddedPath(filePath: string): boolean {
+  return filePath.replace(/\\/g, '/').includes('/~BUN/root/')
+}
+
 function resolveTreeSitterWasm(scriptDir: string): string {
   const override = process.env[TREE_SITTER_WASM_ENV_VAR]
-  if (override && fs.existsSync(override)) {
-    return override
+  if (override) {
+    if (fs.existsSync(override) || isBunEmbeddedPath(override)) {
+      return override
+    }
   }
 
   const fallback = path.join(scriptDir, 'tree-sitter.wasm')
-  if (fs.existsSync(fallback)) {
+  if (fs.existsSync(fallback) || isBunEmbeddedPath(fallback)) {
     return fallback
   }