Embed tree-sitter wasm as base64 string literal in CLI binary

Freebuff 0.0.62 still crashed on Windows with the same "Internal error:
tree-sitter.wasm not found" — surfaced this time through the late
renderer-cleanup handler ("Unhandled rejection: error: ...") instead of
the early one, so it appeared *after* the logo had rendered. CI Windows
smoke passed because the rejection fires past the 5s kill timer (after
React mounts and the renderer is up), and even when it does fire, the
boot screen has already matched our positive signal.

Root cause: the previous fix's `fs.readFileSync(treeSitterWasmPath)` of
the bunfs path silently fails on Windows for some user environments,
its catch block falls through, globalThis stays unset, and init-node
then hits the broken path-based fallback. CI Windows happened to pass
fs.readFileSync — user Windows didn't.

Bypass the filesystem entirely: bake the wasm bytes into the JS source
as a base64 string literal that bun --compile bundles into the binary's
text segment. No runtime fs read, no path normalization, no platform
quirks.

- cli/src/pre-init/tree-sitter-wasm-bytes.ts: committed stub with empty
  base64. Dev mode and unit tests see this and fall through to
  code-map's path-based resolution (which works locally because
  node_modules/web-tree-sitter/tree-sitter.wasm exists).
- cli/scripts/build-binary.ts: overwrites the stub with the real bytes
  before `bun build --compile`, restores it after. `process.on('exit',
  restore)` is a backstop so a crash mid-build doesn't leave a multi-MB
  diff in the working tree.
- cli/src/pre-init/tree-sitter-wasm.ts: drop the `with { type: 'file' }`
  + readFileSync path, decode the embedded base64 directly.
- cli/scripts/smoke-binary.ts: bump the run window from 5s to 10s and
  match the late-handler form ("Unhandled rejection:" / "Uncaught
  exception:") in addition to the early one. The 0.0.62 regression
  fired *after* the boot screen rendered, so a positive boot signal
  alone isn't enough — we need to keep watching for fatal markers
  through the full window.

Verified locally: full bun --compile build embeds 205KB of wasm as
274KB of base64, stub is restored after build (and after a simulated
mid-build crash via the exit handler), binary boots cleanly to the
chat surface with no wasm errors.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: jahooma

cli/scripts/build-binary.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env bun
 
 import { spawnSync, type SpawnSyncOptions } from 'child_process'
+import { createRequire } from 'module'
 import {
   chmodSync,
   existsSync,
@@ -144,6 +145,11 @@ async function main() {
   patchOpenTuiAssetPaths()
   await ensureOpenTuiNativeBundle(targetInfo)
 
+  const restoreTreeSitterWasmStub = embedTreeSitterWasmAsBase64()
+  // Restore the stub even on build failure so a developer's git working
+  // tree doesn't end up with a multi-megabyte modified file.
+  process.on('exit', restoreTreeSitterWasmStub)
+
   const outputFilename =
     targetInfo.platform === 'win32' ? `${binaryName}.exe` : binaryName
   const outputFile = join(binDir, outputFilename)
@@ -185,6 +191,11 @@ async function main() {
 
   runCommand('bun', buildArgs, { cwd: cliRoot })
 
+  // Build done — restore the stub so a developer's working tree doesn't show
+  // a multi-megabyte diff. (The exit handler above is a backstop for crashes;
+  // the eager call here keeps a successful build clean.)
+  restoreTreeSitterWasmStub()
+
   if (targetInfo.platform !== 'win32') {
     chmodSync(outputFile, 0o755)
   }
@@ -203,6 +214,50 @@ main().catch((error: unknown) => {
   process.exit(1)
 })
 
+/**
+ * Inline the contents of `web-tree-sitter/tree-sitter.wasm` as a base64 string
+ * literal in `cli/src/pre-init/tree-sitter-wasm-bytes.ts`. The committed
+ * file is a stub; this overwrites it with the real bytes immediately before
+ * `bun build --compile`, so the bytes get baked into the binary's text
+ * segment instead of being placed at a bunfs path that has to be fs-read at
+ * runtime.
+ *
+ * Returns a function that restores the stub. Always invoke it (success or
+ * failure) so a developer's working tree doesn't show a multi-MB diff.
+ */
+function embedTreeSitterWasmAsBase64(): () => void {
+  const stubPath = join(cliRoot, 'src', 'pre-init', 'tree-sitter-wasm-bytes.ts')
+  const originalStub = readFileSync(stubPath, 'utf8')
+  let restored = false
+  const restore = (): void => {
+    if (restored) return
+    restored = true
+    try {
+      writeFileSync(stubPath, originalStub)
+    } catch (error) {
+      console.error('Failed to restore tree-sitter-wasm-bytes stub:', error)
+    }
+  }
+
+  // Resolve from the CLI workspace so monorepo hoisting differences don't
+  // matter — `web-tree-sitter` is an SDK dep, but the CLI imports it
+  // transitively and the bundler walks it from here.
+  const cliRequire = createRequire(join(cliRoot, 'package.json'))
+  const wasmPath = cliRequire.resolve('web-tree-sitter/tree-sitter.wasm')
+  const wasmBytes = readFileSync(wasmPath)
+  const base64 = wasmBytes.toString('base64')
+
+  const generated =
+    `// AUTO-GENERATED by cli/scripts/build-binary.ts during \`bun build --compile\`.\n` +
+    `// Restored to the empty stub after the build finishes — do not commit a\n` +
+    `// non-empty value here.\n` +
+    `export const TREE_SITTER_WASM_BASE64 = ${JSON.stringify(base64)}\n`
+
+  writeFileSync(stubPath, generated)
+  log(`Embedded tree-sitter.wasm (${wasmBytes.length} bytes → ${base64.length} chars base64)`)
+  return restore
+}
+
 function patchOpenTuiAssetPaths() {
   const coreDir = join(cliRoot, 'node_modules', '@opentui', 'core')
   if (!existsSync(coreDir)) {

cli/scripts/smoke-binary.ts

@@ -57,14 +57,29 @@ const BOOT_SIGNAL_PATTERNS = [
 // regressions of bugs we've already seen. The boot-signal check above is
 // the real gate: it fails on *any* startup problem, including ones whose
 // error text we never thought to add here.
+//
+// Note both paths the cli error handlers print: "Fatal error during
+// startup" (earlyFatalHandler in cli/src/index.tsx, fires while main()
+// is still wiring up) and "Unhandled rejection:" / "Uncaught exception:"
+// (installProcessCleanupHandlers in cli/src/utils/renderer-cleanup.ts,
+// fires after the renderer is up). The wasm-load rejection on freebuff
+// 0.0.62 surfaced through the *late* renderer-cleanup path, after the
+// boot screen had already rendered.
 const FATAL_PATTERNS = [
   /Fatal error during startup/i,
+  /Unhandled rejection:/i,
+  /Uncaught exception:/i,
   /Internal error: tree-sitter\.wasm not found/i,
   /UnhandledPromiseRejection/i,
   /Cannot find module/i,
 ] as const
 
-const DEFAULT_RUN_SECONDS = 5
+// Long enough that an unhandled rejection from the eager Parser.init has
+// time to surface through the renderer-cleanup handler — that path is
+// what tripped freebuff 0.0.62 in the wild while a 5s window let CI pass.
+// Async wasm rejections can fire >5s after spawn (after React mounts and
+// the renderer is up).
+const DEFAULT_RUN_SECONDS = 10
 
 async function main(): Promise<void> {
   const binary = process.argv[2]

cli/src/pre-init/tree-sitter-wasm-bytes.ts

@@ -0,0 +1,16 @@
+// Stub committed for dev mode and tests. The real wasm bytes are inlined
+// here as base64 by `cli/scripts/build-binary.ts` immediately before
+// `bun build --compile`, then restored to the empty stub after the build
+// completes. Dev mode and unit tests see the empty stub and fall back to
+// path-based resolution in `packages/code-map/src/init-node.ts` (which
+// works locally because `node_modules/web-tree-sitter/tree-sitter.wasm`
+// exists on the filesystem).
+//
+// Why a string literal instead of `with { type: 'file' }` + readFileSync:
+// the file-import approach left the bytes in bunfs and required a runtime
+// fs read, which silently failed on Windows (`fs.readFileSync` for
+// `B:\~BUN\root\...` paths) and let the singleton fall through to a
+// path-based fallback that also failed there. A base64 string literal in
+// the JS source compiles into the bun binary's text segment, with no
+// filesystem step on the hot path.
+export const TREE_SITTER_WASM_BASE64 = ''

cli/src/pre-init/tree-sitter-wasm.ts

@@ -1,37 +1,35 @@
 // Embed tree-sitter.wasm into the bun-compile binary so the SDK's tree-sitter
 // parser singleton can find it at runtime. Must be the very first import in
 // `index.tsx`: subsequent imports (the SDK / code-map) eagerly construct the
-// parser, and its init reads what we publish here on `globalThis` and `process.env`.
+// parser, and its init reads what we publish here on `globalThis`.
 //
-// Why not just `locateFile` + a path? On Windows, bun --compile reports the
-// embedded path as `B:\~BUN\root\...`, and `fs.existsSync` returns false for
-// that path inside the running binary even though `fs.readFileSync` works. So
-// we read the bytes once at startup and pass them straight to `Parser.init`
-// via `wasmBinary`, sidestepping filesystem resolution entirely.
-
-import * as fs from 'fs'
-
-// @ts-expect-error - Bun's `with { type: 'file' }` returns a string path; TS resolves
-// the .wasm file via web-tree-sitter's exports map and has no loader for it.
-import treeSitterWasmPath from 'web-tree-sitter/tree-sitter.wasm' with {
-  type: 'file',
-}
+// Why not `with { type: 'file' }` + a runtime fs read? That's what the prior
+// fix tried, and it silently failed on Windows: bun --compile reports the
+// embedded asset path as `B:\~BUN\root\...`, and on some Windows configs
+// `fs.readFileSync` of that path throws (caught silently), so the SDK fell
+// back to path-based resolution that also failed there.
+//
+// The base64 string in `tree-sitter-wasm-bytes.ts` is replaced with the real
+// wasm contents by `cli/scripts/build-binary.ts` right before `bun build
+// --compile` and restored after. The bytes end up in the binary's text
+// segment as a JS string literal — no filesystem step on the hot path. In
+// dev / unit tests the stub is empty and code-map falls back to the
+// node_modules wasm, which works because the file actually exists locally.
 
-if (treeSitterWasmPath) {
-  // Path stays for any consumer (tests, dev runs) that still resolves via fs.
-  process.env.CODEBUFF_TREE_SITTER_WASM_PATH = treeSitterWasmPath
+import { TREE_SITTER_WASM_BASE64 } from './tree-sitter-wasm-bytes'
 
-  try {
-    const binary = fs.readFileSync(treeSitterWasmPath)
-    // globalThis is the only cross-bundle channel: the SDK pre-built bundle
-    // inlines its own copy of `init-node.ts`, so a module-level variable in
-    // the source package wouldn't be visible to the singleton initialized
-    // via the SDK.
-    ;(globalThis as { __CODEBUFF_TREE_SITTER_WASM_BINARY__?: Uint8Array }).__CODEBUFF_TREE_SITTER_WASM_BINARY__ =
-      new Uint8Array(binary.buffer, binary.byteOffset, binary.byteLength)
-  } catch {
-    // readFileSync failure is unexpected (the file is supposed to be embedded)
-    // but we let init-node.ts fall back to path-based resolution and surface
-    // a clearer error if that also fails.
-  }
+if (TREE_SITTER_WASM_BASE64.length > 0) {
+  const buf = Buffer.from(TREE_SITTER_WASM_BASE64, 'base64')
+  // globalThis is the only cross-bundle channel: the SDK pre-built bundle
+  // inlines its own copy of `init-node.ts`, so a module-level variable in
+  // the source package isn't visible to the singleton initialized via the
+  // SDK. Slice into a fresh Uint8Array view instead of handing over the
+  // Buffer's shared underlying ArrayBuffer.
+  ;(
+    globalThis as { __CODEBUFF_TREE_SITTER_WASM_BINARY__?: Uint8Array }
+  ).__CODEBUFF_TREE_SITTER_WASM_BINARY__ = new Uint8Array(
+    buf.buffer,
+    buf.byteOffset,
+    buf.byteLength,
+  )
 }