--wip-- [skip ci]

Author: GuillaumeLagrange

.gitignore

@@ -132,3 +132,5 @@ packages/app/.env
 # turbo
 .turbo/
 .rollup.cache/
+
+.codspeed

CLAUDE.md

@@ -94,4 +94,25 @@ Based on the codebase analysis, to add stats access features:
   - Build: `pnpm turbo run build --filter=<package-name>`
   - Typecheck: `pnpm turbo run typecheck --filter=<package-name>`
   - Lint: `pnpm turbo run lint --filter=<package-name>`
-  - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`).
+  - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`).
+
+## Testing a plugin during development
+
+A plugin behaves differently depending on whether CodSpeed is driving the run,
+so exercise all three of these when developing or reviewing a plugin change
+(build the plugin first — the benches import from `dist`):
+
+1. **Fallback (not under CodSpeed).** No env vars. The plugin must stay out of
+   the way and let the framework run its benchmarks normally (no instrumentation,
+   no hijacked output). e.g. `pnpm turbo run bench --filter=<package-name>`.
+2. **Instrumentation / simulation.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=simulation`
+   (or `instrumentation`). The plugin hijacks the run to do a single instrumented
+   pass per benchmark and prints `Measured/Checked <uri>` instead of the normal
+   harness output.
+3. **Walltime.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=walltime`. The plugin
+   instruments the framework's real benchmark loop and collects walltime results.
+
+Running these locally outside the CodSpeed runner is expected to log
+`instrument-hooks: failed to write environment.json` and skip actual measurement
+writes — the point is to verify the plugin's control flow and output per mode,
+not to produce real measurements.

packages/vitest-plugin/benches/flat.bench.ts

@@ -1,4 +1,4 @@
-import { bench, describe } from "vitest";
+import { describe, test } from "vitest";
 import parsePr from "./parsePr";
 
 const LONG_BODY =
@@ -9,12 +9,16 @@ const LONG_BODY =
     .join("\n") + "fixes #123";
 
 describe("parsePr", () => {
-  bench("short body", () => {
-    parsePr({ body: "fixes #123", title: "test-1", number: 1 });
+  test("short body", async ({ bench }) => {
+    await bench("short body", () => {
+      parsePr({ body: "fixes #123", title: "test-1", number: 1 });
+    }).run();
   });
 
-  bench("long body", () => {
-    parsePr({ body: LONG_BODY, title: "test-2", number: 2 });
+  test("long body", async ({ bench }) => {
+    await bench("long body", () => {
+      parsePr({ body: LONG_BODY, title: "test-2", number: 2 });
+    }).run();
   });
 });
 
@@ -24,10 +28,14 @@ function fibo(n: number): number {
 }
 
 describe("fibo", () => {
-  bench("fibo 10", () => {
-    fibo(10);
+  test("fibo 10", async ({ bench }) => {
+    await bench("fibo 10", () => {
+      fibo(10);
+    }).run();
   });
-  bench("fibo 15", () => {
-    fibo(15);
+  test("fibo 15", async ({ bench }) => {
+    await bench("fibo 15", () => {
+      fibo(15);
+    }).run();
   });
 });

packages/vitest-plugin/benches/hooks.bench.ts

@@ -1,37 +1,42 @@
-import {
-  afterAll,
-  afterEach,
-  beforeAll,
-  beforeEach,
-  bench,
-  describe,
-  expect,
-} from "vitest";
+import { describe, expect, test } from "vitest";
 
+// Exercises tinybench's per-benchmark hooks, which Vitest 5 exposes through the
+// `bench(name, options, fn)` options object (`beforeAll`/`beforeEach`/...).
 describe("hooks", () => {
   let count = 0;
+
   describe("run", () => {
-    beforeAll(() => {
-      count += 10;
-    });
-    beforeEach(() => {
-      count += 1;
-    });
-    afterEach(() => {
-      count -= 1;
-    });
-    afterAll(() => {
-      count -= 10;
-    });
+    const hooks = {
+      beforeAll: () => {
+        count += 10;
+      },
+      beforeEach: () => {
+        count += 1;
+      },
+      afterEach: () => {
+        count -= 1;
+      },
+      afterAll: () => {
+        count -= 10;
+      },
+    };
 
-    bench("one", () => {
-      expect(count).toBe(11);
+    test("one", async ({ bench }) => {
+      await bench("one", hooks, () => {
+        expect(count).toBe(11);
+      }).run();
     });
-    bench("two", () => {
-      expect(count).toBe(11);
+
+    test("two", async ({ bench }) => {
+      await bench("two", hooks, () => {
+        expect(count).toBe(11);
+      }).run();
     });
   });
-  bench("end", () => {
-    expect(count).toBe(0);
+
+  test("end", async ({ bench }) => {
+    await bench("end", () => {
+      expect(count).toBe(0);
+    }).run();
   });
 });

packages/vitest-plugin/benches/macos.bench.ts

@@ -1,4 +1,4 @@
-import { bench, describe } from "vitest";
+import { describe, test } from "vitest";
 
 const isMacOS = process.platform === "darwin";
 
@@ -10,7 +10,9 @@ function fibo(n: number): number {
 // macOS-only benchmark: skipped on every other platform, so it only runs on
 // the `codspeed-walltime-macos` CI job (see .github/workflows/codspeed.yml).
 describe.skipIf(!isMacOS)("macos only", () => {
-  bench("fibo darwin", () => {
-    fibo(30);
+  test("fibo darwin", async ({ bench }) => {
+    await bench("fibo darwin", () => {
+      fibo(30);
+    }).run();
   });
 });

packages/vitest-plugin/benches/parsePr.bench.ts

@@ -1,4 +1,4 @@
-import { bench, describe } from "vitest";
+import { describe, test } from "vitest";
 import parsePr from "./parsePr";
 
 const LONG_BODY =
@@ -8,44 +8,52 @@ const LONG_BODY =
     )
     .join("\n") + "fixes #123";
 
+function benchShortBody() {
+  parsePr({ body: "fixes #123", title: "test", number: 124 });
+}
+
+function benchLongBody() {
+  parsePr({ body: LONG_BODY, title: "test", number: 124 });
+}
+
 describe("parsePr", () => {
-  bench("short body", () => {
-    parsePr({ body: "fixes #123", title: "test", number: 124 });
+  test("short body", async ({ bench }) => {
+    await bench("short body", benchShortBody).run();
   });
 
-  bench("long body", () => {
-    parsePr({ body: LONG_BODY, title: "test", number: 124 });
+  test("long body", async ({ bench }) => {
+    await bench("long body", benchLongBody).run();
   });
 
   describe("nested suite", () => {
-    bench("short body", () => {
-      parsePr({ body: "fixes #123", title: "test", number: 124 });
+    test("short body", async ({ bench }) => {
+      await bench("short body", benchShortBody).run();
     });
 
-    bench("long body", () => {
-      parsePr({ body: LONG_BODY, title: "test", number: 124 });
+    test("long body", async ({ bench }) => {
+      await bench("long body", benchLongBody).run();
     });
 
     describe("deeply nested suite", () => {
-      bench("short body", () => {
-        parsePr({ body: "fixes #123", title: "test", number: 124 });
+      test("short body", async ({ bench }) => {
+        await bench("short body", benchShortBody).run();
       });
     });
   });
 });
 
 describe("another parsePr", () => {
-  bench("short body", () => {
-    parsePr({ body: "fixes #123", title: "test", number: 124 });
+  test("short body", async ({ bench }) => {
+    await bench("short body", benchShortBody).run();
   });
 
-  bench("long body", () => {
-    parsePr({ body: LONG_BODY, title: "test", number: 124 });
+  test("long body", async ({ bench }) => {
+    await bench("long body", benchLongBody).run();
   });
 
   describe("nested suite", () => {
-    bench("short body", () => {
-      parsePr({ body: "fixes #123", title: "test", number: 124 });
+    test("short body", async ({ bench }) => {
+      await bench("short body", benchShortBody).run();
     });
   });
 });

packages/vitest-plugin/benches/timing.bench.ts

@@ -1,4 +1,4 @@
-import { bench, describe, type BenchOptions } from "vitest";
+import { describe, test, type BenchOptions } from "vitest";
 
 const busySleep = (ms: number): void => {
   const end = performance.now() + ms;
@@ -13,27 +13,21 @@ const timingBenchOptions: BenchOptions = {
 };
 
 describe("timing tests", () => {
-  bench(
-    "wait 1ms",
-    async () => {
+  test("wait 1ms", async ({ bench }) => {
+    await bench("wait 1ms", async () => {
       busySleep(1);
-    },
-    timingBenchOptions,
-  );
+    }).run(timingBenchOptions);
+  });
 
-  bench(
-    "wait 500ms",
-    async () => {
+  test("wait 500ms", async ({ bench }) => {
+    await bench("wait 500ms", async () => {
       busySleep(500);
-    },
-    timingBenchOptions,
-  );
+    }).run(timingBenchOptions);
+  });
 
-  bench(
-    "wait 1sec",
-    async () => {
+  test("wait 1sec", async ({ bench }) => {
+    await bench("wait 1sec", async () => {
       busySleep(1_000);
-    },
-    timingBenchOptions,
-  );
+    }).run(timingBenchOptions);
+  });
 });

packages/vitest-plugin/src/globalSetup.ts

@@ -9,13 +9,22 @@ function logCodSpeed(message: string) {
   console.log(`[CodSpeed] ${message}`);
 }
 
+let setupHappened = false;
 let teardownHappened = false;
 
+// TODO: Check if this can be avoided
+// Vitest 5 forks a dedicated `(bench)` project from the base one and runs
+// globalSetup for both it and the root project, so setup/teardown fire more than
+// once against this shared module. Log only the first pass and make the rest
+// no-ops rather than treating the repeat as an error.
 export default function () {
-  logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - setup`);
+  if (!setupHappened) {
+    setupHappened = true;
+    logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - setup`);
+  }
 
   return () => {
-    if (teardownHappened) throw new Error("teardown called twice");
+    if (teardownHappened) return;
     teardownHappened = true;
 
     logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - teardown`);