Support vitest 5

.gitignore

@@ -132,3 +132,5 @@ packages/app/.env
 # turbo
 .turbo/
 .rollup.cache/
+
+.codspeed

CLAUDE.md

@@ -94,4 +94,25 @@ Based on the codebase analysis, to add stats access features:
   - Build: `pnpm turbo run build --filter=<package-name>`
   - Typecheck: `pnpm turbo run typecheck --filter=<package-name>`
   - Lint: `pnpm turbo run lint --filter=<package-name>`
-  - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`).
+  - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`).
+
+## Testing a plugin during development
+
+A plugin behaves differently depending on whether CodSpeed is driving the run,
+so exercise all three of these when developing or reviewing a plugin change
+(build the plugin first — the benches import from `dist`):
+
+1. **Fallback (not under CodSpeed).** No env vars. The plugin must stay out of
+   the way and let the framework run its benchmarks normally (no instrumentation,
+   no hijacked output). e.g. `pnpm turbo run bench --filter=<package-name>`.
+2. **Instrumentation / simulation.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=simulation`
+   (or `instrumentation`). The plugin hijacks the run to do a single instrumented
+   pass per benchmark and prints `Measured/Checked <uri>` instead of the normal
+   harness output.
+3. **Walltime.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=walltime`. The plugin
+   instruments the framework's real benchmark loop and collects walltime results.
+
+Running these locally outside the CodSpeed runner is expected to log
+`instrument-hooks: failed to write environment.json` and skip actual measurement
+writes — the point is to verify the plugin's control flow and output per mode,
+not to produce real measurements.

examples/with-vitest-v4/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "with-vitest-v4",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "bench-vitest": "vitest bench --run"
+  },
+  "devDependencies": {
+    "@codspeed/vitest-plugin": "workspace:*",
+    "typescript": "^5.1.3",
+    "vitest": "^4.1.9"
+  }
+}

examples/with-vitest-v4/src/fibonacci.bench.ts

@@ -0,0 +1,20 @@
+import { bench, describe } from "vitest";
+import { iterativeFibonacci, recursiveFibonacci } from "./fibonacci";
+
+describe("fibonacci", () => {
+  bench("recursive fibo 15", () => {
+    recursiveFibonacci(15);
+  });
+
+  bench("recursive fibo 20", () => {
+    recursiveFibonacci(20);
+  });
+
+  bench("iterative fibo 15", () => {
+    iterativeFibonacci(15);
+  });
+
+  bench("iterative fibo 20", () => {
+    iterativeFibonacci(20);
+  });
+});

examples/with-vitest-v4/src/fibonacci.ts

@@ -0,0 +1,17 @@
+export function recursiveFibonacci(n: number): number {
+  if (n < 2) {
+    return n;
+  }
+  return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2);
+}
+
+export function iterativeFibonacci(n: number): number {
+  let a = 0;
+  let b = 1;
+  for (let i = 0; i < n; i++) {
+    const temp = a + b;
+    a = b;
+    b = temp;
+  }
+  return a;
+}

examples/with-vitest-v4/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "lib": ["es2023"],
+    "module": "ESNext",
+    "verbatimModuleSyntax": true,
+    "target": "es2022",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "moduleResolution": "Node"
+  }
+}

examples/with-vitest-v4/vitest.config.ts

@@ -0,0 +1,6 @@
+import codspeedPlugin from "@codspeed/vitest-plugin";
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  plugins: [codspeedPlugin()],
+});

packages/vitest-plugin/benches/flat.bench.ts

@@ -1,4 +1,4 @@
-import { bench, describe } from "vitest";
+import { describe, test } from "vitest";
 import parsePr from "./parsePr";
 
 const LONG_BODY =
@@ -9,12 +9,16 @@ const LONG_BODY =
     .join("\n") + "fixes #123";
 
 describe("parsePr", () => {
-  bench("short body", () => {
-    parsePr({ body: "fixes #123", title: "test-1", number: 1 });
+  test("short body", async ({ bench }) => {
+    await bench("short body", () => {
+      parsePr({ body: "fixes #123", title: "test-1", number: 1 });
+    }).run();
   });
 
-  bench("long body", () => {
-    parsePr({ body: LONG_BODY, title: "test-2", number: 2 });
+  test("long body", async ({ bench }) => {
+    await bench("long body", () => {
+      parsePr({ body: LONG_BODY, title: "test-2", number: 2 });
+    }).run();
   });
 });
 
@@ -24,10 +28,14 @@ function fibo(n: number): number {
 }
 
 describe("fibo", () => {
-  bench("fibo 10", () => {
-    fibo(10);
+  test("fibo 10", async ({ bench }) => {
+    await bench("fibo 10", () => {
+      fibo(10);
+    }).run();
   });
-  bench("fibo 15", () => {
-    fibo(15);
+  test("fibo 15", async ({ bench }) => {
+    await bench("fibo 15", () => {
+      fibo(15);
+    }).run();
   });
 });

packages/vitest-plugin/benches/hooks.bench.ts

@@ -1,37 +1,42 @@
-import {
-  afterAll,
-  afterEach,
-  beforeAll,
-  beforeEach,
-  bench,
-  describe,
-  expect,
-} from "vitest";
+import { describe, expect, test } from "vitest";
 
+// Exercises tinybench's per-benchmark hooks, which Vitest 5 exposes through the
+// `bench(name, options, fn)` options object (`beforeAll`/`beforeEach`/...).
 describe("hooks", () => {
   let count = 0;
+
   describe("run", () => {
-    beforeAll(() => {
-      count += 10;
-    });
-    beforeEach(() => {
-      count += 1;
-    });
-    afterEach(() => {
-      count -= 1;
-    });
-    afterAll(() => {
-      count -= 10;
-    });
+    const hooks = {
+      beforeAll: () => {
+        count += 10;
+      },
+      beforeEach: () => {
+        count += 1;
+      },
+      afterEach: () => {
+        count -= 1;
+      },
+      afterAll: () => {
+        count -= 10;
+      },
+    };
 
-    bench("one", () => {
-      expect(count).toBe(11);
+    test("one", async ({ bench }) => {
+      await bench("one", hooks, () => {
+        expect(count).toBe(11);
+      }).run();
     });
-    bench("two", () => {
-      expect(count).toBe(11);
+
+    test("two", async ({ bench }) => {
+      await bench("two", hooks, () => {
+        expect(count).toBe(11);
+      }).run();
     });
   });
-  bench("end", () => {
-    expect(count).toBe(0);
+
+  test("end", async ({ bench }) => {
+    await bench("end", () => {
+      expect(count).toBe(0);
+    }).run();
   });
 });

packages/vitest-plugin/benches/macos.bench.ts

@@ -1,4 +1,4 @@
-import { bench, describe } from "vitest";
+import { describe, test } from "vitest";
 
 const isMacOS = process.platform === "darwin";
 
@@ -10,7 +10,9 @@ function fibo(n: number): number {
 // macOS-only benchmark: skipped on every other platform, so it only runs on
 // the `codspeed-walltime-macos` CI job (see .github/workflows/codspeed.yml).
 describe.skipIf(!isMacOS)("macos only", () => {
-  bench("fibo darwin", () => {
-    fibo(30);
+  test("fibo darwin", async ({ bench }) => {
+    await bench("fibo darwin", () => {
+      fibo(30);
+    }).run();
   });
 });

packages/vitest-plugin/benches/parsePr.bench.ts

@@ -1,4 +1,4 @@
-import { bench, describe } from "vitest";
+import { describe, test } from "vitest";
 import parsePr from "./parsePr";
 
 const LONG_BODY =
@@ -8,44 +8,52 @@ const LONG_BODY =
     )
     .join("\n") + "fixes #123";
 
+function benchShortBody() {
+  parsePr({ body: "fixes #123", title: "test", number: 124 });
+}
+
+function benchLongBody() {
+  parsePr({ body: LONG_BODY, title: "test", number: 124 });
+}
+
 describe("parsePr", () => {
-  bench("short body", () => {
-    parsePr({ body: "fixes #123", title: "test", number: 124 });
+  test("short body", async ({ bench }) => {
+    await bench("short body", benchShortBody).run();
   });
 
-  bench("long body", () => {
-    parsePr({ body: LONG_BODY, title: "test", number: 124 });
+  test("long body", async ({ bench }) => {
+    await bench("long body", benchLongBody).run();
   });
 
   describe("nested suite", () => {
-    bench("short body", () => {
-      parsePr({ body: "fixes #123", title: "test", number: 124 });
+    test("short body", async ({ bench }) => {
+      await bench("short body", benchShortBody).run();
     });
 
-    bench("long body", () => {
-      parsePr({ body: LONG_BODY, title: "test", number: 124 });
+    test("long body", async ({ bench }) => {
+      await bench("long body", benchLongBody).run();
     });
 
     describe("deeply nested suite", () => {
-      bench("short body", () => {
-        parsePr({ body: "fixes #123", title: "test", number: 124 });
+      test("short body", async ({ bench }) => {
+        await bench("short body", benchShortBody).run();
       });
     });
   });
 });
 
 describe("another parsePr", () => {
-  bench("short body", () => {
-    parsePr({ body: "fixes #123", title: "test", number: 124 });
+  test("short body", async ({ bench }) => {
+    await bench("short body", benchShortBody).run();
   });
 
-  bench("long body", () => {
-    parsePr({ body: LONG_BODY, title: "test", number: 124 });
+  test("long body", async ({ bench }) => {
+    await bench("long body", benchLongBody).run();
   });
 
   describe("nested suite", () => {
-    bench("short body", () => {
-      parsePr({ body: "fixes #123", title: "test", number: 124 });
+    test("short body", async ({ bench }) => {
+      await bench("short body", benchShortBody).run();
     });
   });
 });

packages/vitest-plugin/benches/timing.bench.ts

@@ -1,4 +1,4 @@
-import { bench, describe, type BenchOptions } from "vitest";
+import { describe, test, type BenchOptions } from "vitest";
 
 const busySleep = (ms: number): void => {
   const end = performance.now() + ms;
@@ -13,27 +13,21 @@ const timingBenchOptions: BenchOptions = {
 };
 
 describe("timing tests", () => {
-  bench(
-    "wait 1ms",
-    async () => {
+  test("wait 1ms", async ({ bench }) => {
+    await bench("wait 1ms", async () => {
       busySleep(1);
-    },
-    timingBenchOptions,
-  );
+    }).run(timingBenchOptions);
+  });
 
-  bench(
-    "wait 500ms",
-    async () => {
+  test("wait 500ms", async ({ bench }) => {
+    await bench("wait 500ms", async () => {
       busySleep(500);
-    },
-    timingBenchOptions,
-  );
+    }).run(timingBenchOptions);
+  });
 
-  bench(
-    "wait 1sec",
-    async () => {
+  test("wait 1sec", async ({ bench }) => {
+    await bench("wait 1sec", async () => {
       busySleep(1_000);
-    },
-    timingBenchOptions,
-  );
+    }).run(timingBenchOptions);
+  });
 });

packages/vitest-plugin/package.json

@@ -41,13 +41,13 @@
   "peerDependencies": {
     "tinybench": ">=2.9.0",
     "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0",
-    "vitest": "^3.2 || ^4"
+    "vitest": "^3.2 || ^4 || ^5.0.0-beta"
   },
   "devDependencies": {
     "@total-typescript/shoehorn": "^0.1.1",
     "execa": "^8.0.1",
     "tinybench": "^2.9.0",
     "vite": "^7.0.0",
-    "vitest": "^4.0.18"
+    "vitest": "5.0.0-beta.5"
   }
 }