feat(telemetry): give every failure an error code and report where a run stopped

.changeset/init-telemetry-stages.md

@@ -0,0 +1,6 @@
+---
+"clerk": minor
+---
+
+Give every failure a specific error code, and record which step a multi-step command reached.
+Agent-mode JSON now carries a code for failures that previously reported only a message, and `clerk init` reports the step it stopped at rather than a single generic failure.

README.md

@@ -60,14 +60,14 @@ Commands:
 
 ## Telemetry
 
-The Clerk CLI collects usage telemetry: command name, flag names, duration, outcome,
-environment signals (OS, install method, terminal), a random machine identifier — and
-your workspace and app IDs when a project is linked. It never collects command
-arguments, option values, file paths, or personal data. The first run only shows a
-disclosure notice and sends nothing (CI environments send from the first run), and
-`clerk --verbose` prints every event before it is sent. Shell completion (`clerk
-completion <shell>` and the `__complete` helper behind Tab) sends nothing at all.
-See https://clerk.com/docs/telemetry for details.
+The Clerk CLI collects usage telemetry: command name, flag names, duration, outcome, the
+step a multi-step command reached, environment signals (OS, install method, terminal), a
+random machine identifier — and your workspace and app IDs when a project is linked. It
+never collects command arguments, option values, file paths, or personal data. The first
+run only shows a disclosure notice and sends nothing (CI environments send from the
+first run), and `clerk --verbose` prints every event before it is sent. Shell completion
+(`clerk completion <shell>` and the `__complete` helper behind Tab) sends nothing at
+all. See https://clerk.com/docs/telemetry for details.
 
 Opt out with `clerk telemetry disable`, or by setting `CLERK_TELEMETRY_DISABLED=1`
 (the standard `DO_NOT_TRACK=1` also works). `clerk telemetry status` shows the

packages/cli-core/src/commands/deploy/index.ts

@@ -187,6 +187,7 @@ async function startNewDeploy(ctx: DeployContext): Promise<void> {
     throw new CliError(
       "Production instance was created but Clerk did not return a domain. " +
         "Run `clerk deploy` again to retry domain provisioning.",
+      { code: ERROR_CODE.DEPLOY_DOMAIN_MISSING },
     );
   }
 

packages/cli-core/src/commands/env/pull.ts

@@ -147,6 +147,7 @@ async function pullKeylessKeys(
       throw new CliError(
         `The publishable key found locally doesn't belong to the application the secret key from \`${keyless.source}\` addresses. Writing this pair would leave the server trusting one app while the browser talks to another.\n` +
           `Remove the mismatched ${publishableKeyName} from your env files, or run \`clerk auth login\` to claim the intended application, then pull again.`,
+        { code: ERROR_CODE.KEY_PAIR_MISMATCH },
       );
     }
   }

packages/cli-core/src/commands/init/bootstrap.ts

@@ -2,7 +2,7 @@ import { join } from "node:path";
 import { statSync } from "node:fs";
 import { confirm, text } from "../../lib/prompts.ts";
 import { search, filterChoices } from "../../lib/listage.ts";
-import { throwUserAbort, throwUsageError, CliError } from "../../lib/errors.js";
+import { throwUserAbort, throwUsageError, CliError, ERROR_CODE } from "../../lib/errors.js";
 import { log } from "../../lib/log.js";
 import type { FrameworkInfo } from "../../lib/framework.js";
 import { dirExists, hasPackageJson } from "./context.js";
@@ -49,6 +49,7 @@ async function pickFramework(frameworkOverride?: FrameworkInfo): Promise<Bootstr
   const supported = BOOTSTRAP_REGISTRY.map((e) => e.label).join(", ");
   throw new CliError(
     `Bootstrap is not supported for ${frameworkOverride.name}. Supported: ${supported}`,
+    { code: ERROR_CODE.BOOTSTRAP_UNSUPPORTED },
   );
 }
 
@@ -109,7 +110,9 @@ export async function findAvailableProjectName(cwd: string, base: string): Promi
     const candidate = `${base}-${i}`;
     if (!(await dirExists(join(cwd, candidate)))) return candidate;
   }
-  throw new CliError(`Could not find an available project name based on '${base}'.`);
+  throw new CliError(`Could not find an available project name based on '${base}'.`, {
+    code: ERROR_CODE.PROJECT_DIR_EXISTS,
+  });
 }
 
 async function askProjectName(entry: BootstrapEntry, cwd: string): Promise<string> {
@@ -137,7 +140,9 @@ async function generateProject(label: string, command: string[], cwd: string): P
 
   const exitCode = await spawnInherited(command, cwd);
   if (exitCode !== 0) {
-    throw new CliError(`Project generation failed (exit code ${exitCode}).`);
+    throw new CliError(`Project generation failed (exit code ${exitCode}).`, {
+      code: ERROR_CODE.GENERATOR_FAILED,
+    });
   }
 }
 
@@ -231,13 +236,16 @@ export async function promptAndBootstrap(
   if (await dirExists(projectDir)) {
     throw new CliError(
       `Directory '${projectName}' already exists. Pick a different name or remove it first.`,
+      { code: ERROR_CODE.PROJECT_DIR_EXISTS },
     );
   }
 
   await generateProject(entry.label, entry.buildCommand(pm, projectName), cwd);
 
   if (!(await hasPackageJson(projectDir))) {
-    throw new CliError("Generator did not create a package.json.");
+    throw new CliError("Generator did not create a package.json.", {
+      code: ERROR_CODE.GENERATOR_FAILED,
+    });
   }
 
   await installDependencies(pm, projectDir);

packages/cli-core/src/commands/init/index.test.ts

@@ -19,7 +19,10 @@ import {
   skillsMod,
   bootstrapMod,
   nextStepsMod,
+  mockExistingProject,
+  mockMiddlewareScaffold,
 } from "../../test/lib/init-harness.ts";
+import * as telemetryMod from "../../lib/telemetry.ts";
 import { init } from "./index.ts";
 
 describe("init", () => {
@@ -584,4 +587,85 @@ describe("init", () => {
       cwd: FAKE_BOOTSTRAP.projectDir,
     });
   });
+
+  describe("telemetry stages", () => {
+    /** Spy registered with the harness so its calls reset between tests. */
+    function trackStages() {
+      const stage = spyOn(telemetryMod, "setTelemetryStage");
+      track(stage);
+      return () => stage.mock.calls.map((call) => call[0]);
+    }
+
+    test("a completed run reports the terminal stage", async () => {
+      setup({ email: "test@test.com" });
+      mockExistingProject(FAKE_CTX);
+      mockMiddlewareScaffold();
+      const stages = trackStages();
+
+      await init({ yes: true });
+
+      expect(stages().at(-1)).toBe("done");
+    });
+
+    test("a run with nothing to do stops at already_set_up", async () => {
+      setup({ email: "test@test.com" });
+      mockExistingProject(FAKE_CTX);
+      const stages = trackStages();
+
+      await init({ yes: true });
+
+      expect(stages().at(-1)).toBe("already_set_up");
+    });
+
+    // A run that dies in flag validation must not claim any later stage —
+    // that's what makes the funnel readable.
+    test("a rejected flag combination stops at the flags stage", async () => {
+      setup({ email: "test@test.com" });
+      const stages = trackStages();
+
+      await expect(init({ keyless: true, login: true })).rejects.toThrow();
+
+      expect(stages()).toEqual(["flags"]);
+    });
+
+    test("declining the scaffold preview stops at the scaffold stage", async () => {
+      setup({ email: "test@test.com" });
+      mockExistingProject(FAKE_CTX);
+      mockMiddlewareScaffold();
+      track(spyOn(previewMod, "previewAndConfirm").mockResolvedValue(false));
+      const stages = trackStages();
+
+      await expect(init({})).rejects.toThrow();
+
+      expect(stages().at(-1)).toBe("scaffold");
+    });
+
+    // Declining the overwrite prompt is the default answer on --starter, so it
+    // is a common drop-off — and it happens before bootstrapAndDetect runs.
+    test("declining the starter overwrite prompt stops at the bootstrap stage", async () => {
+      setup({ email: "test@test.com" });
+      spyOn(context, "gatherContext").mockResolvedValue(FAKE_CTX);
+      spyOn(config, "resolveProfile").mockResolvedValue({ profile: { appId: "app_123" } } as never);
+      track(
+        spyOn(bootstrapMod, "confirmOverwrite").mockRejectedValue(
+          Object.assign(new Error(), { name: "UserAbortError" }),
+        ),
+      );
+      const stages = trackStages();
+
+      await expect(init({ starter: true })).rejects.toMatchObject({ name: "UserAbortError" });
+
+      expect(stages().at(-1)).toBe("bootstrap");
+    });
+
+    test("a failure inside the generator stops at the bootstrap stage", async () => {
+      setup();
+      track(spyOn(bootstrapMod, "promptAndBootstrap").mockRejectedValue(new Error("gen failed")));
+      const stages = trackStages();
+
+      await expect(init({})).rejects.toThrow();
+
+      expect(stages().at(-1)).toBe("bootstrap");
+    });
+  });
 });

packages/cli-core/src/commands/init/index.ts

@@ -5,7 +5,13 @@ import { link } from "../link/index.js";
 import { pull } from "../env/pull.js";
 import { isAgent } from "../../mode.js";
 import { dim, bold } from "../../lib/color.js";
-import { throwUserAbort, throwUsageError, CliError, errorMessage } from "../../lib/errors.js";
+import {
+  throwUserAbort,
+  throwUsageError,
+  CliError,
+  ERROR_CODE,
+  errorMessage,
+} from "../../lib/errors.js";
 import {
   lookupFramework,
   isNpmFramework,
@@ -15,6 +21,7 @@ import {
 import { resolveProfile } from "../../lib/config.js";
 import { deriveProjectName } from "../../lib/project-name.js";
 import { log } from "../../lib/log.js";
+import { setTelemetryStage } from "../../lib/telemetry.js";
 import { confirm } from "../../lib/prompts.ts";
 import {
   createAccountlessApp,
@@ -81,6 +88,7 @@ export async function init(options: InitOptions = {}) {
   const cwd = process.cwd();
   const agent = isAgent();
 
+  setTelemetryStage("flags");
   await assertUsableFlags(options, agent);
 
   const frameworkOverride = options.framework
@@ -96,6 +104,7 @@ export async function init(options: InitOptions = {}) {
 
   intro("Setting up Clerk");
 
+  setTelemetryStage("detect");
   const resolved = options.starter
     ? await handleStarter(cwd, frameworkOverride, overrides)
     : await resolveProjectContext(cwd, frameworkOverride, overrides);
@@ -119,6 +128,7 @@ export async function init(options: InitOptions = {}) {
   // stale/broken credential ends up blocked on an interactive browser OAuth
   // round-trip it can never complete. So agent mode validates the credential
   // (it can fall back to keyless) instead of trusting mere presence.
+  setTelemetryStage("strategy");
   const authed = optsKeyless
     ? false
     : agent
@@ -141,6 +151,7 @@ export async function init(options: InitOptions = {}) {
   assertKeylessOnlyFlags(options, strategy);
 
   if (strategy === "authenticate") {
+    setTelemetryStage("link");
     bar();
     const createIfMissing = agent
       ? await deriveProjectName(ctx.cwd, bootstrap?.projectName)
@@ -156,6 +167,7 @@ export async function init(options: InitOptions = {}) {
   const { alreadySetUp } = await detectAndInstall(ctx.cwd, ctx, skipScaffoldConfirm);
 
   if (alreadySetUp) {
+    setTelemetryStage("already_set_up");
     log.success("\nClerk is already set up in this project.");
     if (agent && strategy === "manual") {
       printBootstrapManualSetupInfo(ctx.framework);
@@ -164,6 +176,7 @@ export async function init(options: InitOptions = {}) {
     return;
   }
 
+  setTelemetryStage("keys");
   bar();
   await runStrategy(strategy, ctx, {
     template: options.template,
@@ -173,6 +186,7 @@ export async function init(options: InitOptions = {}) {
 
   // Native platforms (iOS/Android) have no npx/Node toolchain to run `skills add` with.
   if (options.skills !== false && isNpmFramework(ctx.framework)) {
+    setTelemetryStage("skills");
     bar();
     await installSkills(ctx.cwd, ctx.framework.dep, ctx.packageManager, overrides.skipConfirm);
   }
@@ -183,6 +197,7 @@ export async function init(options: InitOptions = {}) {
     printBootstrapNextSteps(bootstrap, strategy === "keyless");
   }
 
+  setTelemetryStage("done");
   await outro("Done");
 }
 
@@ -275,11 +290,14 @@ async function bootstrapAndDetect(
   frameworkOverride: FrameworkInfo | undefined,
   overrides: BootstrapOverrides,
 ): Promise<ResolvedContext> {
+  setTelemetryStage("bootstrap");
   const bootstrap = await promptAndBootstrap(cwd, frameworkOverride, overrides);
 
   const ctx = await gatherContext(bootstrap.projectDir);
   if (!ctx) {
-    throw new CliError("Project generation did not produce a detectable framework.");
+    throw new CliError("Project generation did not produce a detectable framework.", {
+      code: ERROR_CODE.FRAMEWORK_UNDETECTED,
+    });
   }
   return { ctx, bootstrap };
 }
@@ -289,6 +307,7 @@ async function handleStarter(
   frameworkOverride: FrameworkInfo | undefined,
   overrides: BootstrapOverrides,
 ): Promise<ResolvedContext> {
+  setTelemetryStage("bootstrap");
   if (!overrides.skipConfirm) {
     await confirmOverwrite(cwd);
   }
@@ -323,6 +342,7 @@ async function resolveProjectContext(
   if (!isBlank) {
     throw new CliError(
       `Could not detect a framework. Install the appropriate Clerk SDK manually: https://clerk.com/docs`,
+      { code: ERROR_CODE.FRAMEWORK_UNDETECTED },
     );
   }
 
@@ -559,11 +579,13 @@ async function detectAndInstall(
   if (ctx.existingClerk) {
     log.info(dim(`${ctx.framework.sdk} is already installed`));
   } else if (isNpmFramework(ctx.framework)) {
+    setTelemetryStage("install");
     await installSdk(ctx);
   }
   // Non-npm ecosystems (Swift Package Manager, Gradle) can't be installed by a
   // package manager here — the framework's scaffold plan prints install steps.
 
+  setTelemetryStage("scaffold");
   return scaffoldAndWrite(cwd, ctx, skipConfirm);
 }
 

packages/cli-core/src/commands/switch-env/index.ts

@@ -16,7 +16,7 @@ import {
   isValidEnv,
   setCurrentEnv,
 } from "../../lib/environment.ts";
-import { CliError } from "../../lib/errors.ts";
+import { CliError, ERROR_CODE } from "../../lib/errors.ts";
 import { log } from "../../lib/log.ts";
 import { isHuman } from "../../mode.ts";
 import { select } from "../../lib/listage.ts";
@@ -44,6 +44,7 @@ export async function switchEnv(environmentArg: string | undefined): Promise<voi
     } else if (isHuman() && available.length > 1 && !process.stdin.isTTY) {
       throw new CliError(
         "No interactive terminal available — pass an environment name explicitly: `clerk switch-env <name>`",
+        { code: ERROR_CODE.NO_INTERACTIVE_TERMINAL },
       );
     } else if (available.length <= 1) {
       log.info(`Current environment: ${current}`);
@@ -61,6 +62,7 @@ export async function switchEnv(environmentArg: string | undefined): Promise<voi
   if (!isValidEnv(target)) {
     throw new CliError(
       `Unknown environment "${target}". Available environments: ${available.join(", ")}`,
+      { code: ERROR_CODE.INVALID_ENVIRONMENT },
     );
   }