feat(webapp): enforce watch plan limits

.server-changes/agent-watch-plan-limits.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: feature
+---
+
+Watches now respect your plan's limits: free plans can run a limited number of watches at once and for a shorter window, with a prompt to upgrade for more.

.server-changes/dashboard-agent-investigation-sweep-backoff.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+A single stuck assistant investigation can no longer hold up others from being tidied away.

apps/webapp/app/routes/api.v1.dashboard-agent.watches.ts

@@ -111,7 +111,9 @@ export async function action({ request }: ActionFunctionArgs) {
 
     if (!result.ok) {
       const status =
-        result.code === "limit_reached" || result.code === "duplicate"
+        result.code === "limit_reached" ||
+        result.code === "watch_limit_reached" ||
+        result.code === "duplicate"
           ? 409
           : result.code === "invalid_target"
             ? 404

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts

@@ -545,6 +545,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
     if (!result.ok) {
       const status =
         result.code === "limit_reached" ||
+        result.code === "watch_limit_reached" ||
         result.code === "duplicate" ||
         result.code === "request_conflict"
           ? 409

apps/webapp/app/services/dashboardAgentInvestigationSweep.server.ts

@@ -5,8 +5,11 @@
 
 import {
   listStaleOpenInvestigations,
+  recordInvestigationSweepAttempt,
   settleInvestigationAndCloseCard,
+  settleInvestigationAsInconclusive,
   type Investigation,
+  type SettledInvestigation,
   type SettledInvestigationCard,
 } from "@internal/dashboard-agent-db";
 import { UNSETTLED_INVESTIGATION_NOTE } from "@internal/dashboard-agent-contracts";
@@ -22,6 +25,13 @@ export const INVESTIGATION_STALE_MS = 30 * 60 * 1000;
 /** Per-run cap. Oldest first, so the rest land next run. */
 const SWEEP_BATCH_LIMIT = 100;
 
+/**
+ * After this many failed settle attempts a row is force-abandoned: settled `inconclusive`
+ * WITHOUT the closing card, so a card that never renders leaves the queue instead of
+ * looping forever. The rare stuck spinner is the price of not starving every other row.
+ */
+export const MAX_SWEEP_ATTEMPTS = 5;
+
 export type InvestigationSweepResult = {
   /** Stale `in_progress` rows seen. */
   stale: number;
@@ -30,6 +40,8 @@ export type InvestigationSweepResult = {
   closed: number;
   /** A turn (or another sweep) settled it first. */
   alreadySettled: number;
+  /** Rows past the attempt cap, force-settled without a card so they leave the queue. */
+  abandoned: number;
   failed: number;
 };
 
@@ -46,6 +58,10 @@ export type InvestigationSweepDeps = {
     chatId: string;
     note: string;
   }) => Promise<SettledInvestigationCard | null>;
+  /** Record a failed settle out-of-band; returns the new attempt count, or null if gone. */
+  recordAttempt?: (params: { id: string }) => Promise<number | null>;
+  /** Force a poison row terminal without the failing render path. */
+  forceAbandon?: (params: { id: string; note: string }) => Promise<SettledInvestigation | null>;
 };
 
 /**
@@ -61,12 +77,17 @@ export async function sweepDashboardAgentInvestigations(
     deps.listStale ?? ((params) => listStaleOpenInvestigations(dashboardAgentDb, params));
   const settleAndClose =
     deps.settleAndClose ?? ((params) => settleInvestigationAndCloseCard(dashboardAgentDb, params));
+  const recordAttempt =
+    deps.recordAttempt ?? ((params) => recordInvestigationSweepAttempt(dashboardAgentDb, params));
+  const forceAbandon =
+    deps.forceAbandon ?? ((params) => settleInvestigationAsInconclusive(dashboardAgentDb, params));
 
   const result: InvestigationSweepResult = {
     stale: 0,
     settled: 0,
     closed: 0,
     alreadySettled: 0,
+    abandoned: 0,
     failed: 0,
   };
 
@@ -93,10 +114,49 @@ export async function sweepDashboardAgentInvestigations(
       result.settled++;
       if (outcome.closed) result.closed++;
     } catch (error) {
+      // The settle rolled back, so the row is still `in_progress`. Record the attempt in
+      // its own write — this rotates the row to the back of the sweep order (see
+      // `listStaleOpenInvestigations`) so it can't pin the head and starve newer rows.
+      let attempts: number | null = null;
+      try {
+        attempts = await recordAttempt({ id: investigation.id });
+      } catch (recordError) {
+        logger.error("Dashboard agent investigation sweep: failed to record a sweep attempt", {
+          investigationId: investigation.id,
+          chatId: investigation.chatId,
+          error: recordError,
+        });
+      }
+
+      // Past the cap the card will never render; force it terminal without the render
+      // path so it leaves the queue instead of looping forever.
+      if (attempts !== null && attempts >= MAX_SWEEP_ATTEMPTS) {
+        try {
+          await forceAbandon({ id: investigation.id, note: UNSETTLED_INVESTIGATION_NOTE });
+          result.abandoned++;
+          logger.warn(
+            "Dashboard agent investigation sweep: abandoned a card past the attempt cap",
+            {
+              investigationId: investigation.id,
+              chatId: investigation.chatId,
+              attempts,
+            }
+          );
+          continue;
+        } catch (abandonError) {
+          logger.error("Dashboard agent investigation sweep: failed to abandon a poison card", {
+            investigationId: investigation.id,
+            chatId: investigation.chatId,
+            error: abandonError,
+          });
+        }
+      }
+
       result.failed++;
       logger.error("Dashboard agent investigation sweep: failed to settle an investigation", {
         investigationId: investigation.id,
         chatId: investigation.chatId,
+        attempts,
         error,
       });
     }

apps/webapp/app/services/dashboardAgentWatchLimits.server.ts

@@ -0,0 +1,53 @@
+import type { Limits } from "@trigger.dev/platform";
+import { WATCH_MAX_HOURS } from "@internal/dashboard-agent-contracts";
+import { getCachedLimit, isBillingConfigured } from "./platform.v3.server";
+
+// The unlimited sentinel, matching the message quota (TRI-12863 P1). Never Infinity: it
+// serializes to null in the limit cache.
+export const UNLIMITED_WATCH_LIMIT = 100_000_000;
+
+// Filled by cloud billing (TRI-12863 P0). Absent until then, and always on self-hosted, so
+// the fallback applies and the plan floor is off.
+const WATCH_MAX_HOURS_LIMIT_KEY = "agentWatchMaxHours" as keyof Limits;
+const WATCH_COUNT_LIMIT_KEY = "agentWatchers" as keyof Limits;
+
+export type WatchPlanLimits = {
+  /** Longest window one watch may run for, in hours. */
+  maxHours: number;
+  /** How many active watches the org may run at once. */
+  watchers: number;
+};
+
+async function readLimit(organizationId: string, key: keyof Limits): Promise<number> {
+  const cached = await getCachedLimit(organizationId, key, UNLIMITED_WATCH_LIMIT);
+  // A cache error leaves `val` empty; fall open to unlimited.
+  return cached.val ?? UNLIMITED_WATCH_LIMIT;
+}
+
+/**
+ * The org's plan floors for watches. Fails open: an absent limit (self-hosted, or before the
+ * cloud side ships) resolves to the unlimited sentinel, so neither floor bites.
+ */
+export async function resolveWatchPlanLimits(organizationId: string): Promise<WatchPlanLimits> {
+  const [maxHours, watchers] = await Promise.all([
+    readLimit(organizationId, WATCH_MAX_HOURS_LIMIT_KEY),
+    readLimit(organizationId, WATCH_COUNT_LIMIT_KEY),
+  ]);
+  return { maxHours, watchers };
+}
+
+/**
+ * The window ceiling actually in force: the plan floor under the code ceiling. A plan that
+ * allows 100 hours still caps at {@link WATCH_MAX_HOURS}.
+ */
+export function effectiveWatchMaxHours(planMaxHours: number): number {
+  return Math.min(planMaxHours, WATCH_MAX_HOURS);
+}
+
+/**
+ * A watch-limit refusal, plus an upgrade nudge when billing is present. Self-hosted never
+ * hits this (fails open above), and the nudge is gated so a stray refusal stays quiet there.
+ */
+export function watchLimitHint(base: string, billingConfigured = isBillingConfigured()): string {
+  return billingConfigured ? `${base} Upgrade your plan for more.` : base;
+}

apps/webapp/app/services/dashboardAgentWatches.server.ts

@@ -10,6 +10,7 @@ import {
   cancelWatch,
   chatExists,
   claimWatchSubmission,
+  countActiveWatchesForOrg,
   createChat,
   createWatch,
   generateWatchId,
@@ -68,6 +69,12 @@ import {
 import { watchCreationCheckDeps } from "~/services/dashboardAgentWatchChecks.server";
 import { normalizeErrorFingerprint } from "~/services/dashboardAgentWatchErrorChecks";
 import { subscribeUserToWatchAlerts } from "~/services/dashboardAgentWatchAlerts.server";
+import {
+  effectiveWatchMaxHours,
+  resolveWatchPlanLimits,
+  watchLimitHint,
+  type WatchPlanLimits,
+} from "~/services/dashboardAgentWatchLimits.server";
 import {
   mintDashboardAgentWatchBatchToken,
   mintDashboardAgentWatchToken,
@@ -165,6 +172,7 @@ export async function authorizeWatchEnvironmentById(params: {
 
 export type CreateWatchErrorCode =
   | "limit_reached"
+  | "watch_limit_reached"
   | "duplicate"
   | "invalid_target"
   | "chat_not_found"
@@ -273,6 +281,12 @@ export async function createDashboardAgentWatch(params: {
     scheduleTick?: typeof scheduleWatchTick;
     /** Skip the real trigger-config gate when a tick scheduler is injected. */
     configured?: () => boolean;
+    /** Plan floors on window and count. Fails open to unlimited when absent. */
+    resolveLimits?: (organizationId: string) => Promise<WatchPlanLimits>;
+    /** Org-wide active-watch count, for the watcher-count floor. */
+    countActiveWatches?: (organizationId: string) => Promise<number>;
+    /** Gates the upgrade nudge, so self-hosted stays quiet. */
+    billingConfigured?: () => boolean;
   };
 }): Promise<CreateDashboardAgentWatchResult> {
   const { environment, userId, chatId } = params;
@@ -284,6 +298,11 @@ export async function createDashboardAgentWatch(params: {
   const buildCheckDeps = params.deps?.checkDeps ?? watchCreationCheckDeps;
   const scheduleTick = params.deps?.scheduleTick ?? scheduleWatchTick;
   const isDashboardAgentConfigured = params.deps?.configured ?? isDashboardAgentConfiguredDefault;
+  const resolveLimits = params.deps?.resolveLimits ?? resolveWatchPlanLimits;
+  const countActiveWatches =
+    params.deps?.countActiveWatches ??
+    ((organizationId: string) => countActiveWatchesForOrg(dashboardAgentDb, { organizationId }));
+  const hint = (base: string) => watchLimitHint(base, params.deps?.billingConfigured?.());
   const checkDeps = buildCheckDeps(environment, now);
 
   if (!isDashboardAgentConfigured()) {
@@ -317,6 +336,17 @@ export async function createDashboardAgentWatch(params: {
   });
   if (!precheck.ok) return creationGuardrailError(precheck);
 
+  // Plan floors sit below the code ceilings (min(plan, ceiling)). Fails open: an absent
+  // limit resolves to unlimited, so neither floor bites on self-hosted.
+  const planLimits = await resolveLimits(environment.organizationId);
+  if (spec.maxHours > effectiveWatchMaxHours(planLimits.maxHours)) {
+    return {
+      ok: false,
+      code: "watch_limit_reached",
+      error: hint("That watch window is longer than your plan allows."),
+    };
+  }
+
   // `since` is server-set so the model can't backdate a recurrence window.
   const persistedSpec: PersistedWatchSpec =
     spec.kind === "error_recurrence" ? { ...spec, since: now.toISOString() } : spec;
@@ -331,6 +361,18 @@ export async function createDashboardAgentWatch(params: {
     return { ok: true, watching: false, identity, immediate };
   }
 
+  // Counted only now the immediate check didn't answer: a one-shot creates no row and so
+  // consumes no watcher slot. The per-chat cap of 3 still applies independently, in
+  // `createWatch`.
+  const activeCount = await countActiveWatches(environment.organizationId);
+  if (activeCount >= planLimits.watchers) {
+    return {
+      ok: false,
+      code: "watch_limit_reached",
+      error: hint("You've reached the number of active watches your plan allows."),
+    };
+  }
+
   const expiresAt = new Date(now.getTime() + spec.maxHours * 60 * 60 * 1000);
 
   const created = await createWatch(dashboardAgentDb, {