fix(redis-worker): stop fair queue leaking concurrency slots

.changeset/fair-queue-concurrency-slot-leak.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/redis-worker": patch
+---
+
+Fair queue consumers no longer leak the concurrency slots that gate a tenant's throughput. Slots were held by messages that had already finished, were never reclaimed, and once enough of them accumulated every queue belonging to that tenant stopped being served. Slots are now freed on the paths that previously skipped them, freed before the record needed to recover them is discarded, and released before a reclaimed message goes back on the queue. A failed release is now surfaced instead of being silently treated as success.
+
+Concurrency groups keyed on queue metadata rather than the tenant can still resolve to the wrong group when a consumer completes a message it did not enqueue, so this does not yet cover that case.

packages/redis-worker/src/fair-queue/concurrency.ts

@@ -103,7 +103,37 @@ export class ConcurrencyManager {
       pipeline.srem(key, messageId);
     }
 
-    await pipeline.exec();
+    this.#assertPipelineSucceeded(await pipeline.exec(), 1);
+  }
+
+  /**
+   * Throw if any command in a released pipeline failed. ioredis resolves `exec()` even when
+   * individual commands error, so an unchecked pipeline reports success while leaving the
+   * slot held, which strands it permanently once the caller drops the in-flight record.
+   */
+  #assertPipelineSucceeded(
+    results: Array<[Error | null, unknown]> | null,
+    messageCount: number
+  ): void {
+    if (results === null) {
+      throw new Error(
+        `Concurrency release pipeline for ${messageCount} message(s) was discarded without executing`
+      );
+    }
+
+    const errors = results
+      .map(([error]) => error)
+      .filter((error): error is Error => Boolean(error));
+
+    if (errors.length > 0) {
+      throw new Error(
+        `Failed to release ${errors.length} of ${
+          results?.length ?? 0
+        } concurrency slot commands across ${messageCount} message(s): ${errors
+          .map((error) => error.message)
+          .join("; ")}`
+      );
+    }
   }
 
   /**
@@ -127,7 +157,7 @@ export class ConcurrencyManager {
       }
     }
 
-    await pipeline.exec();
+    this.#assertPipelineSucceeded(await pipeline.exec(), messages.length);
   }
 
   /**

packages/redis-worker/src/fair-queue/index.ts

@@ -25,6 +25,7 @@ import type {
   FairScheduler,
   QueueCooloffState,
   QueueDescriptor,
+  ReclaimedMessageInfo,
   SchedulerContext,
   StoredMessage,
   TenantQueues,
@@ -1245,22 +1246,20 @@ export class FairQueue<TPayloadSchema extends z.ZodTypeAny = z.ZodUnknown> {
       }
     }
 
-    const descriptor: QueueDescriptor = storedMessage
-      ? (this.queueDescriptorCache.get(queueId) ?? {
-          id: queueId,
-          tenantId: storedMessage.tenantId,
-          metadata: storedMessage.metadata ?? {},
-        })
-      : { id: queueId, tenantId: this.keys.extractTenantId(queueId), metadata: {} };
-
-    // Complete in visibility manager
-    await this.visibilityManager.complete(messageId, queueId);
+    const descriptor: QueueDescriptor = this.queueDescriptorCache.get(queueId) ?? {
+      id: queueId,
+      tenantId: storedMessage?.tenantId ?? this.keys.extractTenantId(queueId),
+      metadata: storedMessage?.metadata ?? {},
+    };
 
     // Release concurrency
-    if (this.concurrencyManager && storedMessage) {
+    if (this.concurrencyManager) {
       await this.concurrencyManager.release(descriptor, messageId);
     }
 
+    // Complete in visibility manager
+    await this.visibilityManager.complete(messageId, queueId);
+
     // Update both old and new indexes, clean up caches if queue is empty
     const { queueEmpty } = await this.#updateAllIndexesAfterDequeue(queueId, descriptor.tenantId);
     if (queueEmpty) {
@@ -1300,13 +1299,16 @@ export class FairQueue<TPayloadSchema extends z.ZodTypeAny = z.ZodUnknown> {
       }
     }
 
-    const descriptor: QueueDescriptor = storedMessage
-      ? (this.queueDescriptorCache.get(queueId) ?? {
-          id: queueId,
-          tenantId: storedMessage.tenantId,
-          metadata: storedMessage.metadata ?? {},
-        })
-      : { id: queueId, tenantId: this.keys.extractTenantId(queueId), metadata: {} };
+    const descriptor: QueueDescriptor = this.queueDescriptorCache.get(queueId) ?? {
+      id: queueId,
+      tenantId: storedMessage?.tenantId ?? this.keys.extractTenantId(queueId),
+      metadata: storedMessage?.metadata ?? {},
+    };
+
+    // Release concurrency
+    if (this.concurrencyManager) {
+      await this.concurrencyManager.release(descriptor, messageId);
+    }
 
     // Release back to queue (visibility manager updates dispatch indexes atomically)
     // Dispatch shard is tenant-based, not queue-based
@@ -1324,17 +1326,32 @@ export class FairQueue<TPayloadSchema extends z.ZodTypeAny = z.ZodUnknown> {
       Date.now() // Put at back of queue
     );
 
-    // Release concurrency
-    if (this.concurrencyManager && storedMessage) {
-      await this.concurrencyManager.release(descriptor, messageId);
-    }
-
     this.logger.debug("Message released", {
       messageId,
       queueId,
     });
   }
 
+  /**
+   * Release a concurrency slot for a message we can no longer describe, deriving the
+   * group from the queue id alone. Used on paths that bail out before the stored
+   * message is available, where the slot would otherwise be held with nothing left
+   * to reclaim it.
+   */
+  async #releaseOrphanedConcurrency(messageId: string, queueId: string): Promise<void> {
+    if (!this.concurrencyManager) {
+      return;
+    }
+
+    const descriptor: QueueDescriptor = this.queueDescriptorCache.get(queueId) ?? {
+      id: queueId,
+      tenantId: this.keys.extractTenantId(queueId),
+      metadata: {},
+    };
+
+    await this.concurrencyManager.release(descriptor, messageId);
+  }
+
   /**
    * Mark a message as failed. This will trigger retry logic if configured,
    * or move the message to the dead letter queue.
@@ -1353,6 +1370,7 @@ export class FairQueue<TPayloadSchema extends z.ZodTypeAny = z.ZodUnknown> {
     const dataJson = await this.redis.hget(inflightDataKey, messageId);
     if (!dataJson) {
       this.logger.error("Cannot fail message: not found in in-flight data", { messageId, queueId });
+      await this.#releaseOrphanedConcurrency(messageId, queueId);
       return;
     }
 
@@ -1364,6 +1382,7 @@ export class FairQueue<TPayloadSchema extends z.ZodTypeAny = z.ZodUnknown> {
         messageId,
         queueId,
       });
+      await this.#releaseOrphanedConcurrency(messageId, queueId);
       return;
     }
 
@@ -1411,6 +1430,11 @@ export class FairQueue<TPayloadSchema extends z.ZodTypeAny = z.ZodUnknown> {
           attempt: storedMessage.attempt + 1,
         };
 
+        // Release concurrency
+        if (this.concurrencyManager) {
+          await this.concurrencyManager.release(descriptor, storedMessage.id);
+        }
+
         // Release with delay, passing the updated message data so the Lua script
         // atomically writes the incremented attempt count when re-queuing.
         const tenantQueueIndexKey = this.keys.tenantQueueIndexKey(descriptor.tenantId);
@@ -1427,11 +1451,6 @@ export class FairQueue<TPayloadSchema extends z.ZodTypeAny = z.ZodUnknown> {
           JSON.stringify(updatedMessage)
         );
 
-        // Release concurrency
-        if (this.concurrencyManager) {
-          await this.concurrencyManager.release(descriptor, storedMessage.id);
-        }
-
         this.telemetry.recordRetry();
 
         this.logger.debug("Message scheduled for retry", {
@@ -1445,13 +1464,13 @@ export class FairQueue<TPayloadSchema extends z.ZodTypeAny = z.ZodUnknown> {
       }
     }
 
-    // Move to DLQ
-    await this.#moveToDeadLetterQueue(storedMessage, error?.message);
-
     // Release concurrency
     if (this.concurrencyManager) {
       await this.concurrencyManager.release(descriptor, storedMessage.id);
     }
+
+    // Move to DLQ
+    await this.#moveToDeadLetterQueue(storedMessage, error?.message);
   }
 
   async #moveToDeadLetterQueue(
@@ -1525,51 +1544,81 @@ export class FairQueue<TPayloadSchema extends z.ZodTypeAny = z.ZodUnknown> {
     }
   }
 
-  async #reclaimTimedOutMessages(): Promise<void> {
-    let totalReclaimed = 0;
+  /**
+   * Free every timed-out message's concurrency slot in one pipeline, before any of them
+   * is put back on the queue. Throwing here aborts the requeue for the whole batch, which
+   * leaves the messages in-flight for the next reclaim tick. That is the safe direction:
+   * requeuing a message whose slot is still held is what strands the slot permanently.
+   */
+  async #releaseReclaimedConcurrency(messages: ReclaimedMessageInfo[]): Promise<string[]> {
+    if (!this.concurrencyManager || messages.length === 0) {
+      return [];
+    }
 
-    for (let shardId = 0; shardId < this.shardCount; shardId++) {
-      const reclaimedMessages = await this.visibilityManager.reclaimTimedOut(shardId, (queueId) => {
-        const tenantId = this.keys.extractTenantId(queueId);
-        const dispatchShardId = this.tenantDispatch.getShardForTenant(tenantId);
-        return {
-          queueKey: this.keys.queueKey(queueId),
-          queueItemsKey: this.keys.queueItemsKey(queueId),
-          tenantQueueIndexKey: this.keys.tenantQueueIndexKey(tenantId),
-          dispatchKey: this.keys.dispatchKey(dispatchShardId),
-          tenantId,
-        };
+    const descriptorFor = (message: ReclaimedMessageInfo) => ({
+      id: message.queueId,
+      tenantId: message.tenantId,
+      metadata: message.metadata ?? {},
+    });
+
+    try {
+      await this.concurrencyManager.releaseBatch(
+        messages.map((message) => ({ queue: descriptorFor(message), messageId: message.messageId }))
+      );
+      return [];
+    } catch (error) {
+      this.logger.error("Batch concurrency release failed, retrying message by message", {
+        count: messages.length,
+        error: error instanceof Error ? error.message : String(error),
       });
+    }
 
-      if (reclaimedMessages.length > 0) {
-        // Release concurrency for all reclaimed messages in a single batch
-        // This is critical: when a message times out, its concurrency slot must be freed
-        // so the message can be processed again when it's re-claimed from the queue
-        if (this.concurrencyManager) {
-          try {
-            await this.concurrencyManager.releaseBatch(
-              reclaimedMessages.map((msg) => ({
-                queue: {
-                  id: msg.queueId,
-                  tenantId: msg.tenantId,
-                  metadata: msg.metadata ?? {},
-                },
-                messageId: msg.messageId,
-              }))
-            );
-          } catch (error) {
-            this.logger.error("Failed to release concurrency for reclaimed messages", {
-              count: reclaimedMessages.length,
-              error: error instanceof Error ? error.message : String(error),
-            });
-          }
-        }
+    const failed: string[] = [];
 
-        // Dispatch indexes are updated atomically by the releaseMessage Lua script
-        // inside reclaimTimedOut, so no separate index update needed here.
+    for (const message of messages) {
+      try {
+        await this.concurrencyManager.release(descriptorFor(message), message.messageId);
+      } catch (error) {
+        failed.push(message.messageId);
+        this.logger.error("Failed to release concurrency for reclaimed message", {
+          messageId: message.messageId,
+          queueId: message.queueId,
+          error: error instanceof Error ? error.message : String(error),
+        });
       }
+    }
+
+    return failed;
+  }
 
-      totalReclaimed += reclaimedMessages.length;
+  async #reclaimTimedOutMessages(): Promise<void> {
+    let totalReclaimed = 0;
+
+    for (let shardId = 0; shardId < this.shardCount; shardId++) {
+      try {
+        const reclaimedMessages = await this.visibilityManager.reclaimTimedOut(
+          shardId,
+          (queueId) => {
+            const tenantId = this.keys.extractTenantId(queueId);
+            const dispatchShardId = this.tenantDispatch.getShardForTenant(tenantId);
+            return {
+              queueKey: this.keys.queueKey(queueId),
+              queueItemsKey: this.keys.queueItemsKey(queueId),
+              tenantQueueIndexKey: this.keys.tenantQueueIndexKey(tenantId),
+              dispatchKey: this.keys.dispatchKey(dispatchShardId),
+              tenantId,
+            };
+          },
+          this.#releaseReclaimedConcurrency.bind(this)
+        );
+
+        totalReclaimed += reclaimedMessages.length;
+      } catch (error) {
+        this.logger.error("Failed to reclaim shard, leaving messages in-flight for the next tick", {
+          shardId,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
     }
 
     if (totalReclaimed > 0) {