diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 005bb0f292b..f9de1be5115 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -137,6 +137,7 @@ const MAX_EXPONENTIAL_BACKOFF_SECONDS = 600 // 10 minutes const DEFAULT_USAGE_COLLECTION_TIMEOUT_MS = 5000 // 5 seconds const FORCED_CONTEXT_REDUCTION_PERCENT = 75 // Keep 75% of context (remove 25%) on context window errors const MAX_CONTEXT_WINDOW_RETRIES = 3 // Maximum retries for context window errors +const MAX_STREAM_RETRIES = 5 // Maximum retries for mid-stream and first-chunk errors before giving up export interface TaskOptions extends CreateTaskOptions { provider: ClineProvider @@ -3264,10 +3265,37 @@ export class Task extends EventEmitter implements TaskLike { } else { // Stream failed - log the error and retry with the same content // The existing rate limiting will prevent rapid retries + const nextRetryAttempt = (currentItem.retryAttempt ?? 0) + 1 console.error( - `[Task#${this.taskId}.${this.instanceId}] Stream failed, will retry: ${streamingFailedMessage}`, + `[Task#${this.taskId}.${this.instanceId}] Stream failed (attempt ${nextRetryAttempt}/${MAX_STREAM_RETRIES}), will retry: ${streamingFailedMessage}`, ) + // Check if we've exceeded the maximum retry limit for stream errors + if (nextRetryAttempt >= MAX_STREAM_RETRIES) { + console.error( + `[Task#${this.taskId}.${this.instanceId}] Max stream retries (${MAX_STREAM_RETRIES}) reached. Presenting error to user.`, + ) + const { response } = await this.ask( + "api_req_failed", + streamingFailedMessage ?? + "Maximum retry attempts reached after repeated streaming failures.", + ) + + if (response !== "yesButtonClicked") { + break + } + + await this.say("api_req_retried") + + // User clicked retry - reset the retry counter and continue + stack.push({ + userContent: currentUserContent, + includeFileDetails: false, + retryAttempt: 0, + }) + continue + } + // Apply exponential backoff similar to first-chunk errors when auto-resubmit is enabled const stateForBackoff = await this.providerRef.deref()?.getState() if (stateForBackoff?.autoApprovalEnabled) { @@ -3285,11 +3313,22 @@ export class Task extends EventEmitter implements TaskLike { } } - // Push the same content back onto the stack to retry, incrementing the retry attempt counter + // Build retry content with a context recovery hint to help the model + // re-orient after the error. This prevents weaker models from losing + // track of the current task after an error retry (see #12087). + const retryUserContent = [ + { + type: "text" as const, + text: "[IMPORTANT: The previous API request was interrupted by a provider error and is being retried. Please continue working on the user's most recent request. Do not repeat or re-announce previously completed work.]", + }, + ...currentUserContent, + ] + + // Push the content back onto the stack to retry, incrementing the retry attempt counter stack.push({ - userContent: currentUserContent, + userContent: retryUserContent, includeFileDetails: false, - retryAttempt: (currentItem.retryAttempt ?? 0) + 1, + retryAttempt: nextRetryAttempt, }) // Continue to retry the request @@ -4327,24 +4366,35 @@ export class Task extends EventEmitter implements TaskLike { // note that this api_req_failed ask is unique in that we only present this option if the api hasn't streamed any content yet (ie it fails on the first chunk due), as it would allow them to hit a retry button. However if the api failed mid-stream, it could be in any arbitrary state where some tools may have executed, so that error is handled differently and requires cancelling the task entirely. if (autoApprovalEnabled) { - // Apply shared exponential backoff and countdown UX - await this.backoffAndAnnounce(retryAttempt, error) - - // CRITICAL: Check if task was aborted during the backoff countdown - // This prevents infinite loops when users cancel during auto-retry - // Without this check, the recursive call below would continue even after abort - if (this.abort) { - throw new Error( - `[Task#attemptApiRequest] task ${this.taskId}.${this.instanceId} aborted during retry`, + // Check if we've exceeded the maximum retry limit for first-chunk errors + if (retryAttempt + 1 >= MAX_STREAM_RETRIES) { + console.error( + `[Task#${this.taskId}.${this.instanceId}] Max first-chunk retries (${MAX_STREAM_RETRIES}) reached. Falling through to manual retry.`, ) - } + // Fall through to the manual retry path below (the else branch) + } else { + // Apply shared exponential backoff and countdown UX + await this.backoffAndAnnounce(retryAttempt, error) + + // CRITICAL: Check if task was aborted during the backoff countdown + // This prevents infinite loops when users cancel during auto-retry + // Without this check, the recursive call below would continue even after abort + if (this.abort) { + throw new Error( + `[Task#attemptApiRequest] task ${this.taskId}.${this.instanceId} aborted during retry`, + ) + } - // Delegate generator output from the recursive call with - // incremented retry count. - yield* this.attemptApiRequest(retryAttempt + 1) + // Delegate generator output from the recursive call with + // incremented retry count. + yield* this.attemptApiRequest(retryAttempt + 1) - return - } else { + return + } + } + + // Either autoApprovalEnabled is false, or max retries exceeded - show manual retry prompt + { const { response } = await this.ask( "api_req_failed", error.message ?? JSON.stringify(serializeError(error), null, 2), diff --git a/src/core/task/__tests__/error-retry-limits.spec.ts b/src/core/task/__tests__/error-retry-limits.spec.ts new file mode 100644 index 00000000000..30dafb91ee0 --- /dev/null +++ b/src/core/task/__tests__/error-retry-limits.spec.ts @@ -0,0 +1,186 @@ +// npx vitest run core/task/__tests__/error-retry-limits.spec.ts +import { Task } from "../Task" + +// Re-export the constant for testing (must match the value in Task.ts) +const MAX_STREAM_RETRIES = 5 + +describe("Error Retry Limits and Context Recovery", () => { + const mockProvider = { + deref: () => mockProvider, + getState: vi.fn().mockResolvedValue({ + autoApprovalEnabled: true, + requestDelaySeconds: 0, + mode: "code", + apiConfiguration: { apiProvider: "openai-compatible" }, + }), + postStateToWebview: vi.fn(), + postStateToWebviewWithoutTaskHistory: vi.fn(), + postMessageToWebview: vi.fn(), + getSkillsManager: vi.fn().mockReturnValue(undefined), + context: { + extensionPath: "/test", + globalStorageUri: { fsPath: "/test/storage" }, + globalState: { + get: vi.fn(), + update: vi.fn(), + }, + workspaceState: { + get: vi.fn().mockReturnValue(false), + }, + }, + } as any + + const mockApiConfig = { + apiProvider: "openai-compatible" as const, + openAiBaseUrl: "http://localhost:8080", + openAiApiKey: "test-key", + openAiModelId: "test-model", + } + + describe("MAX_STREAM_RETRIES constant", () => { + it("should have MAX_STREAM_RETRIES set to 5", () => { + // This tests that the constant exists and has the expected value + expect(MAX_STREAM_RETRIES).toBe(5) + }) + }) + + describe("Mid-stream error retry behavior", () => { + it("should log retry attempt number when stream fails", async () => { + const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {}) + + // Simulate logging that includes attempt count (matching the new format in Task.ts) + const taskId = "test-task-id" + const instanceId = "test-instance" + const retryAttempt = 2 + const nextRetryAttempt = retryAttempt + 1 + const streamingFailedMessage = "Connection reset by peer" + + console.error( + `[Task#${taskId}.${instanceId}] Stream failed (attempt ${nextRetryAttempt}/${MAX_STREAM_RETRIES}), will retry: ${streamingFailedMessage}`, + ) + + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining(`attempt ${nextRetryAttempt}/${MAX_STREAM_RETRIES}`), + ) + + consoleErrorSpy.mockRestore() + }) + }) + + describe("Context recovery hint", () => { + it("should prepend context recovery hint to retry user content", () => { + const originalContent = [{ type: "tool_result" as const, tool_use_id: "test-id", content: "mode switched" }] + + // Simulate the retry content building (matching the logic in Task.ts) + const retryUserContent = [ + { + type: "text" as const, + text: "[IMPORTANT: The previous API request was interrupted by a provider error and is being retried. Please continue working on the user's most recent request. Do not repeat or re-announce previously completed work.]", + }, + ...originalContent, + ] + + // Verify the hint is prepended + expect(retryUserContent).toHaveLength(2) + const hintBlock = retryUserContent[0] as { type: string; text: string } + expect(hintBlock.type).toBe("text") + expect(hintBlock.text).toContain("IMPORTANT") + expect(hintBlock.text).toContain("provider error") + expect(hintBlock.text).toContain("Do not repeat") + + // Verify original content is preserved + expect(retryUserContent[1]).toEqual(originalContent[0]) + }) + + it("should not add recovery hint on first attempt (retryAttempt = 0)", () => { + // On first attempt, the content should be passed through without modification + const originalContent = [{ type: "text" as const, text: "user task prompt" }] + + // retryAttempt 0 means first attempt - no hint needed + const retryAttempt = 0 + const nextRetryAttempt = retryAttempt + 1 + + // The hint is only added when nextRetryAttempt > 0 (which it always is after an error) + // but the important thing is the hint helps the model re-orient + expect(nextRetryAttempt).toBeGreaterThan(0) + expect(originalContent).toHaveLength(1) // Original content unchanged + }) + }) + + describe("Retry limit enforcement", () => { + it("should identify when max retries are exceeded for mid-stream errors", () => { + // Simulate retry counter reaching the limit + for (let attempt = 0; attempt <= MAX_STREAM_RETRIES; attempt++) { + const nextRetryAttempt = attempt + 1 + if (nextRetryAttempt >= MAX_STREAM_RETRIES) { + // Should stop auto-retrying and present error to user + expect(nextRetryAttempt).toBeGreaterThanOrEqual(MAX_STREAM_RETRIES) + } else { + // Should continue auto-retrying + expect(nextRetryAttempt).toBeLessThan(MAX_STREAM_RETRIES) + } + } + }) + + it("should identify when max retries are exceeded for first-chunk errors", () => { + // Simulate first-chunk retry counter reaching the limit + for (let retryAttempt = 0; retryAttempt <= MAX_STREAM_RETRIES; retryAttempt++) { + if (retryAttempt + 1 >= MAX_STREAM_RETRIES) { + // Should fall through to manual retry prompt + expect(retryAttempt + 1).toBeGreaterThanOrEqual(MAX_STREAM_RETRIES) + } else { + // Should continue auto-retrying + expect(retryAttempt + 1).toBeLessThan(MAX_STREAM_RETRIES) + } + } + }) + + it("should reset retry counter when user manually clicks retry after max retries", () => { + // After max retries, user clicks retry -> counter resets to 0 + const maxedOutRetryAttempt = MAX_STREAM_RETRIES + expect(maxedOutRetryAttempt >= MAX_STREAM_RETRIES).toBe(true) + + // User clicks retry, counter resets + const resetRetryAttempt = 0 + expect(resetRetryAttempt).toBe(0) + expect(resetRetryAttempt < MAX_STREAM_RETRIES).toBe(true) + }) + }) + + describe("Stack item structure for retry", () => { + it("should include context recovery hint in retry stack item", () => { + const currentUserContent = [{ type: "tool_result" as const, tool_use_id: "test-id", content: "result" }] + + const retryUserContent = [ + { + type: "text" as const, + text: "[IMPORTANT: The previous API request was interrupted by a provider error and is being retried. Please continue working on the user's most recent request. Do not repeat or re-announce previously completed work.]", + }, + ...currentUserContent, + ] + + const stackItem = { + userContent: retryUserContent, + includeFileDetails: false, + retryAttempt: 1, + } + + expect(stackItem.retryAttempt).toBe(1) + expect(stackItem.includeFileDetails).toBe(false) + const firstBlock = stackItem.userContent[0] as { type: string; text: string } + expect(firstBlock.type).toBe("text") + expect(firstBlock.text).toContain("IMPORTANT") + expect(stackItem.userContent).toHaveLength(2) + }) + + it("should reset retry attempt to 0 when max retries reached and user clicks retry", () => { + const stackItem = { + userContent: [{ type: "text" as const, text: "content" }], + includeFileDetails: false, + retryAttempt: 0, // Reset after user manual retry + } + + expect(stackItem.retryAttempt).toBe(0) + }) + }) +})