Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 69 additions & 19 deletions src/core/task/Task.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ const MAX_EXPONENTIAL_BACKOFF_SECONDS = 600 // 10 minutes
const DEFAULT_USAGE_COLLECTION_TIMEOUT_MS = 5000 // 5 seconds
const FORCED_CONTEXT_REDUCTION_PERCENT = 75 // Keep 75% of context (remove 25%) on context window errors
const MAX_CONTEXT_WINDOW_RETRIES = 3 // Maximum retries for context window errors
const MAX_STREAM_RETRIES = 5 // Maximum retries for mid-stream and first-chunk errors before giving up

export interface TaskOptions extends CreateTaskOptions {
provider: ClineProvider
Expand Down Expand Up @@ -3264,10 +3265,37 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
} else {
// Stream failed - log the error and retry with the same content
// The existing rate limiting will prevent rapid retries
const nextRetryAttempt = (currentItem.retryAttempt ?? 0) + 1
console.error(
`[Task#${this.taskId}.${this.instanceId}] Stream failed, will retry: ${streamingFailedMessage}`,
`[Task#${this.taskId}.${this.instanceId}] Stream failed (attempt ${nextRetryAttempt}/${MAX_STREAM_RETRIES}), will retry: ${streamingFailedMessage}`,
)

// Check if we've exceeded the maximum retry limit for stream errors
if (nextRetryAttempt >= MAX_STREAM_RETRIES) {
console.error(
`[Task#${this.taskId}.${this.instanceId}] Max stream retries (${MAX_STREAM_RETRIES}) reached. Presenting error to user.`,
)
const { response } = await this.ask(
"api_req_failed",
streamingFailedMessage ??
"Maximum retry attempts reached after repeated streaming failures.",
)

if (response !== "yesButtonClicked") {
break
}

await this.say("api_req_retried")

// User clicked retry - reset the retry counter and continue
stack.push({
userContent: currentUserContent,
includeFileDetails: false,
retryAttempt: 0,
})
continue
}

// Apply exponential backoff similar to first-chunk errors when auto-resubmit is enabled
const stateForBackoff = await this.providerRef.deref()?.getState()
if (stateForBackoff?.autoApprovalEnabled) {
Expand All @@ -3285,11 +3313,22 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
}
}

// Push the same content back onto the stack to retry, incrementing the retry attempt counter
// Build retry content with a context recovery hint to help the model
// re-orient after the error. This prevents weaker models from losing
// track of the current task after an error retry (see #12087).
const retryUserContent = [
{
type: "text" as const,
text: "[IMPORTANT: The previous API request was interrupted by a provider error and is being retried. Please continue working on the user's most recent request. Do not repeat or re-announce previously completed work.]",
},
...currentUserContent,
]

// Push the content back onto the stack to retry, incrementing the retry attempt counter
stack.push({
userContent: currentUserContent,
userContent: retryUserContent,
includeFileDetails: false,
retryAttempt: (currentItem.retryAttempt ?? 0) + 1,
retryAttempt: nextRetryAttempt,
})

// Continue to retry the request
Expand Down Expand Up @@ -4327,24 +4366,35 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {

// note that this api_req_failed ask is unique in that we only present this option if the api hasn't streamed any content yet (ie it fails on the first chunk due), as it would allow them to hit a retry button. However if the api failed mid-stream, it could be in any arbitrary state where some tools may have executed, so that error is handled differently and requires cancelling the task entirely.
if (autoApprovalEnabled) {
// Apply shared exponential backoff and countdown UX
await this.backoffAndAnnounce(retryAttempt, error)

// CRITICAL: Check if task was aborted during the backoff countdown
// This prevents infinite loops when users cancel during auto-retry
// Without this check, the recursive call below would continue even after abort
if (this.abort) {
throw new Error(
`[Task#attemptApiRequest] task ${this.taskId}.${this.instanceId} aborted during retry`,
// Check if we've exceeded the maximum retry limit for first-chunk errors
if (retryAttempt + 1 >= MAX_STREAM_RETRIES) {
console.error(
`[Task#${this.taskId}.${this.instanceId}] Max first-chunk retries (${MAX_STREAM_RETRIES}) reached. Falling through to manual retry.`,
)
}
// Fall through to the manual retry path below (the else branch)
} else {
// Apply shared exponential backoff and countdown UX
await this.backoffAndAnnounce(retryAttempt, error)

// CRITICAL: Check if task was aborted during the backoff countdown
// This prevents infinite loops when users cancel during auto-retry
// Without this check, the recursive call below would continue even after abort
if (this.abort) {
throw new Error(
`[Task#attemptApiRequest] task ${this.taskId}.${this.instanceId} aborted during retry`,
)
}

// Delegate generator output from the recursive call with
// incremented retry count.
yield* this.attemptApiRequest(retryAttempt + 1)
// Delegate generator output from the recursive call with
// incremented retry count.
yield* this.attemptApiRequest(retryAttempt + 1)

return
} else {
return
}
}

// Either autoApprovalEnabled is false, or max retries exceeded - show manual retry prompt
{
const { response } = await this.ask(
"api_req_failed",
error.message ?? JSON.stringify(serializeError(error), null, 2),
Expand Down
186 changes: 186 additions & 0 deletions src/core/task/__tests__/error-retry-limits.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
// npx vitest run core/task/__tests__/error-retry-limits.spec.ts
import { Task } from "../Task"

// Re-export the constant for testing (must match the value in Task.ts)
const MAX_STREAM_RETRIES = 5

describe("Error Retry Limits and Context Recovery", () => {
const mockProvider = {
deref: () => mockProvider,
getState: vi.fn().mockResolvedValue({
autoApprovalEnabled: true,
requestDelaySeconds: 0,
mode: "code",
apiConfiguration: { apiProvider: "openai-compatible" },
}),
postStateToWebview: vi.fn(),
postStateToWebviewWithoutTaskHistory: vi.fn(),
postMessageToWebview: vi.fn(),
getSkillsManager: vi.fn().mockReturnValue(undefined),
context: {
extensionPath: "/test",
globalStorageUri: { fsPath: "/test/storage" },
globalState: {
get: vi.fn(),
update: vi.fn(),
},
workspaceState: {
get: vi.fn().mockReturnValue(false),
},
},
} as any

const mockApiConfig = {
apiProvider: "openai-compatible" as const,
openAiBaseUrl: "http://localhost:8080",
openAiApiKey: "test-key",
openAiModelId: "test-model",
}

describe("MAX_STREAM_RETRIES constant", () => {
it("should have MAX_STREAM_RETRIES set to 5", () => {
// This tests that the constant exists and has the expected value
expect(MAX_STREAM_RETRIES).toBe(5)
})
})

describe("Mid-stream error retry behavior", () => {
it("should log retry attempt number when stream fails", async () => {
const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {})

// Simulate logging that includes attempt count (matching the new format in Task.ts)
const taskId = "test-task-id"
const instanceId = "test-instance"
const retryAttempt = 2
const nextRetryAttempt = retryAttempt + 1
const streamingFailedMessage = "Connection reset by peer"

console.error(
`[Task#${taskId}.${instanceId}] Stream failed (attempt ${nextRetryAttempt}/${MAX_STREAM_RETRIES}), will retry: ${streamingFailedMessage}`,
)

expect(consoleErrorSpy).toHaveBeenCalledWith(
expect.stringContaining(`attempt ${nextRetryAttempt}/${MAX_STREAM_RETRIES}`),
)

consoleErrorSpy.mockRestore()
})
})

describe("Context recovery hint", () => {
it("should prepend context recovery hint to retry user content", () => {
const originalContent = [{ type: "tool_result" as const, tool_use_id: "test-id", content: "mode switched" }]

// Simulate the retry content building (matching the logic in Task.ts)
const retryUserContent = [
{
type: "text" as const,
text: "[IMPORTANT: The previous API request was interrupted by a provider error and is being retried. Please continue working on the user's most recent request. Do not repeat or re-announce previously completed work.]",
},
...originalContent,
]

// Verify the hint is prepended
expect(retryUserContent).toHaveLength(2)
const hintBlock = retryUserContent[0] as { type: string; text: string }
expect(hintBlock.type).toBe("text")
expect(hintBlock.text).toContain("IMPORTANT")
expect(hintBlock.text).toContain("provider error")
expect(hintBlock.text).toContain("Do not repeat")

// Verify original content is preserved
expect(retryUserContent[1]).toEqual(originalContent[0])
})

it("should not add recovery hint on first attempt (retryAttempt = 0)", () => {
// On first attempt, the content should be passed through without modification
const originalContent = [{ type: "text" as const, text: "user task prompt" }]

// retryAttempt 0 means first attempt - no hint needed
const retryAttempt = 0
const nextRetryAttempt = retryAttempt + 1

// The hint is only added when nextRetryAttempt > 0 (which it always is after an error)
// but the important thing is the hint helps the model re-orient
expect(nextRetryAttempt).toBeGreaterThan(0)
expect(originalContent).toHaveLength(1) // Original content unchanged
})
})

describe("Retry limit enforcement", () => {
it("should identify when max retries are exceeded for mid-stream errors", () => {
// Simulate retry counter reaching the limit
for (let attempt = 0; attempt <= MAX_STREAM_RETRIES; attempt++) {
const nextRetryAttempt = attempt + 1
if (nextRetryAttempt >= MAX_STREAM_RETRIES) {
// Should stop auto-retrying and present error to user
expect(nextRetryAttempt).toBeGreaterThanOrEqual(MAX_STREAM_RETRIES)
} else {
// Should continue auto-retrying
expect(nextRetryAttempt).toBeLessThan(MAX_STREAM_RETRIES)
}
}
})

it("should identify when max retries are exceeded for first-chunk errors", () => {
// Simulate first-chunk retry counter reaching the limit
for (let retryAttempt = 0; retryAttempt <= MAX_STREAM_RETRIES; retryAttempt++) {
if (retryAttempt + 1 >= MAX_STREAM_RETRIES) {
// Should fall through to manual retry prompt
expect(retryAttempt + 1).toBeGreaterThanOrEqual(MAX_STREAM_RETRIES)
} else {
// Should continue auto-retrying
expect(retryAttempt + 1).toBeLessThan(MAX_STREAM_RETRIES)
}
}
})

it("should reset retry counter when user manually clicks retry after max retries", () => {
// After max retries, user clicks retry -> counter resets to 0
const maxedOutRetryAttempt = MAX_STREAM_RETRIES
expect(maxedOutRetryAttempt >= MAX_STREAM_RETRIES).toBe(true)

// User clicks retry, counter resets
const resetRetryAttempt = 0
expect(resetRetryAttempt).toBe(0)
expect(resetRetryAttempt < MAX_STREAM_RETRIES).toBe(true)
})
})

describe("Stack item structure for retry", () => {
it("should include context recovery hint in retry stack item", () => {
const currentUserContent = [{ type: "tool_result" as const, tool_use_id: "test-id", content: "result" }]

const retryUserContent = [
{
type: "text" as const,
text: "[IMPORTANT: The previous API request was interrupted by a provider error and is being retried. Please continue working on the user's most recent request. Do not repeat or re-announce previously completed work.]",
},
...currentUserContent,
]

const stackItem = {
userContent: retryUserContent,
includeFileDetails: false,
retryAttempt: 1,
}

expect(stackItem.retryAttempt).toBe(1)
expect(stackItem.includeFileDetails).toBe(false)
const firstBlock = stackItem.userContent[0] as { type: string; text: string }
expect(firstBlock.type).toBe("text")
expect(firstBlock.text).toContain("IMPORTANT")
expect(stackItem.userContent).toHaveLength(2)
})

it("should reset retry attempt to 0 when max retries reached and user clicks retry", () => {
const stackItem = {
userContent: [{ type: "text" as const, text: "content" }],
includeFileDetails: false,
retryAttempt: 0, // Reset after user manual retry
}

expect(stackItem.retryAttempt).toBe(0)
})
})
})
Loading