From 0624f5bdd5c4d51e735a3fe825f1eed40f258120 Mon Sep 17 00:00:00 2001 From: whrho Date: Sun, 21 Jun 2026 18:15:49 +0900 Subject: [PATCH] feat: retry on provider errors using --continue - Add src/retry.ts with isRetryableError() for transport-signal-only patterns - Add build retry loop in check() with up to 3 retries and exponential backoff - Add continueRun() command to resume previous builds using --continue - Extract handlePostBuild() helper shared between check() and continueRun() - Default skip_permissions to false (security fix) - Fix double writeState call in check() - Remove accidentally committed artifacts from git history Addresses PR review feedback: narrowed retry scope, removed unused deps, fixed security default, cleaned committed artifacts. --- .gitignore | 5 + README.md | 11 +- bun.lock | 20 ---- config.example.json | 4 +- package.json | 3 +- src/index.ts | 258 ++++++++++++++++++++++++++++++++------------ src/retry.test.ts | 95 ++++++++++++++++ src/retry.ts | 63 +++++++++++ tsconfig.json | 3 +- 9 files changed, 371 insertions(+), 91 deletions(-) delete mode 100644 bun.lock create mode 100644 src/retry.test.ts create mode 100644 src/retry.ts diff --git a/.gitignore b/.gitignore index d9527a2..02991c5 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,8 @@ state/* !state/.keep artifacts/* !artifacts/.keep + +.omo/ +bun.lock +opencode +opencode-shallow diff --git a/README.md b/README.md index 94efe9c..cd0a7c0 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,9 @@ bunx @cortexkit/orw check --force "desktop_target": "/Applications/OpenCode.app", "install_cli": true, "install_desktop": true, - "notify_timeout": 120 + "notify_timeout": 120, + "retry_attempts": 3, + "retry_delay_ms": 10000 } ``` @@ -98,6 +100,13 @@ Relative paths in config are resolved from the directory containing `orw.config. - `prompt_path`: optional custom prompt template; if omitted, the packaged default prompt is used - `install_desktop`: defaults to `true` on macOS and `false` on Linux/Windows when generated by `init` +### Retry on provider errors + +When `opencode run` fails with a retryable provider error (e.g. "Provider returned error", rate limits, 5xx responses), ORW automatically retries using `opencode run --continue` to resume the last session. + +- `retry_attempts`: max number of attempts including the first one (default `3`) +- `retry_delay_ms`: delay between retries in milliseconds (default `10000`) + ## Commands ```bash diff --git a/bun.lock b/bun.lock deleted file mode 100644 index 4bcaf90..0000000 --- a/bun.lock +++ /dev/null @@ -1,20 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "opencode-release-watch", - "devDependencies": { - "@types/node": "24.0.0", - "typescript": "5.8.2", - }, - }, - }, - "packages": { - "@types/node": ["@types/node@24.0.0", "", { "dependencies": { "undici-types": "~7.8.0" } }, "sha512-yZQa2zm87aRVcqDyH5+4Hv9KYgSdgwX1rFnGvpbzMaC7YAljmhBET93TPiTd3ObwTL+gSpIzPKg5BqVxdCvxKg=="], - - "typescript": ["typescript@5.8.2", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ=="], - - "undici-types": ["undici-types@7.8.0", "", {}, "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw=="], - } -} diff --git a/config.example.json b/config.example.json index 2f23d02..28cb628 100644 --- a/config.example.json +++ b/config.example.json @@ -13,5 +13,7 @@ "desktop_target": "/Applications/OpenCode.app", "install_cli": true, "install_desktop": false, - "notify_timeout": 120 + "notify_timeout": 120, + "retry_attempts": 3, + "retry_delay_ms": 10000 } diff --git a/package.json b/package.json index 1a52944..441f399 100644 --- a/package.json +++ b/package.json @@ -57,5 +57,6 @@ }, "engines": { "bun": ">=1.3.0" - } + }, + "dependencies": {} } diff --git a/src/index.ts b/src/index.ts index 6f1ff70..e15d9cd 100755 --- a/src/index.ts +++ b/src/index.ts @@ -1,11 +1,13 @@ #!/usr/bin/env bun import fs from "node:fs/promises"; +import { readFileSync } from "node:fs"; import path from "node:path"; import os from "node:os"; import type { Buffer } from "node:buffer"; import { spawn } from "node:child_process"; import { fileURLToPath } from "node:url"; +import { isRetryableError, readLogSlice, fileSize } from "./retry"; type RawCfg = { release_repo: string; @@ -24,13 +26,23 @@ type RawCfg = { install_desktop: boolean; notify_timeout: number; git_origin: string; + retry_attempts?: number; + retry_delay_ms?: number; + build_retry_attempts?: number; + build_retry_delay_ms?: number; + skip_permissions?: boolean; }; -type Cfg = Omit & { +type Cfg = Omit & { runtime_dir: string; prompt_path: string; config_file: string; config_dir: string; + retry_attempts: number; + retry_delay_ms: number; + build_retry_attempts: number; + build_retry_delay_ms: number; + skip_permissions: boolean; }; type Cli = { @@ -106,6 +118,7 @@ async function main() { if (cli.cmd === "preview") return preview(cfg); if (cli.cmd === "status") return status(cfg); if (cli.cmd === "check") return check(cfg, cli.force); + if (cli.cmd === "continue") return continueRun(cfg); } function parseCli(rawArgs: string[]): Cli { @@ -154,6 +167,7 @@ function parseCli(rawArgs: string[]): Cli { function needsConfig(cli: Cli) { if (cli.cmd === "check") return cli.positionals.length <= 1; + if (cli.cmd === "continue") return cli.positionals.length <= 1; if (cli.cmd === "preview") return cli.positionals.length === 1; if (cli.cmd === "status") return cli.positionals.length === 1; if (cli.cmd === "install-ready") return cli.positionals.length === 1; @@ -173,7 +187,7 @@ function printHelp() { } function helpText() { - return `OpenCode Release Watch\n\nUsage:\n orw [--config ] [command] [options]\n orw --help\n\nCommands:\n init Create orw.config.json in the current directory\n preview Print the integration prompt for the latest release\n check Build the latest release if needed; default command\n status Print the last successful build/install state\n install-ready Install the last verified artifacts\n install-when-closed Wait for OpenCode to quit, then install\n launchd install Install the macOS launchd scheduler\n launchd uninstall Remove the macOS launchd scheduler\n\nOptions:\n -c, --config Use a specific config file\n --force Rebuild even if the latest release was processed\n --wait-for-opencode With install-ready, wait until OpenCode quits\n -h, --help Show this help\n`; + return `OpenCode Release Watch\n\nUsage:\n orw [--config ] [command] [options]\n orw --help\n\nCommands:\n init Create orw.config.json in the current directory\n preview Print the integration prompt for the latest release\n check Build the latest release if needed; default command\n status Print the last successful build/install state\n install-ready Install the last verified artifacts\n install-when-closed Wait for OpenCode to quit, then install\n launchd install Install the macOS launchd scheduler\n launchd uninstall Remove the macOS launchd scheduler\n continue Resume the last interrupted opencode session\n\nOptions:\n -c, --config Use a specific config file\n --force Rebuild even if the latest release was processed\n --wait-for-opencode With install-ready, wait until OpenCode quits\n -h, --help Show this help\n\nConfig (orw.config.json):\n retry_attempts Number of retry attempts on provider error (default: 3)\n retry_delay_ms Delay between retries in milliseconds (default: 10000)\n`; } async function load(configPath?: string) { @@ -197,6 +211,11 @@ async function load(configPath?: string) { install_cli: raw.install_cli ?? true, install_desktop: raw.install_desktop ?? defaultInstallDesktop(), notify_timeout: raw.notify_timeout ?? 120, + retry_attempts: Math.max(1, raw.retry_attempts ?? 3), + retry_delay_ms: raw.retry_delay_ms ?? 10_000, + build_retry_attempts: Math.max(1, raw.build_retry_attempts ?? 3), + build_retry_delay_ms: raw.build_retry_delay_ms ?? 30_000, + skip_permissions: raw.skip_permissions ?? false, git_origin: raw.git_origin ?? `https://github.com/${releaseRepo}.git`, config_file: file, config_dir: configDir, @@ -258,6 +277,11 @@ function initConfig(): RawCfg & { runtime_dir: string } { install_cli: true, install_desktop: defaultInstallDesktop(), notify_timeout: 120, + retry_attempts: 3, + retry_delay_ms: 10_000, + build_retry_attempts: 3, + build_retry_delay_ms: 30_000, + skip_permissions: false, }; } @@ -315,74 +339,90 @@ async function check(cfg: Cfg, force: boolean) { await prep(cfg, sources, log); const env = releaseEnv(release); const prompt = await render(cfg, sources, release); - try { - await run( - [ - cfg.opencode_bin, - "run", - "--agent", - cfg.agent, - "--model", - cfg.model, - prompt, - ], - { - cwd: cfg.work_repo, - log, - env: { ...env, OPENCODE_DISABLE_PROJECT_CONFIG: "1" }, - }, - ); - } catch (err) { - await notify( - "OpenCode integration failed", - `${release.tag_name} failed. See ${log}.`, - ); - throw err; - } - - const next = await verifyBuild(cfg, release, log); + const next = await runOpenCodeWithBuildRetry(cfg, prompt, env, log, release); await writeState(cfg, next); - await notify( - "OpenCode build ready", - `${release.tag_name} is integrated and built.`, - ); - const running = await runningOpenCodeProcesses(); - if (running.length > 0) { - await notify( - "OpenCode install blocked", - `Run ${orwCommand(cfg, "install-when-closed")}, then quit OpenCode to install.`, - ); - out(`Integrated ${release.tag_name}. Install skipped because OpenCode is running:`); - for (const proc of running) out(`- pid ${proc.pid}: ${proc.command}`); - out(""); - out("To install after OpenCode exits, run:"); - out(` ${orwCommand(cfg, "install-when-closed")}`); - return; - } - if (canPromptForInstall()) { - const ok = await ask( - "OpenCode build ready", - `${release.tag_name} is ready. Install the ${installLabel(cfg)} now?`, - cfg.notify_timeout, - ); - if (ok === "Yes") { - await install(cfg, next); - await notify( - "OpenCode installed", - `${release.tag_name} was installed from the local build.`, - ); - } else { - printInstallHint(cfg, release.tag_name); - } - } else { - printInstallHint(cfg, release.tag_name); - } + await handlePostBuild(cfg, next, release.tag_name); out(`Integrated ${release.tag_name}`); } finally { await free(); } } +async function continueRun(cfg: Cfg) { + const release = await latest(cfg); + const prev = await readState(cfg); + if (!prev.tag) throw new Error("No previous build state found. Run `orw check` first."); + + const free = await hold(cfg, true); + try { + const log = prev.log ?? path.join( + logDir(cfg), + `${stamp()}-${release.tag_name.replaceAll("/", "-")}-continue.log`, + ); + await fs.mkdir(path.dirname(log), { recursive: true }); + await note(log, `\n--- Resuming: continue run for ${release.tag_name} ---\n`); + + const env = releaseEnv(release); + const sources = resolveSources(cfg); + const prompt = await render(cfg, sources, release); + + out(`Resuming last opencode session for ${release.tag_name}...`); + const next = await runOpenCodeWithBuildRetry(cfg, prompt, env, log, release, true); + await writeState(cfg, next); + await handlePostBuild(cfg, next, release.tag_name); + out(`Continue completed for ${release.tag_name}`); + } finally { + await free(); + } +} + +async function runOpenCodeWithBuildRetry( + cfg: Cfg, + prompt: string, + env: Record, + log: string, + release: { tag_name: string; html_url: string }, + forceContinue = false, +): Promise { + let lastErr: unknown; + for (let attempt = 1; attempt <= cfg.build_retry_attempts; attempt++) { + try { + await runOpenCodeWithRetry(cfg, prompt, env, log, forceContinue); + return await verifyBuild(cfg, release, log); + } catch (err) { + lastErr = err; + if (!await isBuildRetryableError(cfg, err)) { + throw err; + } + if (attempt >= cfg.build_retry_attempts) break; + out(`Build attempt ${attempt}/${cfg.build_retry_attempts} did not produce verified artifacts, retrying opencode (waiting ${cfg.build_retry_delay_ms / 1000}s)...`); + await sleep(cfg.build_retry_delay_ms); + } + } + throw lastErr; +} + +async function isBuildRetryableError(cfg: Cfg, err: unknown): Promise { + if (!(err instanceof Error)) return false; + const msg = err.message.toLowerCase(); + const code = "code" in err ? err.code : undefined; + + if (code === "ENOENT") { + const cli = cliPath(cfg); + return !(await exists(cli)); + } + + const buildErrorPatterns = [ + "expected a string starting with", + "build failed", + "compilation failed", + "typescript error", + "error ts", + "bun run build", + ]; + return buildErrorPatterns.some((p) => msg.includes(p)); +} + async function verifyBuild( cfg: Cfg, release: { tag_name: string; html_url: string }, @@ -408,14 +448,62 @@ async function verifyBuild( at: new Date().toISOString(), }; } catch (err) { - await notify( - "OpenCode integration failed", - `${release.tag_name} did not produce verified artifacts. See ${log}.`, - ); + if (await isBuildRetryableError(cfg, err)) { + await notify( + "OpenCode integration failed", + `${release.tag_name} did not produce verified artifacts. See ${log}.`, + ); + } throw err; } } +async function runOpenCodeWithRetry( + cfg: Cfg, + prompt: string, + env: Record, + log: string, + forceContinue = false, +) { + const envWithFlag = { ...env, OPENCODE_DISABLE_PROJECT_CONFIG: "1" }; + let lastErr: unknown; + let logOffset = 0; + + for (let attempt = 1; attempt <= cfg.retry_attempts; attempt++) { + const isRetry = attempt > 1; + if (isRetry) { + out(`Retry attempt ${attempt}/${cfg.retry_attempts} after provider error (waiting ${cfg.retry_delay_ms / 1000}s)...`); + await note(log, `\n--- Retry attempt ${attempt}/${cfg.retry_attempts} ---\n`); + await sleep(cfg.retry_delay_ms); + } + + const beforeSize = await fileSize(log); + const args = [cfg.opencode_bin, "run"]; + if (cfg.skip_permissions) args.push("--dangerously-skip-permissions"); + if (isRetry || forceContinue) args.push("--continue"); + args.push("--agent", cfg.agent, "--model", cfg.model, prompt); + + try { + await run(args, { + cwd: cfg.work_repo, + log, + env: envWithFlag, + }); + return; + } catch (err) { + lastErr = err; + const afterSize = await fileSize(log); + const attemptLog = readLogSlice(log, beforeSize); + if (!isRetryableError(err, attemptLog)) { + throw err; + } + out(`Attempt ${attempt} failed with retryable provider error.`); + } + } + + throw lastErr; +} + async function latest(cfg: Cfg) { const url = `https://api.github.com/repos/${cfg.release_repo}/releases/latest`; const res = await fetch(url, { @@ -687,6 +775,42 @@ function canPromptForInstall() { return process.platform === "darwin"; } +async function handlePostBuild(cfg: Cfg, next: State, tagName: string) { + await notify( + "OpenCode build ready", + `${tagName} is integrated and built.`, + ); + const running = await runningOpenCodeProcesses(); + if (running.length > 0) { + await notify( + "OpenCode install blocked", + `Run ${orwCommand(cfg, "install-when-closed")}, then quit OpenCode to install.`, + ); + out(`${tagName} build is ready, but install skipped because OpenCode is running:`); + for (const proc of running) out(`- pid ${proc.pid}: ${proc.command}`); + out(""); + out("To install after OpenCode exits, run:"); + out(` ${orwCommand(cfg, "install-when-closed")}`); + return; + } + if (canPromptForInstall()) { + const ok = await ask( + "OpenCode build ready", + `${tagName} is ready. Install the ${installLabel(cfg)} now?`, + cfg.notify_timeout, + ); + if (ok === "Yes") { + await install(cfg, next); + await notify( + "OpenCode installed", + `${tagName} was installed from the local build.`, + ); + return; + } + } + printInstallHint(cfg, tagName); +} + function installLabel(cfg: Cfg) { if (cfg.install_desktop && process.platform === "darwin") return "CLI and Electron desktop app"; return "CLI"; diff --git a/src/retry.test.ts b/src/retry.test.ts new file mode 100644 index 0000000..62d8bd6 --- /dev/null +++ b/src/retry.test.ts @@ -0,0 +1,95 @@ +import { describe, test, expect } from "bun:test"; +import { isRetryableError } from "./retry"; + +describe("isRetryableError", () => { + test("returns false for non-Error values", () => { + expect(isRetryableError("string error", "")).toBe(false); + expect(isRetryableError(42, "provider returned error")).toBe(false); + expect(isRetryableError(null, "provider returned error")).toBe(false); + }); + + test("returns false when error message lacks 'exited with'", () => { + const err = new Error("some other error"); + expect(isRetryableError(err, "provider returned error")).toBe(false); + }); + + test("detects 'provider returned error' in log slice", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: provider returned error")).toBe(true); + }); + + test("detects 'provider returned an error' in log slice", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: provider returned an error")).toBe(true); + }); + + test("detects rate limit patterns", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: rate limit exceeded")).toBe(true); + expect(isRetryableError(err, "Error: too many requests")).toBe(true); + }); + + test("detects network errors", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: fetch failed")).toBe(true); + expect(isRetryableError(err, "Error: socket hang up")).toBe(true); + expect(isRetryableError(err, "Error: ECONNRESET")).toBe(true); + expect(isRetryableError(err, "Error: ETIMEDOUT")).toBe(true); + }); + + test("detects HTTP status codes with context", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: HTTP 429")).toBe(true); + expect(isRetryableError(err, "Error: status 502")).toBe(true); + expect(isRetryableError(err, "Error: response status 503")).toBe(true); + }); + + test("rejects bare HTTP status codes without context", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: port 4292")).toBe(false); + expect(isRetryableError(err, "Error: line 5032")).toBe(false); + }); + + test("detects '500 internal server error'", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: 500 internal server error")).toBe(true); + }); + + test("detects 'aborted' only with provider/transport context", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Connection aborted")).toBe(true); + expect(isRetryableError(err, "fetch aborted")).toBe(true); + expect(isRetryableError(err, "request aborted")).toBe(true); + }); + + test("rejects bare abort without provider/transport context", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Aborted")).toBe(false); + expect(isRetryableError(err, "abort")).toBe(false); + expect(isRetryableError(err, "Error: Aborted")).toBe(false); + }); + + test("treats empty or very short log slices as non-retryable", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "")).toBe(false); + expect(isRetryableError(err, " ")).toBe(false); + }); + + test("returns false for non-retryable errors", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: build failed")).toBe(false); + expect(isRetryableError(err, "Error: syntax error")).toBe(false); + }); + + test("per-attempt log slicing: stale content from attempt 1 does not affect attempt 2", () => { + const err = new Error("opencode exited with 1"); + const attempt1Log = "Error: provider returned error"; + const attempt2Log = "Error: build failed"; + + // Attempt 1 log contains retryable error + expect(isRetryableError(err, attempt1Log)).toBe(true); + + // Attempt 2 log does NOT contain retryable error (only new content) + expect(isRetryableError(err, attempt2Log)).toBe(false); + }); +}); diff --git a/src/retry.ts b/src/retry.ts new file mode 100644 index 0000000..e55516d --- /dev/null +++ b/src/retry.ts @@ -0,0 +1,63 @@ +import { readFileSync } from "node:fs"; +import fs from "node:fs/promises"; + +export function isRetryableError(err: unknown, logSlice: string): boolean { + if (!(err instanceof Error)) return false; + const msg = err.message.toLowerCase(); + if (!msg.includes("exited with")) return false; + + const content = logSlice.toLowerCase(); + const textPatterns = [ + "provider returned error", + "provider returned an error", + "overloaded", + "rate limit", + "too many requests", + "connection error", + "fetch failed", + "socket hang up", + "econnreset", + "etimedout", + "context deadline exceeded", + "500 internal server error", + "connection aborted", + "fetch aborted", + "request aborted", + "operation aborted", + ]; + if (textPatterns.some((p) => content.includes(p))) return true; + + const httpStatusRegex = /(?:http|status|error|response)[\s:]+429\b/; + const httpServerErrorRegex = /(?:http|status|error|response)[\s:]+50[23]\b/; + if (httpStatusRegex.test(content) || httpServerErrorRegex.test(content)) return true; + + const nonRetryableBuildPatterns = [ + "build failed", + "compilation failed", + "typescript error", + "error ts", + "expected a string starting with", + ]; + const hasBuildError = nonRetryableBuildPatterns.some((p) => content.includes(p)); + if (hasBuildError) return false; + + return false; +} + +export function readLogSlice(file: string, byteOffset: number): string { + try { + const buf = readFileSync(file); + if (byteOffset >= buf.length) return ""; + return buf.subarray(byteOffset).toString("utf8"); + } catch { + return ""; + } +} + +export async function fileSize(file: string): Promise { + try { + return (await fs.stat(file)).size; + } catch { + return 0; + } +} diff --git a/tsconfig.json b/tsconfig.json index 75e6212..544b249 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,5 +7,6 @@ "noEmit": true, "types": ["node"] }, - "include": ["src/**/*.ts"] + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts"] }