diff --git a/crates/agent-gateway/test/webui/web-settings.test.mjs b/crates/agent-gateway/test/webui/web-settings.test.mjs index a6c19ec4d..970182705 100644 --- a/crates/agent-gateway/test/webui/web-settings.test.mjs +++ b/crates/agent-gateway/test/webui/web-settings.test.mjs @@ -1663,3 +1663,45 @@ test("gateway sync keeps all web font families local", () => { current.customSettings, ); }); + +test("webui model failover round-trips through gateway settings sync", () => { + const providers = [ + { + id: "provider-primary", + name: "Primary", + type: "claude_code", + baseUrl: "https://primary.example.com", + apiKey: "key-primary", + models: [{ id: "claude-fable-5", contextWindow: 200000, maxOutputToken: 8192 }], + activeModels: ["claude-fable-5"], + }, + { + id: "provider-backup", + name: "Backup", + type: "claude_code", + baseUrl: "https://backup.example.com", + apiKey: "key-backup", + models: [{ id: "claude-fable-5", contextWindow: 200000, maxOutputToken: 8192 }], + activeModels: ["claude-fable-5"], + }, + ]; + const edited = settings.updateModelFailover( + settings.normalizeSettings({ customProviders: providers }), + "claude_code", + { enabled: true, queue: ["provider-backup"], cooldownSeconds: 120 }, + ); + + // The changed field must be part of the outgoing update... + const update = settingsSync.buildGatewaySettingsSyncUpdatePayload( + settings.normalizeSettings({ customProviders: providers }), + edited, + ); + assert.equal(update.modelFailover?.claude_code.enabled, true); + + // ...and a receiver applying the full payload converges on the same config. + const received = settingsSync.applyGatewaySettingsSyncPayload( + settings.normalizeSettings({ customProviders: providers }), + settingsSync.buildGatewaySettingsSyncPayload(edited), + ); + assert.deepEqual(received.modelFailover, edited.modelFailover); +}); diff --git a/crates/agent-gateway/web/src/i18n/config.ts b/crates/agent-gateway/web/src/i18n/config.ts index 128865f22..c558a183a 100644 --- a/crates/agent-gateway/web/src/i18n/config.ts +++ b/crates/agent-gateway/web/src/i18n/config.ts @@ -1544,15 +1544,36 @@ export const translations: Record> = { "settings.deleteConfirm": "确认删除", "settings.deleteConfirmDesc": "此操作不可撤销。", "settings.deleteConfirmYes": "确定删除", - "settings.customSettings": "自定义设置", - "settings.openCustomSettings": "打开自定义设置", - "settings.closeCustomSettings": "关闭自定义设置", + "settings.customSettings": "高级设置", + "settings.openCustomSettings": "打开高级设置", + "settings.closeCustomSettings": "关闭高级设置", "settings.conversationTitleGeneration": "对话标题生成", "settings.conversationTitleModel": "标题生成模型", "settings.conversationTitleModelFollowCurrent": "使用当前对话模型", "settings.conversationTitleModelHint": "未选择时，标题生成会使用当前对话使用的模型。", "settings.customSettingsModelEmpty": "当前 Provider 未配置模型。", + /* ── Settings Model Failover ── */ + "settings.failoverTitle": "自动故障转移", + "settings.failoverEnabled": "已开启", + "settings.failoverToggleHint": + "开启后，{vendor} 请求失败时会按队列顺序自动把当前对话的模型切换到下一个 {vendor} 供应商继续请求（模型不变，只换供应商），并在成功后停留在该供应商。仅在同厂商供应商之间转移，不会跨厂商。", + "settings.failoverQueueTitle": "故障转移队列", + "settings.failoverQueueHint": + "按优先级排列的 {vendor} 备用供应商（P1 优先）。当前供应商请求失败时，用同一个模型依次尝试；未激活该模型的供应商会被跳过。", + "settings.failoverQueueAdd": "选择供应商加入队列", + "settings.failoverQueueEmpty": "队列为空。添加至少一个备用供应商后，自动故障转移才会生效。", + "settings.failoverQueueMoveUp": "上移", + "settings.failoverQueueMoveDown": "下移", + "settings.failoverQueueRemove": "移除", + "settings.failoverMaxSwitches": "单次请求最大切换次数", + "settings.failoverMaxSwitchesHint": "一次请求失败后最多切换的备用供应商个数（1-10）。", + "settings.failoverFailureThreshold": "失败阈值", + "settings.failoverFailureThresholdHint": "连续失败多少次后熔断该供应商（建议 3-10）。", + "settings.failoverCooldownSeconds": "熔断冷却时间（秒）", + "settings.failoverCooldownSecondsHint": + "熔断后跳过该供应商的时长，到期后放行探测请求（5-3600 秒）。", + /* ── Settings Prompt ── */ "settings.agentsTitle": "全局提示词", "settings.agentsDesc": "管理可复用的全局提示词模板", @@ -3754,9 +3775,9 @@ export const translations: Record> = { "settings.deleteConfirm": "Confirm Delete", "settings.deleteConfirmDesc": "This action cannot be undone.", "settings.deleteConfirmYes": "Delete", - "settings.customSettings": "Custom Settings", - "settings.openCustomSettings": "Open custom settings", - "settings.closeCustomSettings": "Close custom settings", + "settings.customSettings": "Advanced Settings", + "settings.openCustomSettings": "Open advanced settings", + "settings.closeCustomSettings": "Close advanced settings", "settings.conversationTitleGeneration": "Conversation title generation", "settings.conversationTitleModel": "Title generation model", "settings.conversationTitleModelFollowCurrent": "Use current chat model", @@ -3765,6 +3786,30 @@ export const translations: Record> = { "settings.customSettingsModelEmpty": "No active models are configured for the current providers.", + /* ── Settings Model Failover ── */ + "settings.failoverTitle": "Auto Failover", + "settings.failoverEnabled": "Enabled", + "settings.failoverToggleHint": + "When a {vendor} request fails, automatically retry it on the next {vendor} provider in the queue with the same model (provider changes, model does not) and stay on the provider that answered. Failover never crosses vendors.", + "settings.failoverQueueTitle": "Failover queue", + "settings.failoverQueueHint": + "Fallback {vendor} providers in priority order (P1 first), tried with the conversation's current model when the active provider fails. Providers without that model active are skipped.", + "settings.failoverQueueAdd": "Add a provider to the queue", + "settings.failoverQueueEmpty": + "The queue is empty. Auto failover only takes effect after at least one fallback provider is added.", + "settings.failoverQueueMoveUp": "Move up", + "settings.failoverQueueMoveDown": "Move down", + "settings.failoverQueueRemove": "Remove", + "settings.failoverMaxSwitches": "Max switches per request", + "settings.failoverMaxSwitchesHint": + "Maximum number of fallback providers tried after a failed request (1-10).", + "settings.failoverFailureThreshold": "Failure threshold", + "settings.failoverFailureThresholdHint": + "Consecutive failures before a provider's circuit breaker opens (suggested 3-10).", + "settings.failoverCooldownSeconds": "Cooldown (seconds)", + "settings.failoverCooldownSecondsHint": + "How long an open circuit skips the provider before allowing a probe request (5-3600 seconds).", + /* ── Settings Prompt ── */ "settings.agentsTitle": "Prompt", "settings.agentsDesc": "Manage reusable global prompt templates", diff --git a/crates/agent-gateway/web/src/lib/settings/index.ts b/crates/agent-gateway/web/src/lib/settings/index.ts index eeba39875..7fb776b5d 100644 --- a/crates/agent-gateway/web/src/lib/settings/index.ts +++ b/crates/agent-gateway/web/src/lib/settings/index.ts @@ -158,6 +158,62 @@ export type CustomSettings = { fontScale: FontScaleSettings; }; +/** + * cc-switch style automatic provider failover: an ordered fallback queue of + * same-vendor *providers* tried when the active model's request fails with a + * provider-fault-class error, plus circuit breaker knobs mirroring cc-switch's + * 失败阈值/冷却时间 settings. + * + * Failover switches providers, never models (matching cc-switch): the failed + * request is re-sent to the next provider in the queue with the *same model + * id* the conversation was using. Providers that don't have that model active + * are skipped at plan time. + * + * Failover is scoped per vendor type (mirroring cc-switch's Claude/Codex/ + * Gemini app tabs): a Claude request only fails over to Claude providers, a + * Codex request only to Codex providers, never across vendors. + */ +export type ProviderFailoverSettings = { + enabled: boolean; + /** Ordered fallback provider ids (P1 → P2 → …), same vendor type only. */ + queue: string[]; + /** Max provider switches per request (attempts = switches + 1). */ + maxSwitches: number; + /** Consecutive failures before a target's circuit breaker opens. */ + failureThreshold: number; + /** Seconds an open breaker skips its target before a half-open probe. */ + cooldownSeconds: number; +}; + +/** Per-vendor failover settings, keyed by the provider tab type. */ +export type ModelFailoverSettings = Record; + +export const MODEL_FAILOVER_QUEUE_LIMIT = 8; + +export const PROVIDER_FAILOVER_TYPES: readonly ProviderId[] = [ + "claude_code", + "codex", + "gemini", + "xai", +]; + +export const DEFAULT_PROVIDER_FAILOVER_SETTINGS: ProviderFailoverSettings = { + enabled: false, + queue: [], + maxSwitches: 3, + failureThreshold: 4, + cooldownSeconds: 60, +}; + +export function getDefaultModelFailoverSettings(): ModelFailoverSettings { + return { + claude_code: { ...DEFAULT_PROVIDER_FAILOVER_SETTINGS }, + codex: { ...DEFAULT_PROVIDER_FAILOVER_SETTINGS }, + gemini: { ...DEFAULT_PROVIDER_FAILOVER_SETTINGS }, + xai: { ...DEFAULT_PROVIDER_FAILOVER_SETTINGS }, + }; +} + export type SystemProxyType = "socks5" | "http"; // 系统级出站代理：注入本地 shell 命令 env，并供勾选了 useSystemProxy 的 @@ -393,6 +449,7 @@ export type AppSettings = { remote: RemoteSettings; memory: MemorySettings; customSettings: CustomSettings; + modelFailover: ModelFailoverSettings; skills: SkillsSettings; chatRuntimeControls: ChatRuntimeControls; selectedModel?: SelectedModel; @@ -2217,6 +2274,107 @@ export function normalizeCustomSettings( }; } +function clampFailoverInteger(input: unknown, min: number, max: number, fallback: number): number { + const value = + typeof input === "number" && Number.isFinite(input) + ? Math.round(input) + : typeof input === "string" && input.trim() !== "" + ? Math.round(Number(input)) + : Number.NaN; + if (!Number.isFinite(value)) return fallback; + return Math.min(max, Math.max(min, value)); +} + +/** + * Normalizes one vendor's failover config. Queue entries must reference an + * existing provider of `providerType` — cross-vendor entries (e.g. a Codex + * provider inside the Claude queue) are dropped so failover can never mix + * vendors. + * + * Legacy entry migration: the queue used to hold {customProviderId, model} + * objects. Those collapse to their provider id (deduped), because failover now + * always re-sends the conversation's own model to the fallback provider. + */ +export function normalizeProviderFailoverSettings( + input: unknown, + customProviders: CustomProvider[], + providerType: ProviderId, +): ProviderFailoverSettings { + const obj = (input && typeof input === "object" ? input : {}) as Record; + const defaults = DEFAULT_PROVIDER_FAILOVER_SETTINGS; + + const queue: string[] = []; + const seen = new Set(); + if (Array.isArray(obj.queue)) { + for (const raw of obj.queue) { + const providerId = + typeof raw === "string" + ? raw + : raw && + typeof raw === "object" && + typeof (raw as SelectedModel).customProviderId === "string" + ? (raw as SelectedModel).customProviderId + : ""; + if (!providerId) continue; + const provider = customProviders.find((item) => item.id === providerId); + if (!provider || provider.type !== providerType) continue; + if (seen.has(providerId)) continue; + seen.add(providerId); + queue.push(providerId); + if (queue.length >= MODEL_FAILOVER_QUEUE_LIMIT) break; + } + } + + return { + // An enabled toggle with an empty queue is a harmless no-op at runtime; + // keep the user's toggle state instead of silently flipping it off. + enabled: obj.enabled === true, + queue, + maxSwitches: clampFailoverInteger(obj.maxSwitches, 1, 10, defaults.maxSwitches), + failureThreshold: clampFailoverInteger(obj.failureThreshold, 1, 10, defaults.failureThreshold), + cooldownSeconds: clampFailoverInteger(obj.cooldownSeconds, 5, 3600, defaults.cooldownSeconds), + }; +} + +/** True for the pre-per-vendor persisted shape ({enabled, queue, ...}). */ +function isLegacyFlatModelFailoverShape(obj: Record): boolean { + return ( + !PROVIDER_FAILOVER_TYPES.some((type) => type in obj) && + ("enabled" in obj || "queue" in obj || "maxSwitches" in obj) + ); +} + +export function normalizeModelFailoverSettings( + input: unknown, + customProviders: CustomProvider[], +): ModelFailoverSettings { + const obj = (input && typeof input === "object" ? input : {}) as Record; + + // Legacy migration: the old single global config becomes each vendor's + // config. Cross-vendor queue entries are filtered per tab by the per-vendor + // normalizer, so a mixed legacy queue splits cleanly into its vendors. + if (isLegacyFlatModelFailoverShape(obj)) { + const result = getDefaultModelFailoverSettings(); + for (const type of PROVIDER_FAILOVER_TYPES) { + const migrated = normalizeProviderFailoverSettings(obj, customProviders, type); + // Only vendors that actually kept queue entries stay enabled; an empty + // migrated queue with enabled=true would surface confusing "on but + // empty" warnings on tabs the user never configured. + result[type] = { + ...migrated, + enabled: migrated.enabled && migrated.queue.length > 0, + }; + } + return result; + } + + const result = getDefaultModelFailoverSettings(); + for (const type of PROVIDER_FAILOVER_TYPES) { + result[type] = normalizeProviderFailoverSettings(obj[type], customProviders, type); + } + return result; +} + export function getDefaultSettings(): AppSettings { const customProviders = getBuiltinCustomProviders(); return { @@ -2255,6 +2413,7 @@ export function getDefaultSettings(): AppSettings { }, memory: normalizeMemorySettings({}, customProviders), customSettings: normalizeCustomSettings({}, customProviders), + modelFailover: normalizeModelFailoverSettings({}, customProviders), skills: { enabled: true, selected: mergeAlwaysEnabledSkillNames([]), @@ -2289,6 +2448,10 @@ export function normalizeSettings(input?: Partial | null): AppSetti obj.customSettings ?? defaults.customSettings, customProviders, ), + modelFailover: normalizeModelFailoverSettings( + obj.modelFailover ?? defaults.modelFailover, + customProviders, + ), skills: normalizeSkillsSettings(obj.skills ?? defaults.skills), chatRuntimeControls: normalizeChatRuntimeControls( obj.chatRuntimeControls ?? defaults.chatRuntimeControls, @@ -2436,6 +2599,23 @@ export function updateCustomSettings( }); } +export function updateModelFailover( + prev: AppSettings, + providerType: ProviderId, + patch: Partial, +): AppSettings { + return normalizeSettings({ + ...prev, + modelFailover: { + ...prev.modelFailover, + [providerType]: { + ...prev.modelFailover[providerType], + ...patch, + }, + }, + }); +} + const RIGHT_DOCK_WRITER_ID_STORAGE_KEY = "liveagent.client-id"; let cachedRightDockWriterId = ""; diff --git a/crates/agent-gateway/web/src/lib/settings/sync.ts b/crates/agent-gateway/web/src/lib/settings/sync.ts index 23bb8090f..01120cee9 100644 --- a/crates/agent-gateway/web/src/lib/settings/sync.ts +++ b/crates/agent-gateway/web/src/lib/settings/sync.ts @@ -58,6 +58,7 @@ export type GatewaySettingsSyncPayload = { "enableWebTerminal" | "enableWebSshTerminal" | "enableWebGit" | "enableWebTunnels" >; memory: AppSettings["memory"]; + modelFailover: AppSettings["modelFailover"]; customSettings: GatewaySettingsSyncCustomSettings; skills: AppSettings["skills"]; chatRuntimeControls: AppSettings["chatRuntimeControls"]; @@ -82,6 +83,7 @@ const GATEWAY_SETTINGS_SYNC_FIELDS = [ "ssh", "remote", "memory", + "modelFailover", "customSettings", "skills", "chatRuntimeControls", @@ -1071,6 +1073,7 @@ export function buildGatewaySettingsSyncPayload( enableWebTunnels: settings.remote.enableWebTunnels, }, memory: settings.memory, + modelFailover: settings.modelFailover, customSettings: syncableCustomSettings(settings.customSettings), skills: settings.skills, chatRuntimeControls: settings.chatRuntimeControls, @@ -1206,6 +1209,9 @@ export function applyGatewaySettingsSyncPayload( ? applySyncedSshPatch(current.ssh, source.sshPatch, sshSecretUpdates) : current.ssh, memory: memory as AppSettings["memory"], + modelFailover: Object.hasOwn(source, "modelFailover") + ? (source.modelFailover as AppSettings["modelFailover"]) + : current.modelFailover, customSettings: { ...incomingCustomSettings, rightDock: Object.hasOwn(incomingCustomSettings, "rightDock") diff --git a/crates/agent-gateway/web/src/pages/settings/ProvidersSection.tsx b/crates/agent-gateway/web/src/pages/settings/ProvidersSection.tsx index 868f93f08..b11d82724 100644 --- a/crates/agent-gateway/web/src/pages/settings/ProvidersSection.tsx +++ b/crates/agent-gateway/web/src/pages/settings/ProvidersSection.tsx @@ -2,6 +2,8 @@ import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } fr import { createPortal } from "react-dom"; import { Check, + ChevronDown, + ChevronUp, ClaudeIcon, ClipboardPaste, ExternalLink, @@ -72,11 +74,14 @@ import { type CodexRequestFormat, type CustomProvider, getDefaultUsageQueryConfig, + MODEL_FAILOVER_QUEUE_LIMIT, + type ProviderFailoverSettings, type ProviderId, type ProviderModelConfig, type UsageQueryMode, updateCustomProviders, updateCustomSettings, + updateModelFailover, } from "../../lib/settings"; import { createUuid } from "../../lib/shared/id"; import { useModalMotion } from "../../lib/shared/modalMotion"; @@ -2589,8 +2594,257 @@ function ProviderModal({ providerType, initialData, onSave, onClose }: ModalProp ); } -function CustomSettingsDrawer(props: SettingsSectionProps & { onClose: () => void }) { - const { settings, setSettings, onClose } = props; +function FailoverNumberField(props: { + label: string; + hint: string; + value: number; + min: number; + max: number; + onCommit: (value: number) => void; +}) { + const { label, hint, value, min, max, onCommit } = props; + const [draft, setDraft] = useState(String(value)); + + useEffect(() => { + setDraft(String(value)); + }, [value]); + + function commitDraft() { + const parsed = Number(draft); + if (!Number.isFinite(parsed)) { + setDraft(String(value)); + return; + } + const next = Math.min(max, Math.max(min, Math.round(parsed))); + setDraft(String(next)); + if (next !== value) onCommit(next); + } + + return ( +
+ + setDraft(event.target.value)} + onBlur={commitDraft} + onKeyDown={(event) => { + if (event.key === "Enter") commitDraft(); + }} + className="h-9 rounded-lg border-foreground/10 bg-white/70 text-[13px] shadow-sm dark:bg-background/40" + /> +

{hint}

+
+ ); +} + +function FailoverSettingsCard(props: SettingsSectionProps & { providerType: ProviderId }) { + const { settings, setSettings, providerType } = props; + const { t } = useLocale(); + const failover = settings.modelFailover[providerType]; + // Same-vendor guard: only providers of this tab's vendor type are offered, + // so a Claude queue can never contain a Codex provider (and vice versa). + // Failover keeps the conversation's model and only switches which provider + // serves it, so the queue holds providers, not models. + const vendorProviders = useMemo( + () => settings.customProviders.filter((provider) => provider.type === providerType), + [settings.customProviders, providerType], + ); + + const queueValues = useMemo(() => new Set(failover.queue), [failover.queue]); + const addableProviders = useMemo( + () => vendorProviders.filter((provider) => !queueValues.has(provider.id)), + [vendorProviders, queueValues], + ); + + function patchFailover(patch: Partial) { + setSettings((prev) => updateModelFailover(prev, providerType, patch)); + } + + function queueEntryLabel(providerId: string) { + const provider = settings.customProviders.find((item) => item.id === providerId); + return provider?.name ?? providerId; + } + + function queueEntryDetail(providerId: string) { + const provider = settings.customProviders.find((item) => item.id === providerId); + return provider?.baseUrl ?? ""; + } + + function addQueueEntry(providerId: string) { + if (!providerId || queueValues.has(providerId)) return; + patchFailover({ queue: [...failover.queue, providerId] }); + } + + function removeQueueEntry(index: number) { + patchFailover({ queue: failover.queue.filter((_, i) => i !== index) }); + } + + function moveQueueEntry(index: number, delta: -1 | 1) { + const target = index + delta; + if (target < 0 || target >= failover.queue.length) return; + const queue = failover.queue.slice(); + const [entry] = queue.splice(index, 1); + queue.splice(target, 0, entry); + patchFailover({ queue }); + } + + return ( +
+
+
+
+ + + + {getProviderLabel(providerType)} + + {failover.enabled ? ( + + {t("settings.failoverEnabled")} + + ) : null} +
+

+ {t("settings.failoverToggleHint").replaceAll( + "{vendor}", + getProviderLabel(providerType), + )} +

+
+ patchFailover({ enabled: checked })} + ariaLabel={t("settings.failoverTitle")} + /> +
+ +
+ +

+ {t("settings.failoverQueueHint").replaceAll("{vendor}", getProviderLabel(providerType))} +

+ {failover.queue.length > 0 ? ( +
+ {failover.queue.map((entry, index) => ( +
+ + {index + 1} + + + {queueEntryLabel(entry)} + {queueEntryDetail(entry) ? ( + + {queueEntryDetail(entry)} + + ) : null} + + + + +
+ ))} +
+ ) : ( +
+ {t("settings.failoverQueueEmpty")} +
+ )} + {failover.queue.length < MODEL_FAILOVER_QUEUE_LIMIT && addableProviders.length > 0 ? ( + + ) : null} +
+ +
+ patchFailover({ maxSwitches: value })} + /> + patchFailover({ failureThreshold: value })} + /> + patchFailover({ cooldownSeconds: value })} + /> +
+
+ ); +} + +function CustomSettingsDrawer( + props: SettingsSectionProps & { providerType: ProviderId; onClose: () => void }, +) { + const { settings, setSettings, providerType, onClose } = props; const { t } = useLocale(); const [closing, setClosing] = useState(false); const closeTimerRef = useRef | null>(null); @@ -2673,9 +2927,6 @@ function CustomSettingsDrawer(props: SettingsSectionProps & { onClose: () => voi > {t("settings.customSettings")} -
- {t("settings.conversationTitleModelHint")} -
+ + + + ))} + + ) : ( +
+ {t("settings.failoverQueueEmpty")} +
+ )} + {failover.queue.length < MODEL_FAILOVER_QUEUE_LIMIT && addableProviders.length > 0 ? ( + + ) : null} + + +
+ patchFailover({ maxSwitches: value })} + /> + patchFailover({ failureThreshold: value })} + /> + patchFailover({ cooldownSeconds: value })} + /> +
+ + ); +} + +function CustomSettingsDrawer( + props: SettingsSectionProps & { providerType: ProviderId; onClose: () => void }, +) { + const { settings, setSettings, providerType, onClose } = props; const { t } = useLocale(); const [closing, setClosing] = useState(false); const closeTimerRef = useRef | null>(null); @@ -2725,9 +2978,6 @@ function CustomSettingsDrawer(props: SettingsSectionProps & { onClose: () => voi > {t("settings.customSettings")} -
- {t("settings.conversationTitleModelHint")} -