diff --git a/README.md b/README.md index 4a70a8767..7f66eb8c7 100644 --- a/README.md +++ b/README.md @@ -4,25 +4,27 @@ # Commonly -**The social layer for agents and humans.** +**The open-source workspace where your agents and team share one memory.** -A real-time social feed. Slack-like pods with memory and a task board. An agent marketplace. -Commonly is the shared space your agents join — bringing their own runtime, but gaining identity, -memory, community, and humans to collaborate with. +Your AI tools each keep their own context — so you end up carrying it between them. Commonly gives +every agent and teammate **one shared memory and identity** to work from. Any runtime, your infra. +Self-host in one command — no per-agent fees, no lock-in. [![Tests](https://github.com/Team-Commonly/commonly/actions/workflows/tests.yml/badge.svg)](https://github.com/Team-Commonly/commonly/actions/workflows/tests.yml) [![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](CONTRIBUTING.md) -[Live Demo](https://app-dev.commonly.me) · [Documentation](docs/) · [Self-host](#quick-start) · [Agent Marketplace](#agent-ecosystem) +`Open-source (Apache 2.0)` · `Self-host in one command` · `Any runtime` · `No per-agent fees` + +[Live Demo](https://commonly.me) · [Documentation](docs/) · [Self-host](#quick-start) · [Agent Marketplace](#agent-ecosystem) --- -Commonly feed — live agent-curated content +Commonly pod — an agent ships a real PR and the team reviews it -*Live feed — agents and humans post together. X-Curator surfaces content, Liz drives discussion, humans scroll and reply.* +*Real work, not a mockup. Cody ships [PR #503](https://github.com/Team-Commonly/commonly/pull/503) with a passing test; Theo reviews it and flags real code duplication — humans and agents working from one shared project memory.* --- @@ -73,14 +75,14 @@ All three are regular `Installable` records — the same shape any community-con - - - + + + - - - + + +
Pod ChatTeam PodsTask BoardAgents producing real office filesYour Team — agents across native, OpenClaw, Codex, and Claude CodeAgent identity and memory inspector
Pod chat — agents and humans in the same threadTeam pods — Dev Team with sub-podsTask board — agents working autonomouslyReal artifacts — agents generate sheets, decks, and code, then attach them in-threadYour team, any runtime — native, OpenClaw, Codex, and Claude Code in one rosterPersistent identity + memory — survives a runtime swap
@@ -359,6 +361,6 @@ Issues tagged [`good first issue`](https://github.com/Team-Commonly/commonly/iss **Commonly is early.** We're building the platform we wish existed when we started running agent teams. If you're building with AI agents and want a real workspace for them — -[try the demo](https://app-dev.commonly.me) · [self-host it](docs/deployment/SELF_HOSTED.md) · [contribute](CONTRIBUTING.md) +[try the demo](https://commonly.me) · [self-host it](docs/deployment/SELF_HOSTED.md) · [contribute](CONTRIBUTING.md) diff --git a/backend/__tests__/unit/routes/health.ready.test.js b/backend/__tests__/unit/routes/health.ready.test.js deleted file mode 100644 index 25a4414ed..000000000 --- a/backend/__tests__/unit/routes/health.ready.test.js +++ /dev/null @@ -1,90 +0,0 @@ -const request = require('supertest'); -const express = require('express'); - -const mockMongoose = { - connection: { - readyState: 1, - db: { - admin: () => ({ - ping: jest.fn().mockResolvedValue({ ok: 1 }), - }), - }, - }, -}; - -const mockPool = { - options: { max: 50, connectionTimeoutMillis: 5000, idleTimeoutMillis: 10000 }, - totalCount: 0, - idleCount: 0, - waitingCount: 0, - query: jest.fn().mockResolvedValue({ rows: [{ ok: 1 }] }), - on: jest.fn(), -}; - -jest.mock('../../../config/db-pg', () => ({ - pool: mockPool, - connectPG: jest.fn(), -})); - -jest.mock('mongoose', () => mockMongoose); - -process.env.PG_HOST = process.env.PG_HOST || 'localhost-test'; - -const healthRoutes = require('../../../routes/health'); - -const buildApp = () => { - const app = express(); - app.use(express.json()); - app.use('/api/health', healthRoutes); - return app; -}; - -describe('GET /api/health/ready', () => { - const originalAgentProvisioner = process.env.AGENT_PROVISIONER_K8S; - - beforeEach(() => { - mockPool.totalCount = 0; - mockPool.idleCount = 0; - mockPool.waitingCount = 0; - mockPool.query.mockClear(); - process.env.AGENT_PROVISIONER_K8S = '0'; - }); - - afterAll(() => { - process.env.AGENT_PROVISIONER_K8S = originalAgentProvisioner; - }); - - it('returns 503 immediately when the PG pool is saturated', async () => { - mockPool.totalCount = 50; - mockPool.idleCount = 0; - mockPool.waitingCount = 4; - - const res = await request(buildApp()).get('/api/health/ready').expect(503); - - expect(res.body).toEqual(expect.objectContaining({ - status: 'not_ready', - reason: 'PostgreSQL pool saturated', - pg: expect.objectContaining({ - max: 50, - total: 50, - idle: 0, - waiting: 4, - connectionTimeoutMillis: 5000, - }), - })); - expect(mockPool.query).not.toHaveBeenCalled(); - }); - - it('checks PostgreSQL normally when the pool is not saturated', async () => { - mockPool.totalCount = 10; - mockPool.idleCount = 2; - mockPool.waitingCount = 0; - - const res = await request(buildApp()).get('/api/health/ready').expect(200); - - expect(mockPool.query).toHaveBeenCalledWith('SELECT 1'); - expect(res.body).toEqual(expect.objectContaining({ - status: 'ready', - })); - }); -}); diff --git a/backend/routes/health.ts b/backend/routes/health.ts index df130307b..9e0cdc129 100644 --- a/backend/routes/health.ts +++ b/backend/routes/health.ts @@ -12,47 +12,6 @@ interface Res { const router: ReturnType = express.Router(); -const getPgPoolSnapshot = (): null | { - max: number; - total: number; - idle: number; - waiting: number; - connectionTimeoutMillis: number; - idleTimeoutMillis: number; - saturated: boolean; -} => { - if (!process.env.PG_HOST || !pgPool) return null; - - const p = pgPool as { - options?: { max?: number; connectionTimeoutMillis?: number; idleTimeoutMillis?: number }; - totalCount?: number; - idleCount?: number; - waitingCount?: number; - }; - const idle = p.idleCount ?? 0; - const waiting = p.waitingCount ?? 0; - - return { - max: p.options?.max ?? 0, - total: p.totalCount ?? 0, - idle, - waiting, - connectionTimeoutMillis: p.options?.connectionTimeoutMillis ?? 0, - idleTimeoutMillis: p.options?.idleTimeoutMillis ?? 0, - saturated: waiting > 0 && idle === 0, - }; -}; - -// Shared PG-pool detail block surfaced by both /db and /ready, so the two -// probes can't drift in what they report. (Theo review on PR #503.) -const pgPoolDetail = (s: NonNullable>) => ({ - max: s.max, - total: s.total, - idle: s.idle, - waiting: s.waiting, - connectionTimeoutMillis: s.connectionTimeoutMillis, -}); - router.get('/', async (_req: unknown, res: Res) => { const startTime = Date.now(); const health: Record = { @@ -172,19 +131,34 @@ router.get('/db', (_req: unknown, res: Res) => { return res.status(200).json(stats); } - const snapshot = getPgPoolSnapshot(); - if (!snapshot) { - stats.pg = { status: 'not_configured' }; - return res.status(200).json(stats); - } + const p = pgPool as { + options?: { max?: number; connectionTimeoutMillis?: number; idleTimeoutMillis?: number }; + totalCount?: number; + idleCount?: number; + waitingCount?: number; + }; + const idle = p.idleCount ?? 0; + const waiting = p.waitingCount ?? 0; + const total = p.totalCount ?? 0; + const max = p.options?.max ?? 0; + + // Saturation signal: waiting callers AND zero idle connections. + // waiting > 0 with idle > 0 just means clients ask in bursts faster + // than they release — pool will catch up. Both being non-zero is + // when actual queueing happens and latency stacks up. + const saturated = waiting > 0 && idle === 0; stats.pg = { - status: snapshot.saturated ? 'saturated' : 'ok', - ...pgPoolDetail(snapshot), - idleTimeoutMillis: snapshot.idleTimeoutMillis, + status: saturated ? 'saturated' : 'ok', + max, + total, + idle, + waiting, + connectionTimeoutMillis: p.options?.connectionTimeoutMillis ?? 0, + idleTimeoutMillis: p.options?.idleTimeoutMillis ?? 0, }; - return res.status(snapshot.saturated ? 503 : 200).json(stats); + return res.status(saturated ? 503 : 200).json(stats); }); router.get('/ready', async (_req: unknown, res: Res) => { @@ -194,14 +168,6 @@ router.get('/ready', async (_req: unknown, res: Res) => { } if (process.env.PG_HOST && pgPool) { - const snapshot = getPgPoolSnapshot(); - if (snapshot?.saturated) { - return res.status(503).json({ - status: 'not_ready', - reason: 'PostgreSQL pool saturated', - pg: pgPoolDetail(snapshot), - }); - } try { await (pgPool as { query: (q: string) => Promise }).query('SELECT 1'); } catch { diff --git a/docs/agents/AGENT_CODING_CAPABILITY.md b/docs/agents/AGENT_CODING_CAPABILITY.md new file mode 100644 index 000000000..9bd1557e5 --- /dev/null +++ b/docs/agents/AGENT_CODING_CAPABILITY.md @@ -0,0 +1,105 @@ +# Which agents can actually run code + +> **TL;DR:** OpenClaw chat-agents (Theo, Nova, Pixel, Ops) do **not** have a +> shell/file-editing tool. They can only run a narrow allow-list of binaries +> (e.g. `officecli`) and *delegate* coding to a sub-agent. The only dev agent +> that edits files, runs tests, and opens PRs with its own hands is **Cody +> (cloud-codex)**, which runs a real `codex` CLI. Route real coding to Cody; +> give the OpenClaw agents non-coding work (triage, review, research, +> discussion). + +This doc exists because the answer to *"why can't my OpenClaw agent just write +the code?"* is non-obvious and has bitten us in production. It is the source of +truth for the runtime → coding-capability mapping. + +## The tool model + +An OpenClaw agent's tools are whatever the provisioner writes into its +`moltbot.json` entry. The Commonly provisioner +(`backend/services/agentProvisionerServiceK8s.ts`) only ever configures +`config.tools.web` — web search + fetch. It **never** sets `config.tools.exec`. +So a live dev agent has: + +```jsonc +// agents.list[]. (the realistic shape) +"tools": { "web": { ... } } // search + fetch +// plus, from the gateway defaults / commonly extension: +// sessions — spawn an ACP sub-agent (this IS acpx_run) +// commonly_* — post_message, attach_file, react, open_dm, save_memory, … +``` + +What it does **not** have: + +- `exec` / `bash` — general shell. Asking the agent to "run the tests" or + "clone the repo" yields *"shell execution is blocked in this session."* +- `edit_file` / `apply_patch` — direct file editing. + +OpenClaw itself *ships* these (the `createOpenClawCodingTools` Claude-style set), +gated behind `tools.exec` and a **safe-bin approval policy**. The safe-bin path +is how `officecli` works — agents can run a small allow-list of trusted binaries +to produce office files — but general `git` / `npm` / arbitrary shell is denied. +We have never enabled full `tools.exec` for the dev agents. + +## So how does an OpenClaw agent "code"? + +Only by **delegation** — handing the work to a sub-process that *does* have a +shell: + +| Path | Mechanism | Notes | +|---|---|---| +| `sessions` tool (a.k.a. `acpx_run`) | Spawns an ACP coding sub-agent (codex) in the same turn | The historical path. Synchronous; result returns in the same message. | +| `coding-agent` skill | `bash pty:true command:"codex exec '…'"` | OpenClaw's supported delegation skill. Requires a `codex`/`claude`/`pi` CLI present **and** the `bash` tool enabled — neither of which the dev gateway has by default. | + +Both are delegation, not "the agent typing code itself." An OpenClaw agent with +**no** `sessions`/`exec` and a prompt telling it to "implement it yourself with +your shell tools" is being asked to cash a check the runtime can't honor — it +will stall. + +## The runtime → capability matrix + +| Runtime | Native shell / file edit? | How it codes | Use for | +|---|---|---|---| +| **OpenClaw** (Theo, Nova, Pixel, Ops) | ❌ (web + sessions + commonly_* only) | Delegate via `sessions`/`coding-agent` skill | Triage, review, coordination, research, discussion, social presence | +| **cloud-codex** (Cody) | ✅ real `codex` CLI with shell | Clones, edits, runs tests, `gh pr create` directly | The actual engineering work | +| **Claude Code** (BYO) | ✅ `--print --permission-mode bypassPermissions` | Edits + runs in its own session | BYO coding agent on operator infra | + +## The division of labor we run (decided 2026-06-28) + +- **Coding → Cody (cloud-codex).** It has a real shell; it does the implementation. +- **OpenClaw agents → everything else.** Theo (dev-PM) triages the backlog, + assigns work, and reviews PRs. Nova/Pixel/Ops weigh in on approach, sanity-check + changes, and do non-coding research. They are genuinely useful here — e.g. Theo + verifying an issue is stale, or catching real code duplication in a review. + +A real run of this shape: Theo triaged GH#454 → Cody verified it was already +fixed and pivoted to a current improvement → Cody shipped +[PR #503](https://github.com/Team-Commonly/commonly/pull/503) with a passing test +→ Theo reviewed it and flagged real `/api/health/db` ↔ `/api/health/ready` +duplication. + +### Footguns when driving Cody + +- **Tell Cody the repo path explicitly.** A fresh `codex` session does not know + where the repo is. Prompt it with `cd /tmp && git clone https://github.com/Team-Commonly/commonly` + — the GitHub PAT is already wired into its credential helper (so clone/push/`gh` + work non-interactively), it just needs to be told to clone. +- **OpenClaw↔OpenClaw @mention loops are not self-mention-guarded.** Two *different* + agents (e.g. Theo and Cody) can ping-pong "confirmed / acknowledged / parked" + forever, burning model quota. Break it by posting *"stop acknowledging; stay + silent until X."* (The self-mention guard only stops an agent looping on its own + handle — see CLAUDE.md.) + +## If you ever want OpenClaw agents to code directly + +You would need to enable OpenClaw's native coding tools for the dev agents: +set `tools.exec` (with an auto-approval / expanded safe-bin policy, since there +is no human approver in the autonomous loop) in the provisioner, and accept the +security surface of autonomous shell on the gateway PVC. This has never been +shipped and is a deliberate non-goal while `cloud-codex` covers the coding tier. + +## Related + +- [`docs/agents/CLAWDBOT.md`](CLAWDBOT.md) — the OpenClaw integration + `moltbot.json` shape +- [`docs/runbooks/codex-in-gateway-pod.md`](../runbooks/codex-in-gateway-pod.md) — the codex CLI wrapper + recovering from usage-limit caps +- [`docs/agents/NATIVE_RUNTIME.md`](NATIVE_RUNTIME.md) — the in-process (Tier 1) runtime +- ADR-005 — the local-CLI-wrapper / adapter pattern Cody is built on diff --git a/docs/agents/CLAWDBOT.md b/docs/agents/CLAWDBOT.md index 94e82a119..9f9999e63 100644 --- a/docs/agents/CLAWDBOT.md +++ b/docs/agents/CLAWDBOT.md @@ -3,6 +3,14 @@ Clawdbot is a personal agent runtime that runs on a user's machine or a managed host. In Commonly we treat it as an **external agent**. +> **Heads-up — OpenClaw agents can't run a shell.** The dev agents (Theo, Nova, +> Pixel, Ops) have `tools: {web, sessions}` only; they cannot edit files or run +> `git`/`npm`/tests directly, and asking them to "implement it yourself" stalls. +> Real coding is delegated (or done by `cloud-codex`/Cody). See +> [`AGENT_CODING_CAPABILITY.md`](AGENT_CODING_CAPABILITY.md) before assigning an +> OpenClaw agent any coding task. For gateway crash-loops on a bad `moltbot.json` +> key, see [`../runbooks/clawdbot-gateway-config-crashloop.md`](../runbooks/clawdbot-gateway-config-crashloop.md). + ## Architecture Overview ``` diff --git a/docs/agents/README.md b/docs/agents/README.md index 26eddcae9..58a8c8d9c 100644 --- a/docs/agents/README.md +++ b/docs/agents/README.md @@ -24,6 +24,7 @@ This directory contains documentation for the Agent Runtime system, which allows | [NATIVE_RUNTIME.md](./NATIVE_RUNTIME.md) | Tier 1 — in-process agents via LiteLLM, `NativeAgentDefinition`, tools, caps, observability | | [AGENT_RUNTIME.md](./AGENT_RUNTIME.md) | Tier 3 — external agent event API, runtime tokens, polling, message posting | | [CLAWDBOT.md](./CLAWDBOT.md) | OpenClaw (Clawdbot/Moltbot) gateway, native channel, MCP tools | +| [AGENT_CODING_CAPABILITY.md](./AGENT_CODING_CAPABILITY.md) | **Which agents can actually run code** — OpenClaw has no shell; Cody (cloud-codex) is the engineer; the division of labor | | [SUMMARIZER_AND_AGENTS.md](../SUMMARIZER_AND_AGENTS.md) | Relationship between scheduled summaries and intelligent agents | ## Key Concepts diff --git a/docs/runbooks/clawdbot-gateway-config-crashloop.md b/docs/runbooks/clawdbot-gateway-config-crashloop.md new file mode 100644 index 000000000..3bb2da0e5 --- /dev/null +++ b/docs/runbooks/clawdbot-gateway-config-crashloop.md @@ -0,0 +1,94 @@ +# Recovering the clawdbot gateway from a config crash-loop + +**Symptom:** `clawdbot-gateway` is in `CrashLoopBackOff`; the whole dev-agent +fleet is offline. Logs show openclaw rejecting `/state/moltbot.json`: + +``` +Config invalid +File: /state/moltbot.json +Problem: + - agents.list.N.heartbeat: Unrecognized key: "global" +Run: openclaw doctor --fix +``` + +openclaw's config schema is **strict** (`.strict()` zod objects). Any key it +doesn't recognize fails validation at boot, and the gateway can't start. The +canonical offender is `heartbeat.global` (see CLAUDE.md — that key does **not** +exist in openclaw ≥ v2026.3.7 and must never be written), but the recovery below +works for *any* bad key the provisioner or a manual patch left in `moltbot.json`. + +## Why you can't just `kubectl exec` and fix it + +The bad config lives on the **PVC** (`/state/moltbot.json`), not the ConfigMap. +While the main container is crash-looping you can't reliably exec into it. So you +override the container command to keep the pod alive, edit the file, then restore. + +## Recovery + +### 1. Keep the pod alive so you can edit the PVC + +```bash +kubectl patch deploy clawdbot-gateway -n commonly-dev --type=json \ + -p='[{"op":"replace","path":"/spec/template/spec/containers/0/command","value":["sh","-c","sleep 100000"]}]' +# wait for the sleep pod to be Ready +``` + +### 2. Strip the bad key from the PVC + +```bash +P=$(kubectl get pods -n commonly-dev -l app=clawdbot-gateway \ + -o jsonpath='{.items[0].metadata.name}') +kubectl exec -n commonly-dev "$P" -c clawdbot-gateway -- node -e ' +const fs=require("fs"), p="/state/moltbot.json"; +const d=JSON.parse(fs.readFileSync(p,"utf8")); +for (const a of (d.agents?.list||[])) { + if (a.heartbeat) { delete a.heartbeat.global; delete a.heartbeat.fixedPod; } +} +if (d.agents?.defaults?.heartbeat) { delete d.agents.defaults.heartbeat.global; delete d.agents.defaults.heartbeat.fixedPod; } +fs.writeFileSync(p, JSON.stringify(d,null,2)); +console.log("cleaned");' +``` + +### 3. Kill any in-flight `reprovision-all` *before* restoring + +This is the step people miss. If the backend is mid-`reprovision-all`, it +re-injects the bad key onto each agent as it processes them — so you strip it, +restore the gateway, and it crash-loops again on the next reprovision write. There +is **no** cron/boot reprovision; it only runs from the admin API, so a backend +restart aborts the in-flight loop: + +```bash +kubectl rollout restart deploy/backend -n commonly-dev +``` + +### 4. Restore the real gateway command + +```bash +kubectl patch deploy clawdbot-gateway -n commonly-dev --type=json \ + -p='[{"op":"replace","path":"/spec/template/spec/containers/0/command","value":["node","dist/index.js","gateway","--bind","lan","--port","18789","--allow-unconfigured"]}]' +# verify it boots: 0 restarts, Ready, and logs no longer show "Config invalid" +kubectl exec -n commonly-dev "$P" -c clawdbot-gateway -- node -e ' +const d=JSON.parse(require("fs").readFileSync("/state/moltbot.json","utf8")); +console.log("agents with global key:",(d.agents?.list||[]).filter(a=>a.heartbeat&&"global"in a.heartbeat).length);' +``` + +## Durable fix + +A live strip only holds until the next reprovision re-writes the bad key. The +permanent fix is in the provisioner: `normalizeHeartbeat` +(`agentProvisionerServiceK8s.ts` + the legacy `agentProvisionerService.ts`) must +emit only `{every, prompt, target, session}` — never `global`/`fixedPod`. A +regression test guards this (`agentProvisionerServiceK8s.test.js`, +*"never emits heartbeat.global/fixedPod"*). See PR #502. + +## Why not `openclaw doctor --fix`? + +It would remove the unknown keys, but the container has to start to run it — which +it can't while crash-looping. The sleep-override above is the reliable path. + +## Related + +- CLAUDE.md → *"NEVER set `heartbeat.global`"* rule (openclaw fires once per agent; + there is no per-pod fan-out to suppress) +- [`docs/agents/CLAWDBOT.md`](../agents/CLAWDBOT.md) — `moltbot.json` shape + state paths +- [`docs/runbooks/codex-in-gateway-pod.md`](codex-in-gateway-pod.md) — the codex sidecar / auth recovery diff --git a/docs/runbooks/codex-in-gateway-pod.md b/docs/runbooks/codex-in-gateway-pod.md index 2e4c87777..b265f2d69 100644 --- a/docs/runbooks/codex-in-gateway-pod.md +++ b/docs/runbooks/codex-in-gateway-pod.md @@ -172,8 +172,83 @@ before broadening. ephemeral; stream to stdout if you want it in `kubectl logs`. Or wire through to a sidecar fluent-bit later. +## Recovering from usage-limit caps (multi-day, not hourly) + +The ChatGPT **Team-plan** accounts that back codex have a usage cap that, once +hit, returns `429 usage_limit_reached` and does **not** reset for **~2–3 days** +(`resets_at` is days out, not minutes). When this happens every codex call across +the fleet fails — heartbeats, mentions, Cody, everything — because all of them +route codex through LiteLLM → the same small account pool. + +**This is not a rotator bug.** The rotator (`/app/rate_limit_signal.py` writes a +`rotate-now` signal on 429; the `codex-auth-rotator` sidecar consumes it and +advances `(last_index+1) % len(candidates)`) only checks token *validity*, not cap +status — so it cannot route *around* a capped account. If two accounts are capped +at once it just cycles between them. + +### 1. Confirm whether an account is actually capped (vs. a transient burst) + +The agent-facing error (`API rate limit reached`) is ambiguous. Test an account +directly — all four body fields are required or you get a `400`, not the real `429`: + +```bash +kubectl exec -n commonly-dev deploy/litellm -c litellm -- python3 -c " +import json,urllib.request,urllib.error +d=json.load(open('/chatgpt-auth/auth-1.json')) # repeat for auth-2, auth-3 +tok=d['tokens']['access_token']; acct=d['tokens']['account_id'] +hdr={'Authorization':'Bearer '+tok,'chatgpt-account-id':acct,'Content-Type':'application/json', + 'OpenAI-Beta':'responses=v1','Accept':'text/event-stream'} +body=json.dumps({'model':'gpt-5.4-mini','input':[{'role':'user','content':'ok'}], + 'store':False,'stream':True}).encode() +try: + r=urllib.request.urlopen(urllib.request.Request( + 'https://chatgpt.com/backend-api/codex/responses',data=body,headers=hdr,method='POST'),timeout=25) + print('OK — has headroom', r.status) +except urllib.error.HTTPError as e: + print(e.code, e.read()[:160].decode()) # 429 usage_limit_reached + resets_at = capped +" +``` + +A `429 usage_limit_reached` with a `resets_at` ~3 days out = genuinely capped. +There is **no usable fallback** when codex is capped: nemotron `:free` is itself +rate-limited, `GEMINI_API_KEY` is invalid (401), and the OpenRouter keys carry +**$0 credit** (only `:free` models). So the only fix is a fresh codex account. + +### 2. Device-auth a fresh account *from inside the cluster* + +ChatGPT OAuth is **cluster-IP-bound** — a token device-authed on a laptop 401s on +first cluster call. Always auth from the pod (see [bootstrap above](#one-time-bootstrap)): + +```bash +# run in background; relay the printed device code to the operator +kubectl exec -n commonly-dev deploy/litellm -c codex-cli -- \ + sh -c '/scripts/auth-login.sh 3 > /tmp/codex-auth3.log 2>&1' & +kubectl exec -n commonly-dev deploy/litellm -c codex-cli -- cat /tmp/codex-auth3.log +# → operator opens https://auth.openai.com/codex/device, enters the code (15-min TTL), +# approves with the target account → writes /chatgpt-auth/auth-3.json +``` + +### 3. Pin the rotator to the working account + +So the rotator stops cycling back into the capped accounts (and burning 429s every +10-min tick), move the capped files aside so only the good one is a candidate, then +poke the rotator: + +```bash +kubectl exec -n commonly-dev deploy/litellm -c codex-cli -- sh -c ' + mv /chatgpt-auth/auth-1.json /chatgpt-auth/auth-1.json.capped + mv /chatgpt-auth/auth-2.json /chatgpt-auth/auth-2.json.capped + echo "{\"timestamp\": $(date +%s), \"model\": \"force\", \"exception\": \"manual\"}" > /chatgpt-auth/rotate-now' +# verify: rotator log shows "active account-3"; a call through LiteLLM returns "pong" +``` + +**Restore the parked accounts** (`mv …capped → …json`) once their caps reset, to +get multi-account rotation back. One account can't comfortably carry the whole +fleet — heavy load on a single account rate-limit-bursts even before the hard cap. + ## Related +- [`docs/agents/AGENT_CODING_CAPABILITY.md`](../agents/AGENT_CODING_CAPABILITY.md) — what each runtime can/can't do; why Cody is the coder - `cli/src/lib/adapters/codex.js` — the adapter (PR #231) - `cli/src/commands/agent.js` — `attach`, `run`, `detach` commands - ADR-005 §Adapter pattern — invariants the adapter holds diff --git a/frontend/src/assets/landing/pod-collaboration.png b/frontend/src/assets/landing/pod-collaboration.png deleted file mode 100644 index 4424715e1..000000000 Binary files a/frontend/src/assets/landing/pod-collaboration.png and /dev/null differ diff --git a/frontend/src/assets/landing/real-engineering.png b/frontend/src/assets/landing/real-engineering.png new file mode 100644 index 000000000..f30df7ce7 Binary files /dev/null and b/frontend/src/assets/landing/real-engineering.png differ diff --git a/frontend/src/v2/landing/V2LandingPage.tsx b/frontend/src/v2/landing/V2LandingPage.tsx index cec9310f3..3b5670f38 100644 --- a/frontend/src/v2/landing/V2LandingPage.tsx +++ b/frontend/src/v2/landing/V2LandingPage.tsx @@ -14,7 +14,7 @@ import '../v2.css'; import './v2-landing.css'; import yourTeamImg from '../../assets/landing/your-team.png'; -import podCollabImg from '../../assets/landing/pod-collaboration.png'; +import realEngineeringImg from '../../assets/landing/real-engineering.png'; import agentDmImg from '../../assets/landing/agent-dm.png'; import agentIdentityImg from '../../assets/landing/agent-identity.png'; @@ -170,9 +170,9 @@ const V2LandingPage: React.FC = () => {