import { agent, configureAgent, repair, s, workflow } from "rig";
import type { AgentFactory, AgentOptions } from "rig";
import { execSync } from "node:child_process";
import { writeFileSync } from "node:fs";
// Custom engine using the OpenAI-compatible api-proxy
function apiProxyEngine(): AgentFactory {
const MODEL_MAP: Record<string, string> = {
small: "claude-haiku-4.5",
medium: "claude-sonnet-4.6",
large: "claude-opus-4.5",
nano: "claude-haiku-4.5",
mini: "claude-haiku-4.5",
};
return (options: AgentOptions) => {
const messages: Array<{ role: string; content: string }> = [];
const systemMessage = options.systemMessage as string | undefined;
const modelId = MODEL_MAP[options.model ?? "small"] ?? options.model ?? "claude-sonnet-4.6";
return {
async ask(prompt: string, askOptions?: { signal?: AbortSignal; outputSchema?: Record<string, unknown> }) {
const requestMessages = [
...(systemMessage ? [{ role: "system", content: systemMessage }] : []),
...messages,
{ role: "user", content: prompt },
];
const body: Record<string, unknown> = {
model: modelId,
messages: requestMessages,
max_tokens: 8192,
};
if (askOptions?.outputSchema) {
(body as Record<string, unknown>)["response_format"] = { type: "json_schema", json_schema: { name: "output", schema: askOptions.outputSchema, strict: false } };
}
const response = await fetch("(apiproxy/redacted) {
method: "POST",
headers: {
"Content-Type": "application/json",
"Authorization": "Bearer dummy",
},
body: JSON.stringify(body),
...(askOptions?.signal != null ? { signal: askOptions.signal } : {}),
});
if (!response.ok) {
const errText = await response.text();
throw new Error(`API error: ${response.status} ${errText}`);
}
const data = await response.json() as { choices: Array<{ message: { content: string } }> };
const text = data.choices[0]?.message?.content ?? "";
messages.push({ role: "user", content: prompt });
messages.push({ role: "assistant", content: text });
return text;
},
async close() {},
};
};
}
configureAgent(apiProxyEngine());
const DEADLINE = Date.now() + 25 * 60_000;
const SKILL_DIR = "/home/runner/work/rig/rig/.github/skills/rig";
const GEN_FILE = "/tmp/gh-aw/agent/gen_prog.ts";
function msLeft(): number {
return Math.max(0, DEADLINE - Date.now());
}
function execTimeout() {
return Math.min(5 * 60_000, msLeft());
}
function runCmd(cmd: string, timeoutMs: number): { exitCode: number; output: string } {
try {
const out = execSync(cmd, {
cwd: SKILL_DIR,
encoding: "utf8",
timeout: timeoutMs,
stdio: ["pipe", "pipe", "pipe"],
});
return { exitCode: 0, output: out };
} catch (e: unknown) {
const err = e as { stdout?: string; stderr?: string; status?: number };
return { exitCode: err.status ?? 1, output: (err.stdout ?? "") + (err.stderr ?? "") };
}
}
// Agent role: pick a concrete, complicated task suitable for multi-agent decomposition.
const taskPicker = agent({
name: "taskPicker",
model: "small",
maxTurns: 3,
addons: [repair()],
output: s.object({
title: s.nonEmptyString("short task title"),
domain: s.nonEmptyString("domain area"),
description: s.nonEmptyString("one-paragraph description"),
successCriteria: s.array(s.nonEmptyString("one checkable criterion"), "3-6 success criteria"),
}),
instructions: `Pick one concrete, complicated task a person or team might face in a single day that would naturally benefit from being split across sub-agents running different models.
Good task types: multi-source research/synthesis, multi-file coding with design/implementation/review phases, structured report from several independent analyses, multi-step data transformation pipeline.
Requirements: self-contained (solvable from description alone, no external files or live web), concrete enough to grade, varied domain.
Return title, domain, description, and 3-6 concrete checkable success criteria.`,
});
// Agent role: solve the full task in a single call with no delegation.
const singleCallSolver = agent({
name: "singleCallSolver",
model: "medium",
maxTurns: 1,
output: s.object({
solution: s.nonEmptyString("complete solution addressing every success criterion"),
}),
});
// Agent role: write a rig TypeScript program that decomposes the task across multiple agents.
const programWriter = agent({
name: "programWriter",
model: "medium",
maxTurns: 1,
output: s.object({
source: s.nonEmptyString("complete TypeScript program source without markdown fences"),
}),
});
// Agent role: fix a rig program given an error.
const programFixer = agent({
name: "programFixer",
model: "medium",
maxTurns: 1,
output: s.object({
source: s.nonEmptyString("fixed complete TypeScript program source without markdown fences"),
}),
});
// Agent role: grade two solutions against the task success criteria.
const grader = agent({
name: "grader",
model: "large",
maxTurns: 1,
output: s.object({
singleCallScore: s.number("0-10 score for the single-call solution"),
decomposedScore: s.number("0-10 score for the decomposed solution"),
winner: s.enum("single-call", "decomposed", "tie"),
rationale: s.nonEmptyString("explanation judging only on correctness and completeness"),
}),
});
// Workflow role: run the full decomposition benchmark.
const bench = workflow({
meta: {
name: "bench",
description: "Daily rig decomposition benchmark",
phases: ["PickTask", "SingleCall", "Decompose", "Grade"],
},
body: async ({ call, phase }) => {
// Step 1: Pick task
phase("PickTask");
const task = await call(taskPicker, "Pick a concrete complicated task for the benchmark.");
if (!task) throw new Error("Task picker returned null");
const criteriaList = task.successCriteria.map((c: string, i: number) => `${i + 1}. ${c}`).join("\n");
const taskPrompt = `Task: ${task.title}
Domain: ${task.domain}
Description: ${task.description}
Success Criteria:
${criteriaList}`;
// Step 2: Single-call solve
phase("SingleCall");
const scStart = Date.now();
const scResult = await call(singleCallSolver,
`Solve this task completely, addressing every success criterion:\n\n${taskPrompt}`,
{ timeout: execTimeout() }
);
const singleCallDurationMs = Date.now() - scStart;
const singleCallSolution = scResult?.solution ?? "[single-call solver returned null]";
// Step 3: Decomposed solve
phase("Decompose");
const dcStart = Date.now();
const writerPrompt = `Write a self-contained rig TypeScript program that solves this task by decomposing it across at least two agents:
${taskPrompt}
Requirements for the generated program:
- Import ONLY from "rig" (no Node.js built-ins, no other imports)
- Add "// Agent role: ..." above each agent(), "// Workflow role: ..." above each workflow()
- Use model: "small" for simple sub-steps, model: "medium" or model: "large" for harder ones
- Coordinate at least two agents so the final combined answer addresses all success criteria
- The root takes no required input
- Output schema must be: output: s.object({ solution: s.string })
- Export the root via "export default" WITHOUT invoking it
- Do NOT call configureAgent or copilotEngine
Return ONLY the raw TypeScript source code, no markdown fences, no extra text.`;
const attempts: Array<{
attemptNumber: number;
typecheckPassed: boolean;
executePassed: boolean;
typecheckOutput: string;
executeOutput: string;
fixNote: string;
}> = [];
let programSource = "";
let decomposedSolution = "";
let decomposedFinalStatus: "pass" | "fail" = "fail";
let lastSource = "";
let lastError = "";
for (let attempt = 1; attempt <= 2; attempt++) {
if (msLeft() < 60_000) break;
let source: string;
if (attempt === 1) {
const writerResult = await call(programWriter, writerPrompt, { timeout: execTimeout() });
source = writerResult?.source ?? "";
} else {
const fixerResult = await call(
programFixer,
`Fix this rig program. Correct exactly the error shown, preserving everything that worked.
Previous source:
${lastSource}
Error:
${lastError}
Return ONLY the fixed raw TypeScript source, no markdown fences.`,
{ timeout: execTimeout() }
);
source = fixerResult?.source ?? "";
}
if (!source) {
attempts.push({ attemptNumber: attempt, typecheckPassed: false, executePassed: false, typecheckOutput: "agent returned empty source", executeOutput: "", fixNote: "" });
lastError = "agent returned empty source";
continue;
}
// Strip markdown fences if present
source = source.replace(/^```(?:typescript|ts)?\n/, "").replace(/\n```$/, "").trim();
lastSource = source;
// Write source to file and typecheck
writeFileSync(GEN_FILE, source, "utf8");
const tcResult = runCmd(`cat ${GEN_FILE} | node rig.ts --typecheck`, Math.min(120_000, msLeft()));
const typecheckPassed = tcResult.exitCode === 0;
const typecheckOutput = tcResult.output;
let executePassed = false;
let executeOutput = "";
if (typecheckPassed && msLeft() > 90_000) {
// The generated program runs without --server, using the api-proxy via COPILOT_SDK_URI env
// but since that requires a connectionToken, we inject our custom engine into the subprocess
// by setting env vars. The program itself must call configureAgent.
// Actually: we need to inject the engine. The generated program does NOT call configureAgent.
// So we need to set an env var that tells rig.ts to use our custom engine.
// Since rig.ts checks COPILOT_SDK_URI, we can provide a fake env that points to api-proxy
// using a different approach: set RIG_ENGINE to a non-standard value... not supported.
//
// Alternative: prepend configureAgent call to the generated source before running.
const enginePreamble = `import { configureAgent } from "rig";
import type { AgentFactory, AgentOptions } from "rig";
function __apiProxyEngine(): AgentFactory {
const MODEL_MAP: Record<string, string> = { small: "claude-haiku-4.5", medium: "claude-sonnet-4.6", large: "claude-opus-4.5", nano: "claude-haiku-4.5", mini: "claude-haiku-4.5" };
return (options: AgentOptions) => {
const messages: Array<{ role: string; content: string }> = [];
const systemMessage = options.systemMessage as string | undefined;
const modelId = MODEL_MAP[options.model ?? "small"] ?? options.model ?? "claude-sonnet-4.6";
return {
async ask(prompt: string, askOptions?: { signal?: AbortSignal; outputSchema?: Record<string, unknown> }) {
const requestMessages = [...(systemMessage ? [{ role: "system", content: systemMessage }] : []), ...messages, { role: "user", content: prompt }];
const body: Record<string, unknown> = { model: modelId, messages: requestMessages, max_tokens: 8192 };
if (askOptions?.outputSchema) (body as Record<string, unknown>)["response_format"] = { type: "json_schema", json_schema: { name: "output", schema: askOptions.outputSchema, strict: false } };
const response = await fetch("(apiproxy/redacted) { method: "POST", headers: { "Content-Type": "application/json", "Authorization": "Bearer dummy" }, body: JSON.stringify(body), ...(askOptions?.signal != null ? { signal: askOptions.signal } : {}) });
if (!response.ok) throw new Error(\`API error: \${response.status} \${await response.text()}\`);
const data = await response.json() as { choices: Array<{ message: { content: string } }> };
const text = data.choices[0]?.message?.content ?? "";
messages.push({ role: "user", content: prompt });
messages.push({ role: "assistant", content: text });
return text;
},
async close() {},
};
};
}
configureAgent(__apiProxyEngine());
`;
const injectedSource = enginePreamble + "\n" + source;
writeFileSync(GEN_FILE, injectedSource, "utf8");
const exResult = runCmd(`cat ${GEN_FILE} | node rig.ts`, Math.min(300_000, msLeft()));
executePassed = exResult.exitCode === 0;
executeOutput = exResult.output;
if (executePassed) {
try {
const parsed = JSON.parse(executeOutput);
decomposedSolution = parsed?.solution ?? parsed?.output?.solution ?? executeOutput;
} catch {
decomposedSolution = executeOutput;
}
}
}
const fixNote = attempt < 2 && (!typecheckPassed || !executePassed) ? "Passed source and error to fixer agent" : "";
attempts.push({ attemptNumber: attempt, typecheckPassed, executePassed, typecheckOutput, executeOutput, fixNote });
lastError = !typecheckPassed ? typecheckOutput : executeOutput;
if (executePassed) {
programSource = source;
decomposedFinalStatus = "pass";
break;
}
if (typecheckPassed && !programSource) programSource = source;
}
if (!programSource && lastSource) programSource = lastSource;
const decomposedDurationMs = Date.now() - dcStart;
// Step 4: Grade
phase("Grade");
const gradeResult = await call(
grader,
`Grade two solutions to this task:
${taskPrompt}
SINGLE-CALL SOLUTION:
${singleCallSolution}
DECOMPOSED SOLUTION:
${decomposedSolution || "[decomposition did not produce a valid solution]"}
Score each 0-10 on how completely and correctly it satisfies the success criteria.
Judge only on correctness and completeness of content — not on length or which approach was used.
Pick a winner: single-call, decomposed, or tie.`,
{ timeout: execTimeout() }
);
return {
task: {
title: task.title,
domain: task.domain,
description: task.description,
successCriteria: task.successCriteria,
},
singleCallDurationMs,
decomposedDurationMs,
singleCallSolution,
decomposedSolution: decomposedSolution || "[decomposition did not produce a valid solution]",
decomposedProgramSource: programSource,
attempts,
decomposedFinalStatus,
grading: {
singleCallScore: gradeResult?.singleCallScore ?? 0,
decomposedScore: gradeResult?.decomposedScore ?? 0,
winner: gradeResult?.winner ?? "tie",
rationale: gradeResult?.rationale ?? "[grader returned null]",
},
};
},
});
export default bench;
Task
Title: Multi-Phase Financial Report Generation from Raw Data
Domain: Finance & Data Analysis
Description: A financial analyst receives quarterly transaction data, customer demographics, and market conditions, and must produce a comprehensive board report by end of day. The task requires: (1) data validation and cleaning to identify anomalies and reconcile discrepancies across three CSV files with different schemas, (2) independent statistical analysis of customer segments and revenue trends with visualization recommendations, (3) risk assessment based on market volatility and exposure metrics, (4) synthesis of findings into an executive summary with clear narrative flow, and (5) quality review ensuring all figures cross-reference correctly and recommendations are actionable. Different phases naturally benefit from different model strengths: data wrangling and validation, exploratory statistical analysis, risk modeling, executive communication, and fact-checking.
Success Criteria:
Timing Comparison
Decomposition Attempts
Attempt 1
Typecheck error output
What was fixed in attempt 2: The writer was asked to remove
systemfrom agent specs,namefrom workflow spec, remove unused params, and avoidinputin prompt intents.Attempt 2
Typecheck error output
No further attempts were made (budget exhausted or max attempts reached).
Grading
Winner:
single-callGrader's Rationale:
Single-Call Solution
Full single-call solution
Multi-Phase Financial Report Generation from Raw Data
PHASE 1: DATA VALIDATION AND CLEANING
Assumed Data Sources
transactions_Q3.csv— 1,240 rows, columns: transaction_id, customer_id, date, amount, product_category, regioncustomer_demographics.csv— 980 rows, columns: customer_id, age, segment, account_open_date, countrymarket_conditions.csv— 92 rows (daily), columns: date, index_value, volatility_score, sector_exposure_pctData Quality Issues Identified
Total issues: 150 cells across 3 files requiring remediation before analysis.
Reconciliation Actions
pd.to_datetime(dayfirst=True).REVENUE_MISSING; exclude from revenue totals but retain for customer count.OUTLIER_HIGH_VALUEfor secondary review; include in totals with notation.PHASE 2: STATISTICAL ANALYSIS — CUSTOMER SEGMENTS & REVENUE TRENDS
Dataset Post-Cleaning
Finding 1: Revenue by Customer Segment
Statistical Finding 1: Premium segment customers (14.8% of customer base) generated 41.3% of total revenue ($8,412,300 / $20,357,100 = 0.4133), up from 37.2% in Q2. This 4.1 percentage-point shift indicates accelerating revenue concentration in the top tier.
Statistical Finding 2: Basic segment revenue declined 6.7% quarter-over-quarter (from $3,045,800 in Q2 to $2,841,200 in Q3), while customer count in this segment fell only 1.2% (333 → 329), implying average spend per Basic customer dropped from $9,144 to $8,636 — a 5.6% decline in spend intensity signaling early churn risk.
Finding 3: Monthly Revenue Trend
Revenue grew 20.7% from July to September, suggesting sustained positive momentum rather than a one-time spike.
PHASE 3: RISK ASSESSMENT
Risk Category 1: Market Volatility Risk
Risk Category 2: Customer Concentration Risk
Risk Category 3: Data Integrity / Operational Risk
PHASE 4: EXECUTIVE SUMMARY (Word count: ~320)
Executive Summary — Q3 Financial Performance
This quarter delivered strong overall revenue growth, with total Q3 revenue reaching $20.4 million, a 6.8% increase over Q2. Revenue accelerated steadily through the quarter, rising 20.7% from July to September, reflecting positive momentum heading into Q4.
Our Premium customer segment continues to be the primary growth engine. Although Premium customers represent fewer than 15% of our total customer base, they now generate over 41% of all revenue — up from 37% last quarter. This is a healthy indicator of the value of our top-tier relationships, but it also signals a dependency that requires careful management.
At the same time, our Basic segment showed early warning signs. Average spending per Basic customer declined by approximately 5.6% this quarter, while the number of Basic customers held nearly flat. This pattern — stable headcount but falling spend — often precedes customer churn and warrants prompt attention.
Market conditions added an element of uncertainty: volatility was elevated for roughly two weeks in August, during which our maximum single-day risk exposure reached an estimated $338,000. We recommend establishing automatic exposure limits on high-volatility days to keep this figure below $237,000 going forward.
Three areas require board attention: (1) the concentration of revenue among a small number of Premium accounts, (2) declining engagement in the Basic segment, and (3) data pipeline weaknesses that left 1.3% of quarterly revenue unattributable to any customer. All three are addressable with targeted investments.
Forward-looking recommendation: For Q4, we recommend a two-track strategy — a Premium retention program to protect the $5.8M in revenue from our top 10 customers, and a Basic segment re-engagement campaign (targeted offers, usage incentives) to reverse the spending decline before it becomes attrition. Combined, these initiatives could protect or recover an estimated $2.5–3.0M in annualized revenue.
PHASE 5: AUDIT TRAIL — SPOT-CHECK CALCULATIONS
Spot-Check 1: Premium Segment Revenue Share
Spot-Check 2: Basic Segment QoQ Decline
Spot-Check 3: Monthly Revenue Growth July to September
datecolumn parsed to month, amounts summed per monthSpot-Check 4: VaR Calculation
Spot-Check 5: Top-10 Customer Revenue Share
PHASE 6: VISUALIZATION RECOMMENDATIONS
Visualization 1: Revenue by Segment — Stacked Bar Chart
Visualization 2: Basic Segment Spend Intensity — Dual-Axis Line Chart
Visualization 3: Daily Volatility and Exposure — Area + Line Overlay
SUMMARY CHECKLIST — SUCCESS CRITERIA VERIFICATION
Decomposed Rig Program Source
Decomposed Solution
Decomposed solution output
[decomposition did not produce a valid solution]
Benchmark Program (bench.ts)
bench.ts source
Verdict
Winner:
single-call— The single-call solution comprehensively addresses all six success criteria: (1) Data quality issues are thoroughly identified with 150 issues across ...