diff --git a/.agents/skills/analyze-ci-failure/SKILL.md b/.agents/skills/analyze-ci-failure/SKILL.md index 2019c5b5f9..6bda9d5d6b 100644 --- a/.agents/skills/analyze-ci-failure/SKILL.md +++ b/.agents/skills/analyze-ci-failure/SKILL.md @@ -1,16 +1,25 @@ --- name: analyze-ci-failure -description: Download and analyze a CI failure log to construct an actionable suggested fix plan and report back +description: Download and analyze a CI failure log to construct an actionable + suggested fix plan and report back --- -When a CI monitoring workflow alerts you to a failed Buildkite job or GitHub check, invoke this skill by running: +When a CI monitoring workflow alerts you to a failed Buildkite job or GitHub +check, invoke this skill by running: ```bash -./.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py "" "" "" "" +./.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py \ + "" "" "" "" ``` ### ✨ What this Skill Does -1. **Resolves Log**: Automatically resolves the Buildkite job download URL or locates existing local log artifacts. -2. **Downloads & Ingests**: Fetches the full raw CI log file and saves it locally. -3. **Smart Error Extraction**: Scans the log lines for critical failure signatures (`Traceback`, `ERROR:`, `FAILED:`, missing packages, compiler aborts). -4. **Fix Plan Synthesis**: Constructs a beautifully structured Markdown suggested plan on how to resolve the root cause. -5. **Natively Notifies**: Dispatches a high-priority summary notification message back to your active agent conversation via `agentapi send-message`! +1. **Resolves Log**: Automatically resolves the Buildkite job download URL or + fetches GitHub Actions logs via `gh` CLI. +2. **Downloads & Ingests**: Fetches the full raw CI log file and saves it + locally. +3. **Smart Error Extraction**: Scans log lines for critical failure signatures + (`Traceback`, `ERROR:`, `FAILED:`, missing packages, compiler aborts, linter + errors). +4. **Fix Plan Synthesis**: Constructs a beautifully structured Markdown + suggested plan on how to resolve the root cause. +5. **Natively Notifies**: Dispatches a high-priority summary notification + message back to your active agent conversation via `agentapi send-message`. diff --git a/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py b/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py index 3b975acfb3..20212f6d23 100644 --- a/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py +++ b/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py @@ -16,6 +16,25 @@ def fetch_log(build_id, job_id, output_path): else: log_url = f"https://buildkite.com/organizations/bazel/pipelines/rules-python-python/builds/{build_id}/jobs/{job_id}/download.txt" + # Check if this is a GitHub Actions job + gh_match = re.search(r"github\.com/.*/job/(\d+)", log_url) or re.search( + r"^(\d+)$", job_id + ) + if gh_match: + gh_job_id = gh_match.group(1) + print(f"📥 Fetching GitHub Action log for job {gh_job_id} using gh CLI...") + cmd = ["gh", "run", "view", "--job", gh_job_id, "--log"] + try: + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + with open(output_path, "w") as f: + f.write(res.stdout) + return True + except Exception as e: + print( + f"⚠️ Failed to fetch GitHub log via gh CLI for job {gh_job_id}: {e}", + file=sys.stderr, + ) + if not log_url.endswith("/download.txt") and "buildkite.com" in log_url: log_url = re.sub(r"/log$", "/download.txt", log_url) @@ -55,6 +74,10 @@ def parse_log(log_path): "no such package", "no such target", "exit code", + "##[error]", + "Would reformat:", + "would be reformatted", + "error]", ] ): errors.append(line.strip()) diff --git a/.agents/skills/monitor-ci-results/SKILL.md b/.agents/skills/monitor-ci-results/SKILL.md index 2b4bc5a460..1cef8293f4 100644 --- a/.agents/skills/monitor-ci-results/SKILL.md +++ b/.agents/skills/monitor-ci-results/SKILL.md @@ -1,31 +1,26 @@ --- name: monitor-ci-results -description: Monitor CI for PRs and notify of status +description: Monitor remote CI results for a PR and autonomously launch + subagents to analyze CI failures --- When the user requests to monitor remote CI results or watch a pull request, -invoke `scripts/monitor_remote_ci.py `. - -This long-running monitoring service runs in the background and continuously -polls both GitHub PR checks and Buildkite workflow executions. +launch the monitoring script in the background: +```bash +./.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py \ + "" & +``` -### ✨ Autonomous Failure and Blocked Job Orchestration -1. **Blocked Jobs**: When a Buildkite job or GitHub check is in a blocked +### ✨ Autonomous Subagent Orchestration +1. **Background Polling**: `monitor_remote_ci.py` continuously polls both + GitHub PR checks and Buildkite workflow executions in the background. +2. **Blocked Jobs**: When a Buildkite job or GitHub check is in a blocked state waiting for user confirmation, it dispatches a notification via `agentapi send-message` so the user is alerted to confirm running the job. -2. **Failure Analysis**: When any CI job completes with errors or returns a - non-zero exit code: - - It automatically downloads the raw CI log file to `ci_logs/`. - - It launches an independent background analyzer script - (`analyze_ci_failure.py`). - - It authors a structured Markdown plan to fix the failure. - - It natively dispatches a high-priority notification back to your active - agent conversation using `agentapi send-message`! - -### Example Invocation -```bash -./scripts/monitor_remote_ci.py 3812 "0be435bd-96aa-4e1b-9c6f-727b31e80fa0" & -``` -*Note: Always include the trailing `&` when launching the monitoring script via -tool calls to ensure it runs as a detached background task without blocking -foreground execution.* +3. **Failure Reporting**: When any GitHub check or Buildkite job completes + with errors, `monitor_remote_ci.py` dispatches a high-priority notification + message reporting the failed check back to your conversation. +4. **Subagent Analysis**: Upon receiving a failure notification message from + the monitoring script, immediately launch a separate subagent using the + `invoke_subagent` tool with the role "CI Failure Analyzer" to run the + `analyze-ci-failure` skill on the reported failure. diff --git a/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py b/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py index 40be4d5d26..f668ed96f5 100755 --- a/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py +++ b/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py @@ -104,6 +104,7 @@ def main(): name = check.get("name", "unknown") state = check.get("state", "UNKNOWN") link = check.get("link", "") + bucket = check.get("bucket", "") if "buildkite" in name.lower() and link: jobs = get_buildkite_jobs(link) @@ -202,7 +203,7 @@ def main(): f"⚠️ Remote CI Buildkite Job '{jname}' completed with errors!\n\n" f"Build ID: {build_id} | Job ID: {jid}\n" f"Log URL: {job.get('log_url', link)}\n\n" - f"Start subagent: run analyze-ci-failure skill on this failure" + f"Action required: Start a subagent to run the analyze-ci-failure skill for '{jname}' with build ID '{build_id}' and job ID '{jid}'." ) subprocess.run( [ @@ -240,12 +241,16 @@ def main(): with open(state_file, "w") as f: json.dump(monitored, f) - elif state in ["FAILURE", "failed"] and name not in monitored: + elif ( + state.upper() in ["FAILURE", "FAIL", "CANCELLED", "TIMED_OUT"] + or bucket == "fail" + ) and name not in monitored: print(f"🚨 Notifying failure for GitHub check '{name}'...") + job_id = link.split("/")[-1] if "/" in link else link msg = ( f"⚠️ Remote CI GitHub Check '{name}' completed with errors!\n\n" f"Link: {link}\n\n" - f"Start subagent: run analyze-ci-failure skill on this failure" + f"Action required: Start a subagent to run the analyze-ci-failure skill for '{name}' with link '{link}' and job ID '{job_id}'." ) subprocess.run( [