Monitor #84
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Monitor | |
| # Runtime anomaly watch, per bot, baseline-relative. See | |
| # .github/AGENT-FLOWS-REDESIGN.md §6.1 and §8 for the design this | |
| # implements. | |
| # | |
| # Runs every 4 hours for every bot whose `roles` (.github/bots.yml) include | |
| # `monitor` — currently vahter, coupon, alita — EXCEPT alita, which runs | |
| # DAILY only (traffic_class: dormant — see the "Alita daily-cadence gate" | |
| # step below): the cron fires every 4h at :20, and alita is only actually | |
| # invoked when the run lands in the first slot of the UTC day (hour < 4) or | |
| # on a manual workflow_dispatch. | |
| # | |
| # Three-job graph: | |
| # gather — VPN-bound (aks-vpn concurrency group, single-occupancy): | |
| # evidence gathering, the per-bot guard, baseline append/push, | |
| # fingerprint fetch, and the mechanical P1 detection. Uploads | |
| # /tmp/evidence as a short-lived artifact (7-day retention — it | |
| # contains user chat text) and emits `clean_bots`, a JSON array | |
| # of bots that passed the guard (suitable for `fromJson`). | |
| # agent — one job PER BOT via `strategy.matrix: bot: fromJson(clean_bots)`, | |
| # `fail-fast: false`, NOT in the aks-vpn group (the agent | |
| # invocation needs no VPN — only gather does). Each matrix leg | |
| # is its own runner, so each gets exactly ONE | |
| # `openai/codex-action@v1` invocation: `safety-strategy: | |
| # drop-sudo` is a one-way, once-per-runner operation, and a | |
| # SECOND invocation on the same runner dies at | |
| # `ensurePasswordlessSudo` (see monitor run 30229614320, step | |
| # 15 — before this split, only the first bot in the old single | |
| # job ever got analysed). Skipped entirely (guarded by `if:`) | |
| # when `clean_bots` is `[]`, since GitHub errors on an empty | |
| # matrix. | |
| # finalize — `if: always()`; fails the run if the gather job reports any | |
| # skipped bot OR if any `agent` matrix leg failed, but not | |
| # merely because `agent` itself was skipped (empty matrix). | |
| # | |
| # Adding a THIRD monitor bot is now purely a `.github/bots.yml` entry with | |
| # `monitor` in its `roles` — no workflow edit needed. | |
| on: | |
| schedule: | |
| - cron: '20 */4 * * *' | |
| workflow_dispatch: | |
| permissions: | |
| contents: write # baseline.sh commit pushes agent-state; escalate jobs call _sre-agent.yml (contents: write) — a reusable workflow cannot escalate beyond its caller, so this must be granted here too (see the jobless startup_failure note in AGENT-FLOWS-REDESIGN.md/CLAUDE memory). | |
| issues: write | |
| pull-requests: write | |
| jobs: | |
| gather: | |
| runs-on: ubuntu-latest | |
| # VPN peer is single-occupancy; concurrent connections break DNS | |
| # (2026-07-23 incident, see product.yml). Only THIS job touches the VPN. | |
| concurrency: | |
| group: aks-vpn | |
| cancel-in-progress: false | |
| outputs: | |
| clean_bots: ${{ steps.guard.outputs.clean_bots }} | |
| skipped_bots: ${{ steps.guard.outputs.skipped_bots }} | |
| p1_vahter: ${{ steps.detect-p1.outputs.p1_vahter }} | |
| p1_coupon: ${{ steps.detect-p1.outputs.p1_coupon }} | |
| p1_alita: ${{ steps.detect-p1.outputs.p1_alita }} | |
| image_vahter: ${{ steps.detect-p1.outputs.image_vahter }} | |
| image_coupon: ${{ steps.detect-p1.outputs.image_coupon }} | |
| image_alita: ${{ steps.detect-p1.outputs.image_alita }} | |
| issue_vahter: ${{ steps.detect-p1.outputs.issue_vahter }} | |
| issue_coupon: ${{ steps.detect-p1.outputs.issue_coupon }} | |
| issue_alita: ${{ steps.detect-p1.outputs.issue_alita }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # change_context needs `git log` over the preceding 72h across the | |
| # whole repo, not just the tip commit. | |
| fetch-depth: 0 | |
| - name: Ensure labels exist (bot:<name>, monitor, anomaly, priority-*) | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| source scripts/gather/lib.sh | |
| for bot in $(bots_yml_json | jq -r '.bots | keys[]'); do | |
| gh label create "bot:${bot}" --repo "${{ github.repository }}" --color "1d76db" \ | |
| --description "Findings for the ${bot} bot" --force | |
| done | |
| gh label create "monitor" --repo "${{ github.repository }}" --color "5319e7" \ | |
| --description "Runtime anomaly finding from the monitor agent" --force | |
| gh label create "anomaly" --repo "${{ github.repository }}" --color "d93f0b" \ | |
| --description "Baseline-relative runtime anomaly" --force | |
| gh label create "priority-high" --repo "${{ github.repository }}" --color "b60205" --force | |
| gh label create "priority-medium" --repo "${{ github.repository }}" --color "fbca04" --force | |
| gh label create "priority-low" --repo "${{ github.repository }}" --color "0e8a16" --force | |
| - name: Alita daily-cadence gate | |
| id: alita-gate | |
| run: | | |
| set -euo pipefail | |
| if [ "${{ github.event_name }}" != "schedule" ]; then | |
| echo "alita_ok=true" >> "$GITHUB_OUTPUT" | |
| echo "manual dispatch — running alita this invocation regardless of time of day" | |
| else | |
| HOUR=$(date -u +%H) | |
| # cron fires at :20 past every 4th hour starting at 0 (0,4,8,...) — | |
| # "first slot of the UTC day" is HOUR 0..3. | |
| if [ "$HOUR" -lt 4 ]; then | |
| echo "alita_ok=true" >> "$GITHUB_OUTPUT" | |
| echo "scheduled run at hour=${HOUR} UTC — first slot of the day, running alita (dormant traffic_class: daily cadence)" | |
| else | |
| echo "alita_ok=false" >> "$GITHUB_OUTPUT" | |
| echo "scheduled run at hour=${HOUR} UTC — not alita's daily slot, skipping alita this cycle" | |
| fi | |
| fi | |
| - name: Fetch or bootstrap agent-state branch | |
| run: | | |
| set -euo pipefail | |
| if git ls-remote --exit-code --heads origin agent-state >/dev/null 2>&1; then | |
| git fetch origin agent-state | |
| git worktree add /tmp/agent-state origin/agent-state | |
| echo "checked out existing agent-state branch" | |
| else | |
| echo "::warning::agent-state branch does not exist yet on origin — bootstrapping an empty orphan branch (first-ever monitor run). See scripts/gather/baseline.sh header." | |
| git worktree add --orphan -b agent-state /tmp/agent-state | |
| fi | |
| - name: Setup WireGuard VPN | |
| run: | | |
| chmod +x scripts/setup-vpn.sh | |
| ./scripts/setup-vpn.sh | |
| env: | |
| WIREGUARD_CONFIG: ${{ secrets.WIREGUARD_CONFIG }} | |
| SPLIT_TUNNEL: "false" # DB_PROD_HOST is reachable only via the VPN peer's full-tunnel route, same as product.yml. | |
| - name: Gather runtime + baseline evidence (bash loop, not matrix — single VPN session) | |
| id: gather | |
| env: | |
| ALITA_OK: ${{ steps.alita-gate.outputs.alita_ok }} | |
| PROMETHEUS_URL: http://prometheus.internal:9090 | |
| LOKI_URL: http://loki.internal | |
| ARGOCD_URL: http://argo.internal | |
| ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }} | |
| DB_PROD_HOST: ${{ secrets.DB_PROD_HOST }} | |
| DB_PROD_USERNAME: ${{ secrets.DB_PROD_USERNAME }} | |
| DB_PROD_PASSWORD: ${{ secrets.DB_PROD_PASSWORD }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| source scripts/gather/lib.sh | |
| chmod +x scripts/gather/runtime.sh scripts/gather/baseline.sh | |
| mkdir -p /tmp/evidence | |
| BOTS="$(all_bots_with_role monitor)" | |
| # Alita is daily-cadence (traffic_class: dormant) — outside its one | |
| # daily slot, skip it entirely here (not just its agent invocation) | |
| # so we don't burn 6x/day Loki/Postgres queries on it or append 6 | |
| # near-duplicate rollups/day into its agent-state history. | |
| if [ "${ALITA_OK:-true}" != "true" ]; then | |
| BOTS="$(echo "$BOTS" | grep -v '^alita$' || true)" | |
| echo "alita not in its daily slot this cycle — excluded from gather entirely" | |
| fi | |
| { | |
| echo "bots<<BOTS_EOF" | |
| echo "$BOTS" | |
| echo "BOTS_EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| START_72H=$(date -u -d '72 hours ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-72H +%Y-%m-%dT%H:%M:%SZ) | |
| for bot in $BOTS; do | |
| echo "::group::gather runtime evidence for ${bot}" | |
| if ./scripts/gather/runtime.sh "$bot" > "/tmp/evidence/${bot}.runtime.md"; then | |
| echo "runtime.sh OK for ${bot}" | |
| else | |
| echo "WARNING: runtime.sh failed for ${bot} — partial output (if any) is in /tmp/evidence/${bot}.runtime.md; continuing to gather remaining bots" | |
| fi | |
| if ./scripts/gather/baseline.sh gather "$bot" > "/tmp/evidence/${bot}.gather.json"; then | |
| echo "baseline.sh gather OK for ${bot}" | |
| else | |
| echo "WARNING: baseline.sh gather failed for ${bot} — manifest (if any) is in /tmp/evidence/${bot}.gather.json" | |
| fi | |
| # change_context: source-dir merges to main (already have full | |
| # history via fetch-depth 0, REQUIRED — git log alone already | |
| # answers "did something ship recently?") + ArgoCD deploy history, | |
| # preceding 72h. | |
| # | |
| # ArgoCD deploy history is OPTIONAL evidence — enrichment for | |
| # change correlation only, not core evidence (pod health/sync is | |
| # core evidence and is already probed as the REQUIRED "argocd" | |
| # source in runtime.sh, above). A failure here must degrade this | |
| # bot's change_context, NOT skip the bot — see probe_optional() in | |
| # lib.sh and monitor.md. (Fixed 2026-07-27: run 30233211533 hit a | |
| # wrong /history endpoint, 404'd for all three bots, and — because | |
| # it was wired as fail-loud/REQUIRED — blanked clean_bots to `[]` | |
| # and skipped the entire monitor run.) | |
| # | |
| # There is no separate ArgoCD "/history" endpoint. Deploy history | |
| # lives at `.status.history[]` INSIDE the application object — the | |
| # SAME object scripts/verify-deploy.sh's argocd_fetch() already | |
| # reads `.status.sync`/`.status.health`/`.status.summary.images` | |
| # from at `GET /api/v1/applications/<app>`. Each history entry's | |
| # `id`/`revision`/`deployedAt` fields are extracted explicitly | |
| # (verified against .github/prompts/sre.md's "Get deployment | |
| # history" jq, which reads the same fields plus `initiatedBy`); | |
| # the full entry is also included as raw JSON so nothing is lost | |
| # if it carries other fields (e.g. `source`) we didn't name here. | |
| # | |
| # ONE fetch only (probe_optional_body, not probe_optional + | |
| # a second curl for the real body): a second, unguarded curl here | |
| # under this step's `set -e`/`pipefail` would abort the ENTIRE | |
| # gather loop — including every bot processed after this one — on | |
| # a transient failure even though the probe itself just succeeded. | |
| # That silently starves later bots' change_context_manifest.json, | |
| # which is exactly the "emitted no valid manifest" guard fallback | |
| # (fixed 2026-07-27 — see probe_optional_body() in lib.sh). | |
| SOURCE_DIR="$(bot_field "$bot" '.source_dir')" | |
| APP_NAME="$(bot_field "$bot" '.argocd_app')" | |
| APP_PATH="/api/v1/applications/${APP_NAME}" | |
| APP_JSON_FILE="/tmp/evidence/${bot}.argocd_app.json" | |
| ARGOCD_HISTORY_STATUS=$(probe_optional_body "argocd_history" "$APP_JSON_FILE" curl -sSf --connect-timeout 5 --max-time 20 \ | |
| "${ARGOCD_URL}${APP_PATH}" -H "Authorization: Bearer ${ARGOCD_AUTH_TOKEN}") || true | |
| jq -cn --arg s "$ARGOCD_HISTORY_STATUS" '{argocd_history: $s}' > "/tmp/evidence/${bot}.change_context_manifest.json" | |
| { | |
| echo "### Merges to main touching ${SOURCE_DIR} (preceding 72h)" | |
| git log --since="72 hours ago" --oneline -- "$SOURCE_DIR" | sed 's/^/- /' || true | |
| echo | |
| echo "### ArgoCD deploy history (preceding 72h, OPTIONAL source — see monitor.md)" | |
| if [ "$ARGOCD_HISTORY_STATUS" = "ok" ]; then | |
| jq -r --arg since "$START_72H" ' | |
| [.status.history[]? | select(.deployedAt >= $since)] | |
| | if length == 0 then "(none in the last 72h)" | |
| else .[] | "- id=\(.id) revision=\(.revision) deployedAt=\(.deployedAt) raw=\(. | tojson)" | |
| end | |
| ' "$APP_JSON_FILE" | |
| else | |
| echo "UNKNOWN: ArgoCD deploy history was unavailable this run (${ARGOCD_HISTORY_STATUS})." | |
| echo "This is NOT evidence that no deploy occurred — it only means we could not check. Do not treat the absence of entries above as \"no recent deploy\"; rely on the git-log section above for change correlation and say explicitly in your summary that deploy history was unavailable." | |
| fi | |
| } > "/tmp/evidence/${bot}.change_context.md" | |
| echo "::endgroup::" | |
| done | |
| # Full-tunnel VPN routes ALL traffic (including api.github.com and the | |
| # Azure OpenAI endpoint) through the VPN peer, which times out on | |
| # non-AKS-internal public traffic — disconnect as soon as the | |
| # DB/Prometheus/Loki/ArgoCD-dependent gather step is done (same pattern | |
| # as product.yml). | |
| - name: Disconnect VPN (post gather) | |
| run: sudo wg-quick down wg0 || true | |
| - name: Guard — evaluate each bot's evidence independently | |
| id: guard | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GATHER_BOTS: ${{ steps.gather.outputs.bots }} | |
| run: | | |
| set -euo pipefail | |
| chmod +x scripts/report-degraded.sh | |
| SKIPPED="" | |
| declare -a CLEAN_BOTS=() | |
| for bot in $GATHER_BOTS; do | |
| # RUNTIME_MANIFEST and GATHER_JSON carry only REQUIRED sources | |
| # (prometheus/loki/argocd app-status/postgres) and must be valid | |
| # JSON objects. An unreadable one counts as DEGRADED, never as | |
| # clean — the fail-safe direction. (2026-07-27: emit_manifest | |
| # pretty-printed its output, so `head -n1` returned a bare "{"; | |
| # here that crashed the guard at exit 2, and in product.yml the | |
| # same malformed input was swallowed and read as healthy.) | |
| # | |
| # CC_MANIFEST carries only the argocd_history probe, which is | |
| # OPTIONAL evidence (see probe_optional() in lib.sh and the gather | |
| # step above) — an unreadable/missing CC_MANIFEST must NOT force | |
| # this bot degraded, only give it a "degraded(optional)" status so | |
| # it is visible but ignored by the BAD computation below. | |
| RUNTIME_MANIFEST=$(head -n1 "/tmp/evidence/${bot}.runtime.md" 2>/dev/null || echo '') | |
| GATHER_JSON=$(cat "/tmp/evidence/${bot}.gather.json" 2>/dev/null || echo '') | |
| CC_MANIFEST=$(cat "/tmp/evidence/${bot}.change_context_manifest.json" 2>/dev/null || echo '') | |
| MANIFEST_BAD="" | |
| if ! echo "$RUNTIME_MANIFEST" | jq -e 'type == "object"' >/dev/null 2>&1; then | |
| RUNTIME_MANIFEST='{"sources":{"runtime_manifest":"unreadable — runtime.sh emitted no valid manifest line"}}' | |
| MANIFEST_BAD="yes" | |
| fi | |
| if ! echo "$GATHER_JSON" | jq -e 'type == "object"' >/dev/null 2>&1; then | |
| GATHER_JSON='{"sources":{"baseline_manifest":"unreadable — baseline.sh gather emitted no valid manifest"}}' | |
| MANIFEST_BAD="yes" | |
| fi | |
| if ! echo "$CC_MANIFEST" | jq -e 'type == "object"' >/dev/null 2>&1; then | |
| CC_MANIFEST='{"argocd_history":"degraded(optional): change_context ArgoCD-history probe emitted no valid manifest"}' | |
| fi | |
| # Merge ALL THREE manifests (runtime.sh's prometheus/loki/argocd | |
| # probe + baseline.sh gather's prometheus/loki/argocd/postgres probe | |
| # + the change_context step's OPTIONAL argocd_history probe) into | |
| # ONE combined manifest — report-degraded.sh reads `.sources` from | |
| # whatever file it's given, so a postgres-only or argocd-history-only | |
| # failure (each probed by only one of the three) must still show up | |
| # there, not just in a manifest that would misleadingly read all-"ok". | |
| COMBINED_MANIFEST=$(jq -n --arg bot "$bot" \ | |
| --argjson rt "$(echo "$RUNTIME_MANIFEST" | jq '.sources? // {}')" \ | |
| --argjson bl "$(echo "$GATHER_JSON" | jq '.sources? // {}')" \ | |
| --argjson cc "$(echo "$CC_MANIFEST" | jq '. // {}')" \ | |
| '{bot: $bot, sources: ($rt + $bl + $cc)}') | |
| echo "$COMBINED_MANIFEST" > "/tmp/evidence/${bot}.combined-manifest.json" | |
| # REQUIRED-vs-OPTIONAL is purely data-driven off the status STRING, | |
| # not the source key: any status other than "ok" that does NOT start | |
| # with "degraded(optional):" is a required-source failure and marks | |
| # the bot BAD (skipped). A "degraded(optional): ..." status (from | |
| # probe_optional(), currently only argocd_history) is deliberately | |
| # excluded here — it is enrichment evidence, so its failure degrades | |
| # this bot's change_context (visible in combined-manifest.json and | |
| # change_context.md) but must never skip the bot. | |
| BAD=$(echo "$COMBINED_MANIFEST" | jq -r ' | |
| .sources | |
| | to_entries[] | |
| | select(.value != "ok") | |
| | select((.value | startswith("degraded(optional):")) | not) | |
| | .key | |
| ' 2>/dev/null || true) | |
| # Belt and braces: if a REQUIRED manifest was unreadable, force this | |
| # bot degraded even if the merged .sources somehow came out empty. | |
| if [ -n "$MANIFEST_BAD" ] && [ -z "$BAD" ]; then | |
| BAD="manifest" | |
| fi | |
| if [ -n "$BAD" ]; then | |
| echo "::error::unreachable source(s) for bot=${bot}: ${BAD} — SKIPPING only ${bot}'s agent invocation this run." | |
| ./scripts/report-degraded.sh "/tmp/evidence/${bot}.combined-manifest.json" | |
| echo "run_${bot}=false" >> "$GITHUB_OUTPUT" | |
| SKIPPED="${SKIPPED}${SKIPPED:+, }${bot} (${BAD})" | |
| else | |
| OPTIONAL_DEGRADED=$(echo "$COMBINED_MANIFEST" | jq -r ' | |
| .sources | |
| | to_entries[] | |
| | select(.value | startswith("degraded(optional):")) | |
| | "\(.key): \(.value)" | |
| ' 2>/dev/null || true) | |
| if [ -n "$OPTIONAL_DEGRADED" ]; then | |
| echo "::warning::bot=${bot}: evidence clean (required sources ok) but OPTIONAL source(s) degraded — proceeding anyway: ${OPTIONAL_DEGRADED}" | |
| else | |
| echo "bot=${bot}: evidence clean — proceeding." | |
| fi | |
| echo "run_${bot}=true" >> "$GITHUB_OUTPUT" | |
| CLEAN_BOTS+=("$bot") | |
| fi | |
| done | |
| echo "skipped_bots=${SKIPPED}" >> "$GITHUB_OUTPUT" | |
| # JSON array of bots that passed the guard, suitable for the `agent` | |
| # job's `fromJson(...)` matrix. jq's --args/$ARGS.positional handles | |
| # the zero-bots case correctly (emits `[]`), unlike naive string | |
| # concatenation. | |
| CLEAN_JSON=$(jq -cn --args '$ARGS.positional' "${CLEAN_BOTS[@]}") | |
| echo "clean_bots=${CLEAN_JSON}" >> "$GITHUB_OUTPUT" | |
| if [ -n "$SKIPPED" ]; then | |
| echo "::warning::Evidence pipeline degraded for: ${SKIPPED} — see the evidence-pipeline-degraded issue." | |
| else | |
| echo "All gathered bots reported a clean manifest." | |
| fi | |
| - name: Append this run + compute baseline stats + fetch fingerprints (bots that passed the guard only) | |
| id: baseline | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| RUN_VAHTER: ${{ steps.guard.outputs.run_vahter }} | |
| RUN_COUPON: ${{ steps.guard.outputs.run_coupon }} | |
| RUN_ALITA: ${{ steps.guard.outputs.run_alita }} | |
| run: | | |
| set -euo pipefail | |
| chmod +x scripts/gather/fingerprints.sh | |
| declare -A RUN_OK=( [vahter]="$RUN_VAHTER" [coupon]="$RUN_COUPON" [alita]="$RUN_ALITA" ) | |
| for bot in vahter coupon alita; do | |
| [ "${RUN_OK[$bot]:-false}" = "true" ] || continue | |
| ./scripts/gather/baseline.sh append "$bot" /tmp/agent-state "/tmp/evidence/${bot}.gather.json" | |
| ./scripts/gather/baseline.sh stats "$bot" /tmp/agent-state > "/tmp/evidence/${bot}.stats.json" | |
| ./scripts/gather/fingerprints.sh "$bot" /tmp/agent-state > "/tmp/evidence/${bot}.fingerprints.json" | |
| done | |
| - name: Detect mechanical P1 (bash, NOT the agent's judgment — see monitor.md "P1" section) | |
| id: detect-p1 | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| RUN_VAHTER: ${{ steps.guard.outputs.run_vahter }} | |
| RUN_COUPON: ${{ steps.guard.outputs.run_coupon }} | |
| RUN_ALITA: ${{ steps.guard.outputs.run_alita }} | |
| run: | | |
| set -euo pipefail | |
| declare -A RUN_OK=( [vahter]="$RUN_VAHTER" [coupon]="$RUN_COUPON" [alita]="$RUN_ALITA" ) | |
| for bot in vahter coupon alita; do | |
| if [ "${RUN_OK[$bot]:-false}" != "true" ]; then | |
| echo "p1_${bot}=false" >> "$GITHUB_OUTPUT" | |
| echo "image_${bot}=" >> "$GITHUB_OUTPUT" | |
| continue | |
| fi | |
| GATHER_JSON=$(cat "/tmp/evidence/${bot}.gather.json") | |
| STATS_JSON=$(cat "/tmp/evidence/${bot}.stats.json" 2>/dev/null || echo '{}') | |
| READY=$(echo "$GATHER_JSON" | jq -r '.pods.ready_replicas // 0') | |
| DESIRED=$(echo "$GATHER_JSON" | jq -r '.pods.desired_replicas // 0') | |
| HEALTH=$(echo "$GATHER_JSON" | jq -r '.pods.argocd_health // "Unknown"') | |
| # Deliberately conservative and NARROW — see monitor.md "P1" section | |
| # and the Phase 2 report: this gate exists purely so a monitor P1 | |
| # never depends on the LLM's judgment (it spends real budget and | |
| # has write access via the SRE agent). Two direct conditions only: | |
| POD_DOWN="false" | |
| if [ "$DESIRED" -gt 0 ] 2>/dev/null && [ "$READY" -eq 0 ] 2>/dev/null; then | |
| POD_DOWN="true" | |
| fi | |
| # Error-burst P1: requires an extreme one-shot deviation on ALL | |
| # THREE of ratio, z-score, AND an absolute floor — ordinary | |
| # baseline-relative findings (rule 2/3 in monitor.md, ratio | |
| # >=2.5 or z>=3) are the AGENT's job, filed as priority-high but | |
| # NOT auto-escalated to the SRE agent. Only a genuinely extreme | |
| # burst crosses this bar. | |
| ERR_BURST="false" | |
| RATIO=$(echo "$STATS_JSON" | jq -r '.series.log_errors_24h.ratio_vs_28d // "null"') | |
| ZSCORE=$(echo "$STATS_JSON" | jq -r '.series.log_errors_24h.z_score_28d // "null"') | |
| CURRENT=$(echo "$STATS_JSON" | jq -r '.series.log_errors_24h.current // 0') | |
| if [ "$RATIO" != "null" ] && [ "$ZSCORE" != "null" ]; then | |
| if awk -v r="$RATIO" -v z="$ZSCORE" -v c="$CURRENT" 'BEGIN{exit !(r>=5 && z>=5 && c>=20)}'; then | |
| ERR_BURST="true" | |
| fi | |
| fi | |
| P1="false" | |
| REASON="" | |
| if [ "$POD_DOWN" = "true" ]; then | |
| P1="true"; REASON="no healthy replicas (ready=${READY} desired=${DESIRED} health=${HEALTH})" | |
| elif [ "$ERR_BURST" = "true" ]; then | |
| P1="true"; REASON="error burst far outside baseline (current=${CURRENT} ratio_vs_28d=${RATIO} z_score_28d=${ZSCORE})" | |
| fi | |
| echo "p1_${bot}=${P1}" >> "$GITHUB_OUTPUT" | |
| echo "::notice::bot=${bot} p1=${P1} reason=${REASON:-n/a}" | |
| IMAGE=$(echo "$GATHER_JSON" | jq -r '.pods.deployed_image // empty') | |
| echo "image_${bot}=${IMAGE}" >> "$GITHUB_OUTPUT" | |
| # This value is also consumed by the `agent` job (via the evidence | |
| # artifact) since a dynamic matrix can't address a job output by a | |
| # runtime-computed key derived from matrix.bot. | |
| echo "$P1" > "/tmp/evidence/${bot}.p1.txt" | |
| # Best-effort: most recent open priority-high monitor issue for this | |
| # bot, so the SRE agent has something to comment on if the monitor | |
| # agent already filed the P1 finding this run. Optional — omitted | |
| # (empty) is a supported input on _sre-agent.yml. | |
| ISSUE=$(gh issue list --repo "${{ github.repository }}" --label "monitor" --label "bot:${bot}" --label "priority-high" \ | |
| --state open -L 5 --json number,createdAt --jq 'sort_by(.createdAt) | reverse | .[0].number // empty' 2>/dev/null || true) | |
| echo "issue_${bot}=${ISSUE}" >> "$GITHUB_OUTPUT" | |
| done | |
| # ─── commit agent-state (append-only, always attempted) ───────────── | |
| - name: Commit + push agent-state | |
| if: always() | |
| run: | | |
| chmod +x scripts/gather/baseline.sh | |
| ./scripts/gather/baseline.sh commit /tmp/agent-state \ | |
| "monitor: append run $(date -u +%Y-%m-%dT%H:%M:%SZ) (run ${{ github.run_id }})" | |
| - name: Upload evidence artifact for the agent job | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: monitor-evidence | |
| path: /tmp/evidence | |
| # Contains user chat text (Loki log excerpts, error groups) — do not | |
| # keep it around beyond what the agent job needs it for. | |
| retention-days: 7 | |
| # ─── agent — one job per bot that passed the guard, so each runner gets | |
| # exactly ONE openai/codex-action@v1 invocation (see the header comment's | |
| # "agent" bullet for the drop-sudo bug this fixes). ────────────────────── | |
| agent: | |
| needs: gather | |
| # GitHub errors outright on an empty strategy.matrix — this guard is | |
| # mandatory, not optional, whenever clean_bots could legitimately be `[]` | |
| # (e.g. every bot degraded, or alita outside its daily slot with the | |
| # other two also degraded). | |
| if: needs.gather.outputs.clean_bots != '[]' && needs.gather.outputs.clean_bots != '' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| bot: ${{ fromJson(needs.gather.outputs.clean_bots) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download evidence artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: monitor-evidence | |
| path: /tmp/evidence | |
| - name: Load agent prompt and evidence | |
| id: load | |
| run: | | |
| set -euo pipefail | |
| source scripts/gather/lib.sh | |
| BOT="${{ matrix.bot }}" | |
| DISPLAY_NAME="$(bot_field "$BOT" '.display_name')" | |
| TRAFFIC_CLASS="$(bot_field "$BOT" '.traffic_class')" | |
| CADENCE_NOTE="the scheduled runtime monitor run (every 4 hours)" | |
| DORMANT_NOTE="" | |
| if [ "$TRAFFIC_CLASS" = "dormant" ]; then | |
| CADENCE_NOTE="the scheduled runtime monitor run (DAILY cadence for this bot — traffic_class: dormant)" | |
| # Scoped deliberately: the carve-out applies ONLY to traffic/volume | |
| # series (those with dormant_exempt: true). Error series | |
| # (log_errors_24h, log_warnings_24h) are NEVER exempt. The previous | |
| # wording said "disables volume-based rules entirely for this bot", | |
| # and on 2026-07-27 the agent read that and suppressed | |
| # log_errors_24h at z_score 33.59 on a bot that was completely dead. | |
| DORMANT_NOTE=" Remember: the dormant carve-out (monitor.md) applies ONLY to series with dormant_exempt: true (traffic/volume). Error series are never exempt — evaluate them normally." | |
| fi | |
| { | |
| echo "display_name=${DISPLAY_NAME}" | |
| echo "cadence_note=${CADENCE_NOTE}" | |
| echo "dormant_note=${DORMANT_NOTE}" | |
| } >> "$GITHUB_OUTPUT" | |
| P1_FLAG=$(cat "/tmp/evidence/${BOT}.p1.txt" 2>/dev/null || echo "false") | |
| { | |
| PROMPT_DELIM="PROMPT_EOF_$(openssl rand -hex 16)" | |
| echo "AGENT_PROMPT<<${PROMPT_DELIM}" | |
| cat .github/prompts/monitor.md | |
| echo "${PROMPT_DELIM}" | |
| } >> "$GITHUB_ENV" | |
| { | |
| EV_DELIM="EV_EOF_$(openssl rand -hex 16)" | |
| echo "RUNTIME_EVIDENCE<<${EV_DELIM}" | |
| cat "/tmp/evidence/${BOT}.runtime.md" | |
| echo | |
| echo "## Baseline comparison (series)" | |
| echo '```json' | |
| cat "/tmp/evidence/${BOT}.stats.json" | |
| echo '```' | |
| echo | |
| echo "## Change context (preceding 72h)" | |
| cat "/tmp/evidence/${BOT}.change_context.md" | |
| echo | |
| echo "## Known fingerprints + suppressions" | |
| echo '```json' | |
| cat "/tmp/evidence/${BOT}.fingerprints.json" | |
| echo '```' | |
| echo | |
| echo "## Mechanical P1 flag (computed by the workflow, NOT your judgment — see monitor.md)" | |
| echo "p1=${P1_FLAG}" | |
| echo "${EV_DELIM}" | |
| } >> "$GITHUB_ENV" | |
| - name: "Run monitor agent" | |
| uses: openai/codex-action@v1 | |
| with: | |
| openai-api-key: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| responses-api-endpoint: ${{ secrets.AZURE_OPENAI_BASE_URL }} | |
| model: gpt-5-mini | |
| sandbox: workspace-write | |
| safety-strategy: drop-sudo | |
| effort: high | |
| codex-args: '--config sandbox_workspace_write.network_access=true' | |
| prompt: | | |
| <instructions> | |
| ${{ env.AGENT_PROMPT }} | |
| </instructions> | |
| <bot registry-key="${{ matrix.bot }}">${{ steps.load.outputs.display_name }}</bot> | |
| <runtime-evidence> | |
| ${{ env.RUNTIME_EVIDENCE }} | |
| </runtime-evidence> | |
| You have been triggered for ${{ steps.load.outputs.cadence_note }}, for ${{ steps.load.outputs.display_name }} only.${{ steps.load.outputs.dormant_note }} | |
| Every issue you create or comment on must carry the label `bot:${{ matrix.bot }}` in addition to `monitor`, `anomaly`, and a priority label. | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # ─── finalize — surfaces both failure modes (degraded evidence pipeline, | |
| # and any bot's agent invocation failing) without depending on `agent` | |
| # having actually run (an empty matrix is a legitimate, non-failing | |
| # skip). ────────────────────────────────────────────────────────────── | |
| finalize: | |
| needs: [gather, agent] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Fail the run if any bot's evidence was degraded or any bot's agent failed | |
| env: | |
| SKIPPED_BOTS: ${{ needs.gather.outputs.skipped_bots }} | |
| AGENT_RESULT: ${{ needs.agent.result }} | |
| run: | | |
| set -euo pipefail | |
| FAILED="false" | |
| if [ -n "$SKIPPED_BOTS" ]; then | |
| echo "::error::Evidence pipeline degraded for: ${SKIPPED_BOTS} — see the evidence-pipeline-degraded issue. Any other bot's agent still ran normally." | |
| FAILED="true" | |
| fi | |
| # AGENT_RESULT is "skipped" (not "failure") when clean_bots was `[]` — | |
| # that must NOT fail this job, only an actual matrix-leg failure should. | |
| if [ "$AGENT_RESULT" = "failure" ]; then | |
| echo "::error::One or more bot agent invocations failed this run (see the 'agent' job's matrix legs)." | |
| FAILED="true" | |
| fi | |
| if [ "$FAILED" = "true" ]; then | |
| exit 1 | |
| fi | |
| echo "No bots skipped and no agent failures this run — evidence pipeline + agent invocations healthy." | |
| # ─── P1 escalation — one job per bot, gated on the MECHANICAL flag from | |
| # the gather job (never the agent's own judgment, see monitor.md "P1" | |
| # section and the Phase 2 report). commit/argocd-app-name/container-name | |
| # are hardcoded per bot, matching the existing vahter-deploy.yml / | |
| # coupon-deploy.yml / alita-deploy.yml precedent of passing bot identity | |
| # as literal `with:` values into a reusable workflow. These do NOT depend | |
| # on the `agent` job — P1 detection is purely mechanical and independent | |
| # of whether/how the LLM agent ran. ───────────────────────────────────── | |
| escalate-vahter: | |
| needs: gather | |
| if: needs.gather.outputs.p1_vahter == 'true' | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| issues: write | |
| uses: ./.github/workflows/_sre-agent.yml | |
| with: | |
| bot: VahterBanBot | |
| argocd-app-name: vahter-bot | |
| container-name: vahter-bot | |
| docker-image: ${{ needs.gather.outputs.image_vahter != '' && needs.gather.outputs.image_vahter || 'ghcr.io/szer/vahter-bot' }} | |
| commit: ${{ github.sha }} | |
| run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| issue-number: ${{ needs.gather.outputs.issue_vahter }} | |
| failure-class: app | |
| trigger: monitor-p1 | |
| secrets: inherit | |
| escalate-coupon: | |
| needs: gather | |
| if: needs.gather.outputs.p1_coupon == 'true' | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| issues: write | |
| uses: ./.github/workflows/_sre-agent.yml | |
| with: | |
| bot: CouponHubBot | |
| argocd-app-name: coupon-bot | |
| container-name: coupon-bot | |
| docker-image: ${{ needs.gather.outputs.image_coupon != '' && needs.gather.outputs.image_coupon || 'ghcr.io/szer/coupon-bot' }} | |
| commit: ${{ github.sha }} | |
| run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| issue-number: ${{ needs.gather.outputs.issue_coupon }} | |
| failure-class: app | |
| trigger: monitor-p1 | |
| secrets: inherit | |
| escalate-alita: | |
| needs: gather | |
| if: needs.gather.outputs.p1_alita == 'true' | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| issues: write | |
| uses: ./.github/workflows/_sre-agent.yml | |
| with: | |
| bot: AlitaBot | |
| argocd-app-name: alita-bot | |
| container-name: alita-bot | |
| docker-image: ${{ needs.gather.outputs.image_alita != '' && needs.gather.outputs.image_alita || 'ghcr.io/szer/alita-bot' }} | |
| commit: ${{ github.sha }} | |
| run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| issue-number: ${{ needs.gather.outputs.issue_alita }} | |
| failure-class: app | |
| trigger: monitor-p1 | |
| secrets: inherit |