diff --git a/.github/maintainers.json b/.github/maintainers.json new file mode 100644 index 000000000..a17866f63 --- /dev/null +++ b/.github/maintainers.json @@ -0,0 +1,31 @@ +{ + "repo": "TanStack/ai", + "maintainers": [ + { + "github": "tombeckenham", + "discord": null, + "maxOpenAssignments": 10 + }, + { + "github": "AlemTuzlak", + "discord": null, + "maxOpenAssignments": 15 + }, + { + "github": "jherr", + "discord": null, + "maxOpenAssignments": 8 + } + ], + "sla": { + "firstResponseHours": 24, + "followUpResponseHours": 48, + "staleAuthorDays": 14 + }, + "spam": { + "maxAccountAgeDays": 30, + "maxChangedLines": 30 + }, + "botAllowlist": ["renovate", "dependabot", "github-actions"], + "maxCommentsPerRun": 25 +} diff --git a/.github/workflows/maintainer-scorecard.yml b/.github/workflows/maintainer-scorecard.yml new file mode 100644 index 000000000..b861cc9c9 --- /dev/null +++ b/.github/workflows/maintainer-scorecard.yml @@ -0,0 +1,44 @@ +name: Maintainer Scorecard + +# Daily maintainer to-do digest posted to Discord: SLA breaches, ready-to-merge +# PRs, per-maintainer queues, new/stale items, and response-time stats. +# Requires the DISCORD_MAINTAINER_WEBHOOK repo secret; without it the digest +# only lands in the workflow step summary. + +on: + schedule: + # 21:00 UTC ≈ start of the Australian workday + - cron: '0 21 * * *' + workflow_dispatch: + inputs: + dry_run: + description: 'Print the digest instead of posting to Discord' + type: boolean + default: true + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: {} + +jobs: + scorecard: + name: Scorecard + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Tools + uses: TanStack/config/.github/setup@190f659075ff0845850e330883eb26d7ffd0671f # main + - name: Build and post scorecard + run: pnpm maintainer:scorecard + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_MAINTAINER_WEBHOOK }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run && '1' || '' }} diff --git a/.github/workflows/maintainer-sweep.yml b/.github/workflows/maintainer-sweep.yml new file mode 100644 index 000000000..755482ceb --- /dev/null +++ b/.github/workflows/maintainer-sweep.yml @@ -0,0 +1,43 @@ +name: Maintainer Sweep + +# Scheduled triage sweep: assigns open PRs/issues to maintainers, posts a +# one-time ack + pre-review checklist, and reconciles waiting-on labels. +# API-only — PR code is never checked out or executed (no pull_request_target). + +on: + schedule: + - cron: '17 */3 * * *' + workflow_dispatch: + inputs: + dry_run: + description: 'Print planned actions without executing them' + type: boolean + default: true + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: {} + +jobs: + sweep: + name: Sweep + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + issues: write + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Tools + uses: TanStack/config/.github/setup@190f659075ff0845850e330883eb26d7ffd0671f # main + - name: Run sweep + run: pnpm maintainer:sweep + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run && '1' || '' }} diff --git a/package.json b/package.json index bbfd9e496..d6bd3a18c 100644 --- a/package.json +++ b/package.json @@ -14,8 +14,8 @@ "clean": "pnpm --filter \"./packages/**\" run clean", "clean:all": "git clean -fdx --exclude=\"!.env\"", "test": "pnpm run test:ci", - "test:pr": "pnpm run test:react-native && nx affected --targets=test:sherif,test:knip,test:docs,test:kiira,test:eslint,test:lib,test:types,test:build,build && pnpm test:dts", - "test:ci": "pnpm run test:react-native && nx run-many --targets=test:sherif,test:knip,test:docs,test:kiira,test:eslint,test:lib,test:types,test:build,build && pnpm test:dts", + "test:pr": "pnpm run test:react-native && nx affected --targets=test:sherif,test:knip,test:docs,test:kiira,test:maintainer,test:eslint,test:lib,test:types,test:build,build && pnpm test:dts", + "test:ci": "pnpm run test:react-native && nx run-many --targets=test:sherif,test:knip,test:docs,test:kiira,test:maintainer,test:eslint,test:lib,test:types,test:build,build && pnpm test:dts", "test:eslint": "nx affected --target=test:eslint --exclude=examples/**,testing/**", "test:sherif": "sherif", "test:lib": "nx affected --targets=test:lib --exclude=examples/**,testing/**", @@ -26,6 +26,9 @@ "test:knip": "knip", "test:docs": "tsx scripts/verify-links.ts", "test:dts": "node scripts/scan-dangling-dts.mjs", + "test:maintainer": "vitest run --root scripts/maintainer", + "maintainer:sweep": "tsx scripts/maintainer/sweep.ts", + "maintainer:scorecard": "tsx scripts/maintainer/scorecard.ts", "test:kiira": "kiira check", "test:react-native": "pnpm --filter @tanstack/ai-react-native-smoke smoke", "test:e2e": "pnpm --filter @tanstack/ai-e2e test:e2e", @@ -55,6 +58,7 @@ "test:dts", "test:kiira", "test:knip", + "test:maintainer", "test:sherif" ] }, diff --git a/scripts/maintainer/README.md b/scripts/maintainer/README.md new file mode 100644 index 000000000..b4d5308d8 --- /dev/null +++ b/scripts/maintainer/README.md @@ -0,0 +1,61 @@ +# Maintainer toolset + +Automation that keeps on top of open PRs, issues, and discussions: + +- **Sweep** (`.github/workflows/maintainer-sweep.yml`, every 3h): assigns each + open PR/issue to a maintainer, posts a one-time ack comment with a + deterministic pre-review checklist, asks for a reproduction on bug reports + that lack one, and reconciles `waiting-on: *` / `ready-to-merge` / + `needs-repro` / `has-pr` labels. +- **Scorecard** (`.github/workflows/maintainer-scorecard.yml`, daily): posts a + to-do digest to Discord — SLA breaches, ready-to-merge PRs, per-maintainer + queues (including which items got fresh contributor replies), new/flagged/ + stale items, unanswered discussions, and response-time stats. + +Both are **stateless and API-only**: every metric (first-response time, 7-day +deltas, SLA clocks) is recomputed from GitHub timelines each run, idempotency +comes from HTML markers in bot comments plus visible assignment/label state, +and PR code is never checked out or executed (no `pull_request_target`). + +## Configuration — `.github/maintainers.json` + +| Field | Meaning | +| ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `maintainers[].github` | GitHub login; the roster defines who counts as "a maintainer responded". | +| `maintainers[].discord` | Discord user id (snowflake) for digest @-mentions; `null` → plain bold name. | +| `maintainers[].areas` | Optional file globs (`**`, `*`, `?`) giving this maintainer routing priority for matching PRs (issues match package names in title/body). Omitted/empty (the default) → assignment is pure least-loaded rotation. | +| `maintainers[].maxOpenAssignments` | Routing skips a maintainer at this many open assignments. | +| `sla.firstResponseHours` | Deadline for the first human response on a new item (default 24h). | +| `sla.followUpResponseHours` | Deadline for answering a follow-up message (default 48h). | +| `sla.staleAuthorDays` | Author silence before an item is a nudge/close candidate (default 14d). | +| `spam.*` | Drive-by/bounty heuristic: account younger than `maxAccountAgeDays` **and** diff ≤ `maxChangedLines` **and** no linked issue → flagged for human judgment, never auto-acked. | +| `botAllowlist` | Logins always treated as bots (excluded from human metrics and acks). | +| `maxCommentsPerRun` | Safety cap on comments per sweep; overflow defers to the next run. | + +## Secrets + +- `DISCORD_MAINTAINER_WEBHOOK` — Discord webhook URL for the daily digest + (Server Settings → Integrations → Webhooks). Without it, the digest lands in + the workflow step summary only. +- The sweep uses the default `GITHUB_TOKEN` — no extra setup. + +## Running locally + +```bash +# Dry-run: prints the mutation plan / digest, changes nothing +pnpm maintainer:sweep --dry-run +pnpm maintainer:scorecard --dry-run + +# Tests +pnpm test:maintainer +``` + +Both entries use `GITHUB_TOKEN` if set, else fall back to `gh auth token`. + +## Layout + +Pure logic (`classify.ts` waiting-on state machine + SLA clocks, `route.ts` +assignment, `metrics.ts` scorecard, `glob.ts`) is fixture-tested and does no +I/O. `collect.ts` (GraphQL, read-only), `actions.ts` (REST mutations), +`discord.ts` (webhook) are the I/O seams; `sweep.ts` / `scorecard.ts` are the +entry points. diff --git a/scripts/maintainer/actions.ts b/scripts/maintainer/actions.ts new file mode 100644 index 000000000..4e21825f8 --- /dev/null +++ b/scripts/maintainer/actions.ts @@ -0,0 +1,142 @@ +/** + * Mutation planning + execution for the sweep. The sweep builds a plan of + * `Mutation` objects first (printable in dry-run mode), then executes them + * through the REST API. + */ + +import type { GitHubClient } from './github' + +export const MANAGED_LABELS: Array<{ + name: string + color: string + description: string +}> = [ + { + name: 'waiting-on: maintainer', + color: 'd93f0b', + description: 'The ball is in the maintainers’ court', + }, + { + name: 'waiting-on: author', + color: 'fbca04', + description: 'Waiting for the author to respond or update', + }, + { + name: 'ready-to-merge', + color: '0e8a16', + description: 'Approved, CI green, no conflicts', + }, + { + name: 'needs-repro', + color: 'e99695', + description: 'Needs a minimal reproduction before it can be worked on', + }, + { + name: 'has-pr', + color: 'bfd4f2', + description: 'An open PR references this issue', + }, +] + +const MANAGED_LABEL_NAMES = new Set(MANAGED_LABELS.map((l) => l.name)) + +export type Mutation = + | { kind: 'assign'; number: number; assignee: string } + | { kind: 'comment'; number: number; body: string; note: string } + | { kind: 'add-labels'; number: number; labels: Array } + | { kind: 'remove-label'; number: number; label: string } + +export function describeMutation(m: Mutation): string { + switch (m.kind) { + case 'assign': + return `assign #${m.number} → @${m.assignee}` + case 'comment': + return `comment on #${m.number} (${m.note})` + case 'add-labels': + return `label #${m.number} + [${m.labels.join(', ')}]` + case 'remove-label': + return `label #${m.number} − [${m.label}]` + } +} + +/** + * Diff an item's current labels against the desired managed labels; labels + * outside MANAGED_LABELS are never touched. + */ +export function planLabelChanges( + number: number, + currentLabels: Array, + desiredManaged: Array, +): Array { + const current = new Set( + currentLabels.filter((l) => MANAGED_LABEL_NAMES.has(l)), + ) + const desired = new Set(desiredManaged) + const mutations: Array = [] + const toAdd = [...desired].filter((l) => !current.has(l)) + if (toAdd.length > 0) { + mutations.push({ kind: 'add-labels', number, labels: toAdd }) + } + for (const label of current) { + if (!desired.has(label)) { + mutations.push({ kind: 'remove-label', number, label }) + } + } + return mutations +} + +export async function ensureManagedLabels( + client: GitHubClient, + repo: string, +): Promise { + for (const label of MANAGED_LABELS) { + try { + await client.rest('POST', `/repos/${repo}/labels`, label) + } catch (error) { + // 422 = already exists; anything else should surface. + if (!(error instanceof Error) || !error.message.includes('422')) { + throw error + } + } + } +} + +export async function executeMutations( + client: GitHubClient, + repo: string, + mutations: Array, +): Promise { + for (const m of mutations) { + switch (m.kind) { + case 'assign': + await client.rest( + 'POST', + `/repos/${repo}/issues/${m.number}/assignees`, + { + assignees: [m.assignee], + }, + ) + break + case 'comment': + await client.rest( + 'POST', + `/repos/${repo}/issues/${m.number}/comments`, + { + body: m.body, + }, + ) + break + case 'add-labels': + await client.rest('POST', `/repos/${repo}/issues/${m.number}/labels`, { + labels: m.labels, + }) + break + case 'remove-label': + await client.rest( + 'DELETE', + `/repos/${repo}/issues/${m.number}/labels/${encodeURIComponent(m.label)}`, + ) + break + } + } +} diff --git a/scripts/maintainer/classify.test.ts b/scripts/maintainer/classify.test.ts new file mode 100644 index 000000000..5bc38e598 --- /dev/null +++ b/scripts/maintainer/classify.test.ts @@ -0,0 +1,370 @@ +import { describe, expect, it } from 'vitest' +import { classifyDiscussion, classifyIssue, classifyPR } from './classify' +import { ACK_MARKER, REPRO_MARKER } from './comments' +import { + NOW, + comment, + commit, + config, + daysAgo, + hoursAgo, + makeDiscussion, + makeIssue, + makePR, + review, +} from './fixtures' + +describe('classifyPR — waiting-on state machine', () => { + it('brand-new PR with no maintainer response waits on maintainer', () => { + const t = classifyPR(makePR({ createdAt: hoursAgo(3) }), config, NOW) + expect(t.waitingOn).toBe('maintainer') + expect(t.waitingReason).toBe('no-response') + expect(t.unansweredSince).toBe(hoursAgo(3)) + expect(t.slaBreached).toBe(false) // 3h < 24h first-response SLA + }) + + it('breaches first-response SLA after 24h of silence', () => { + const t = classifyPR(makePR({ createdAt: hoursAgo(30) }), config, NOW) + expect(t.slaBreached).toBe(true) + expect(t.unansweredHours).toBeCloseTo(30, 0) + }) + + it('maintainer replied last → waiting on author, SLA clock cleared', () => { + const t = classifyPR( + makePR({ + createdAt: daysAgo(3), + timeline: [ + comment('contributor', daysAgo(2), 'please review'), + comment('alem', daysAgo(1), 'looking at it'), + ], + }), + config, + NOW, + ) + expect(t.waitingOn).toBe('author') + expect(t.waitingReason).toBe('maintainer-replied') + expect(t.unansweredSince).toBeNull() + expect(t.slaBreached).toBe(false) + expect(t.firstResponseHours).toBeCloseTo(48, 0) + }) + + it('author replied after maintainer → back to waiting on maintainer (follow-up SLA)', () => { + const t = classifyPR( + makePR({ + createdAt: daysAgo(6), + timeline: [ + comment('alem', daysAgo(5), 'needs changes'), + comment('contributor', daysAgo(3), 'done, please re-check'), + ], + }), + config, + NOW, + ) + expect(t.waitingOn).toBe('maintainer') + expect(t.waitingReason).toBe('author-replied') + expect(t.freshContributorReply).toBe(true) + expect(t.unansweredSince).toBe(daysAgo(3)) + expect(t.slaBreached).toBe(true) // 72h > 48h follow-up SLA + }) + + it('the PR author being a maintainer does not count as a response', () => { + const t = classifyPR( + makePR({ + author: 'tom', + createdAt: hoursAgo(30), + timeline: [comment('tom', hoursAgo(29), 'self-note')], + }), + config, + NOW, + ) + expect(t.rosterAuthor).toBe(true) + expect(t.lastMaintainerResponseAt).toBeNull() + // roster-authored PRs are exempt from the SLA clock + expect(t.slaBreached).toBe(false) + }) + + it('bot comments never count as maintainer responses', () => { + const t = classifyPR( + makePR({ + createdAt: hoursAgo(30), + timeline: [comment('github-actions[bot]', hoursAgo(29), 'ack', true)], + }), + config, + NOW, + ) + expect(t.lastMaintainerResponseAt).toBeNull() + expect(t.slaBreached).toBe(true) + }) + + it('approved + green + conflict-free → ready to merge, waiting on maintainer', () => { + const t = classifyPR( + makePR({ reviewDecision: 'APPROVED', ciState: 'success' }), + config, + NOW, + ) + expect(t.readyToMerge).toBe(true) + expect(t.waitingOn).toBe('maintainer') + expect(t.waitingReason).toBe('ready-to-merge') + }) + + it('conflicts beat CI and last-mover: waiting on author', () => { + const t = classifyPR( + makePR({ + mergeable: 'CONFLICTING', + timeline: [comment('contributor', hoursAgo(2), 'ping?')], + }), + config, + NOW, + ) + expect(t.waitingOn).toBe('author') + expect(t.waitingReason).toBe('merge-conflicts') + // …but the unanswered-message clock still runs independently + expect(t.unansweredSince).not.toBeNull() + }) + + it('failing CI → waiting on author', () => { + const t = classifyPR(makePR({ ciState: 'failure' }), config, NOW) + expect(t.waitingOn).toBe('author') + expect(t.waitingReason).toBe('ci-failing') + }) + + it('outstanding changes-requested waits on author; author push flips it back', () => { + const requested = classifyPR( + makePR({ + reviewDecision: 'CHANGES_REQUESTED', + timeline: [review('alem', daysAgo(2), 'CHANGES_REQUESTED')], + }), + config, + NOW, + ) + expect(requested.waitingOn).toBe('author') + expect(requested.waitingReason).toBe('changes-requested') + + const pushed = classifyPR( + makePR({ + reviewDecision: 'CHANGES_REQUESTED', + timeline: [ + review('alem', daysAgo(2), 'CHANGES_REQUESTED'), + commit('contributor', daysAgo(1)), + ], + }), + config, + NOW, + ) + expect(pushed.waitingOn).toBe('maintainer') + }) + + it('drafts wait on author and never breach SLA', () => { + const t = classifyPR( + makePR({ isDraft: true, createdAt: daysAgo(10) }), + config, + NOW, + ) + expect(t.waitingOn).toBe('author') + expect(t.waitingReason).toBe('draft') + expect(t.slaBreached).toBe(false) + }) + + it('goes stale after author silence beyond the threshold', () => { + const t = classifyPR( + makePR({ + createdAt: daysAgo(30), + mergeable: 'CONFLICTING', + timeline: [ + comment('alem', daysAgo(20), 'please rebase'), + comment('contributor', daysAgo(16), 'will do'), + ], + }), + config, + NOW, + ) + expect(t.waitingOn).toBe('author') + expect(t.staleAuthor).toBe(true) + }) +}) + +describe('classifyPR — hygiene and flags', () => { + it('flags missing changeset and missing E2E only for published-source PRs', () => { + const src = classifyPR( + makePR({ files: ['packages/ai/src/core/chat.ts'] }), + config, + NOW, + ) + expect(src.missingChangeset).toBe(true) + expect(src.missingE2E).toBe(true) + + const covered = classifyPR( + makePR({ + files: [ + 'packages/ai/src/core/chat.ts', + '.changeset/two-dogs-run.md', + 'testing/e2e/tests/chat.spec.ts', + ], + }), + config, + NOW, + ) + expect(covered.missingChangeset).toBe(false) + expect(covered.missingE2E).toBe(false) + + const docsOnly = classifyPR( + makePR({ files: ['docs/guide.md'] }), + config, + NOW, + ) + expect(docsOnly.missingChangeset).toBe(false) + expect(docsOnly.missingE2E).toBe(false) + }) + + it('suspects spam only when all three signals align', () => { + const spam = classifyPR( + makePR({ + authorAccountCreatedAt: daysAgo(5), + additions: 2, + deletions: 1, + linkedIssues: [], + authorAssociation: 'NONE', + }), + config, + NOW, + ) + expect(spam.suspectedSpam).toBe(true) + expect(spam.spamReasons).toHaveLength(3) + + const youngButSubstantial = classifyPR( + makePR({ + authorAccountCreatedAt: daysAgo(5), + additions: 400, + deletions: 100, + linkedIssues: [], + authorAssociation: 'NONE', + }), + config, + NOW, + ) + expect(youngButSubstantial.suspectedSpam).toBe(false) + + const collaborator = classifyPR( + makePR({ + authorAccountCreatedAt: daysAgo(5), + additions: 2, + deletions: 1, + linkedIssues: [], + authorAssociation: 'COLLABORATOR', + }), + config, + NOW, + ) + expect(collaborator.suspectedSpam).toBe(false) + }) + + it('detects a prior ack comment via the marker', () => { + const t = classifyPR( + makePR({ + timeline: [ + comment( + 'github-actions[bot]', + daysAgo(1), + `Thanks!\n${ACK_MARKER}`, + true, + ), + ], + }), + config, + NOW, + ) + expect(t.hasAckComment).toBe(true) + }) + + it('flags first-time contributors', () => { + const t = classifyPR( + makePR({ authorAssociation: 'FIRST_TIME_CONTRIBUTOR' }), + config, + NOW, + ) + expect(t.firstTimeContributor).toBe(true) + }) +}) + +describe('classifyIssue', () => { + it('needs repro when the body has no link or code fence', () => { + const bare = classifyIssue( + makeIssue({ body: 'it just breaks, please fix' }), + config, + NOW, + new Set(), + ) + expect(bare.needsRepro).toBe(true) + + const withLink = classifyIssue(makeIssue(), config, NOW, new Set()) + expect(withLink.needsRepro).toBe(false) + + const withCode = classifyIssue( + makeIssue({ body: 'crash:\n```ts\nchat()\n```' }), + config, + NOW, + new Set(), + ) + expect(withCode.needsRepro).toBe(false) + }) + + it('links issues to open PRs that close them', () => { + const t = classifyIssue( + makeIssue({ number: 50 }), + config, + NOW, + new Set([50]), + ) + expect(t.hasOpenPR).toBe(true) + }) + + it('detects a prior repro-request comment via the marker', () => { + const t = classifyIssue( + makeIssue({ + body: 'no repro here', + timeline: [ + comment( + 'github-actions[bot]', + daysAgo(1), + `Repro?\n${REPRO_MARKER}`, + true, + ), + ], + }), + config, + NOW, + new Set(), + ) + expect(t.hasReproComment).toBe(true) + }) + + it('runs the same response SLA as PRs', () => { + const t = classifyIssue( + makeIssue({ createdAt: hoursAgo(30) }), + config, + NOW, + new Set(), + ) + expect(t.waitingOn).toBe('maintainer') + expect(t.slaBreached).toBe(true) + }) +}) + +describe('classifyDiscussion', () => { + it('needs attention when unanswered with no maintainer comment', () => { + const t = classifyDiscussion(makeDiscussion(), config, NOW) + expect(t.needsAttention).toBe(true) + expect(t.waitingHours).toBeCloseTo(72, 0) + }) + + it('a maintainer comment clears the attention flag even if unanswered', () => { + const t = classifyDiscussion( + makeDiscussion({ + comments: [{ actor: 'jack', isBot: false, at: daysAgo(1) }], + }), + config, + NOW, + ) + expect(t.needsAttention).toBe(false) + }) +}) diff --git a/scripts/maintainer/classify.ts b/scripts/maintainer/classify.ts new file mode 100644 index 000000000..b648c322c --- /dev/null +++ b/scripts/maintainer/classify.ts @@ -0,0 +1,383 @@ +/** + * Pure classification logic: waiting-on state machine, SLA clocks, spam + * heuristic, and per-item hygiene checks. No I/O — fixture-testable. + */ + +import { isRosterMaintainer } from './config' +import { ACK_MARKER, REPRO_MARKER } from './comments' +import { matchesAnyGlob } from './glob' +import type { + DiscussionItem, + DiscussionTriage, + IssueItem, + IssueTriage, + PRItem, + PRTriage, + RepoSnapshot, + TimelineEvent, + ToolsetConfig, + TriageResult, + WaitingOn, +} from './types' + +export const HOUR_MS = 60 * 60 * 1000 + +export function hoursBetween(fromIso: string, to: Date): number { + return (to.getTime() - new Date(fromIso).getTime()) / HOUR_MS +} + +const PUBLISHED_SRC_GLOBS = ['packages/*/src/**'] +const CHANGESET_GLOBS = ['.changeset/*.md'] +const E2E_GLOBS = ['testing/e2e/**'] + +interface ResponseState { + lastMaintainerResponseAt: string | null + lastContributorActivityAt: string | null + unansweredSince: string | null + firstResponseHours: number | null +} + +/** + * Derives the conversational state of an item from its timeline. + * + * "Maintainer response" = a comment or review by a roster maintainer who is + * not the item's author (your own comments on your own PR aren't responses). + * Bot activity never counts on either side of the clock. + */ +export function deriveResponseState( + timeline: Array, + author: string | null, + createdAt: string, + config: ToolsetConfig, + now: Date, +): ResponseState { + const isMaintainerResponse = (e: TimelineEvent) => + (e.kind === 'comment' || e.kind === 'review') && + !e.isBot && + e.actor !== null && + e.actor !== author && + isRosterMaintainer(e.actor, config) + + const isContributorActivity = (e: TimelineEvent) => + !e.isBot && + (e.actor === author || + e.actor === null || + !isRosterMaintainer(e.actor, config)) + + const sorted = [...timeline].sort( + (a, b) => new Date(a.at).getTime() - new Date(b.at).getTime(), + ) + + let lastMaintainerResponseAt: string | null = null + let firstMaintainerResponseAt: string | null = null + let lastContributorActivityAt: string | null = null + for (const event of sorted) { + if (isMaintainerResponse(event)) { + firstMaintainerResponseAt ??= event.at + lastMaintainerResponseAt = event.at + } else if (isContributorActivity(event)) { + lastContributorActivityAt = event.at + } + } + + // Oldest contributor *message* still awaiting a reply. Pushed commits move + // the "fresh activity" clock but don't demand a written response by + // themselves; the item creation itself does. + let unansweredSince: string | null = null + if (lastMaintainerResponseAt === null) { + unansweredSince = createdAt + } else { + const cutoff = new Date(lastMaintainerResponseAt).getTime() + for (const event of sorted) { + if ( + (event.kind === 'comment' || event.kind === 'review') && + isContributorActivity(event) && + new Date(event.at).getTime() > cutoff + ) { + unansweredSince = event.at + break + } + } + } + + return { + lastMaintainerResponseAt, + lastContributorActivityAt, + unansweredSince, + firstResponseHours: + firstMaintainerResponseAt === null + ? null + : Math.max( + 0, + (new Date(firstMaintainerResponseAt).getTime() - + new Date(createdAt).getTime()) / + HOUR_MS, + ), + } +} + +function slaBreach( + state: ResponseState, + config: ToolsetConfig, + now: Date, +): { unansweredHours: number | null; slaBreached: boolean } { + if (state.unansweredSince === null) { + return { unansweredHours: null, slaBreached: false } + } + const unansweredHours = hoursBetween(state.unansweredSince, now) + const threshold = + state.lastMaintainerResponseAt === null + ? config.sla.firstResponseHours + : config.sla.followUpResponseHours + return { unansweredHours, slaBreached: unansweredHours > threshold } +} + +function hasMarkerComment( + timeline: Array, + marker: string, +): boolean { + return timeline.some( + (e) => + e.kind === 'comment' && e.body !== undefined && e.body.includes(marker), + ) +} + +export function classifyPR( + pr: PRItem, + config: ToolsetConfig, + now: Date, +): PRTriage { + const bot = pr.authorIsBot + const rosterAuthor = isRosterMaintainer(pr.author, config) + const firstTimeContributor = + pr.authorAssociation === 'FIRST_TIME_CONTRIBUTOR' || + pr.authorAssociation === 'FIRST_TIMER' + + const state = deriveResponseState( + pr.timeline, + pr.author, + pr.createdAt, + config, + now, + ) + const { unansweredHours, slaBreached } = slaBreach(state, config, now) + + const hasConflicts = pr.mergeable === 'CONFLICTING' + const ciFailing = pr.ciState === 'failure' + const approved = pr.reviewDecision === 'APPROVED' + const readyToMerge = approved && !hasConflicts && !ciFailing && !pr.isDraft + + // Outstanding changes-requested: the decision stands and the contributor + // hasn't commented or pushed since the last maintainer response. + const changesRequestedOutstanding = + pr.reviewDecision === 'CHANGES_REQUESTED' && + (state.lastContributorActivityAt === null || + (state.lastMaintainerResponseAt !== null && + state.lastContributorActivityAt <= state.lastMaintainerResponseAt)) + + let waitingOn: WaitingOn + let waitingReason: string + if (bot) { + waitingOn = 'nobody' + waitingReason = 'bot' + } else if (pr.isDraft) { + waitingOn = 'author' + waitingReason = 'draft' + } else if (readyToMerge) { + waitingOn = 'maintainer' + waitingReason = 'ready-to-merge' + } else if (hasConflicts) { + waitingOn = 'author' + waitingReason = 'merge-conflicts' + } else if (ciFailing) { + waitingOn = 'author' + waitingReason = 'ci-failing' + } else if (changesRequestedOutstanding) { + waitingOn = 'author' + waitingReason = 'changes-requested' + } else if ( + state.lastMaintainerResponseAt !== null && + (state.lastContributorActivityAt === null || + state.lastContributorActivityAt <= state.lastMaintainerResponseAt) + ) { + waitingOn = 'author' + waitingReason = 'maintainer-replied' + } else { + waitingOn = 'maintainer' + waitingReason = + state.lastMaintainerResponseAt === null ? 'no-response' : 'author-replied' + } + + const spamReasons: Array = [] + if (!bot && !rosterAuthor) { + const trusted = + pr.authorAssociation === 'MEMBER' || + pr.authorAssociation === 'OWNER' || + pr.authorAssociation === 'COLLABORATOR' + const accountAgeDays = + pr.authorAccountCreatedAt === null + ? null + : hoursBetween(pr.authorAccountCreatedAt, now) / 24 + if ( + !trusted && + accountAgeDays !== null && + accountAgeDays < config.spam.maxAccountAgeDays + ) { + spamReasons.push(`account is ${Math.floor(accountAgeDays)}d old`) + } + if ( + !trusted && + pr.additions + pr.deletions <= config.spam.maxChangedLines + ) { + spamReasons.push(`trivial diff (+${pr.additions}/−${pr.deletions})`) + } + if (!trusted && pr.linkedIssues.length === 0) { + spamReasons.push('no linked issue') + } + } + // All three signals together → likely drive-by bounty farming. + const suspectedSpam = spamReasons.length === 3 + + const touchesPublished = pr.files.some((f) => + matchesAnyGlob(f, PUBLISHED_SRC_GLOBS), + ) + const missingChangeset = + touchesPublished && + !pr.files.some((f) => matchesAnyGlob(f, CHANGESET_GLOBS)) + const missingE2E = + touchesPublished && !pr.files.some((f) => matchesAnyGlob(f, E2E_GLOBS)) + + const freshContributorReply = + state.lastContributorActivityAt !== null && + (state.lastMaintainerResponseAt === null || + state.lastContributorActivityAt > state.lastMaintainerResponseAt) + + const staleAuthor = + waitingOn === 'author' && + !pr.isDraft && + hoursBetween(state.lastContributorActivityAt ?? pr.createdAt, now) / 24 > + config.sla.staleAuthorDays + + return { + item: pr, + bot, + rosterAuthor, + firstTimeContributor, + suspectedSpam, + spamReasons, + waitingOn, + waitingReason, + readyToMerge, + hasConflicts, + ciFailing, + missingChangeset, + missingE2E, + lastMaintainerResponseAt: state.lastMaintainerResponseAt, + lastContributorActivityAt: state.lastContributorActivityAt, + unansweredSince: state.unansweredSince, + unansweredHours, + slaBreached: !bot && !rosterAuthor && slaBreached && !pr.isDraft, + firstResponseHours: state.firstResponseHours, + freshContributorReply, + staleAuthor, + hasAckComment: hasMarkerComment(pr.timeline, ACK_MARKER), + suggestedAssignee: null, + } +} + +function bodyHasReproSignal(body: string): boolean { + return /https?:\/\//.test(body) || body.includes('```') +} + +export function classifyIssue( + issue: IssueItem, + config: ToolsetConfig, + now: Date, + openPRIssueNumbers: Set, +): IssueTriage { + const bot = issue.authorIsBot + const rosterAuthor = isRosterMaintainer(issue.author, config) + const state = deriveResponseState( + issue.timeline, + issue.author, + issue.createdAt, + config, + now, + ) + const { unansweredHours, slaBreached } = slaBreach(state, config, now) + + let waitingOn: WaitingOn + if (bot) { + waitingOn = 'nobody' + } else if ( + state.lastMaintainerResponseAt !== null && + (state.lastContributorActivityAt === null || + state.lastContributorActivityAt <= state.lastMaintainerResponseAt) + ) { + waitingOn = 'author' + } else { + waitingOn = 'maintainer' + } + + const staleAuthor = + waitingOn === 'author' && + hoursBetween(state.lastContributorActivityAt ?? issue.createdAt, now) / 24 > + config.sla.staleAuthorDays + + return { + item: issue, + bot, + rosterAuthor, + waitingOn, + needsRepro: !bot && !rosterAuthor && !bodyHasReproSignal(issue.body), + hasOpenPR: openPRIssueNumbers.has(issue.number), + lastMaintainerResponseAt: state.lastMaintainerResponseAt, + lastContributorActivityAt: state.lastContributorActivityAt, + unansweredSince: state.unansweredSince, + unansweredHours, + slaBreached: !bot && !rosterAuthor && slaBreached, + firstResponseHours: state.firstResponseHours, + staleAuthor, + hasReproComment: hasMarkerComment(issue.timeline, REPRO_MARKER), + suggestedAssignee: null, + } +} + +export function classifyDiscussion( + discussion: DiscussionItem, + config: ToolsetConfig, + now: Date, +): DiscussionTriage { + const maintainerReplied = discussion.comments.some( + (c) => !c.isBot && isRosterMaintainer(c.actor, config), + ) + const needsAttention = !discussion.isAnswered && !maintainerReplied + const lastActivity = + discussion.comments.length > 0 + ? discussion.comments[discussion.comments.length - 1]!.at + : discussion.createdAt + return { + item: discussion, + needsAttention, + waitingHours: Math.max(0, hoursBetween(lastActivity, now)), + } +} + +export function classifyAll( + snapshot: RepoSnapshot, + config: ToolsetConfig, + now: Date = new Date(snapshot.takenAt), +): TriageResult { + const openPRIssueNumbers = new Set( + snapshot.prs.flatMap((pr) => pr.linkedIssues), + ) + return { + prs: snapshot.prs.map((pr) => classifyPR(pr, config, now)), + issues: snapshot.issues.map((issue) => + classifyIssue(issue, config, now, openPRIssueNumbers), + ), + discussions: snapshot.discussions.map((d) => + classifyDiscussion(d, config, now), + ), + } +} diff --git a/scripts/maintainer/collect.ts b/scripts/maintainer/collect.ts new file mode 100644 index 000000000..e542552bf --- /dev/null +++ b/scripts/maintainer/collect.ts @@ -0,0 +1,385 @@ +/** + * Snapshot collection: pulls open PRs/issues/discussions (with timelines) and + * recently closed items from the GitHub GraphQL API and normalizes them into + * the shapes in types.ts. Read-only — never mutates anything. + */ + +import { isBotLogin } from './config' +import type { GitHubClient } from './github' +import type { + CIState, + ClosedItem, + DiscussionItem, + IssueItem, + PRItem, + RepoSnapshot, + TimelineEvent, + ToolsetConfig, +} from './types' + +const CLOSED_LOOKBACK_DAYS = 14 + +interface GqlActor { + login?: string + __typename?: string + createdAt?: string +} + +interface GqlTimelineNode { + __typename: string + author?: GqlActor | null + createdAt?: string + submittedAt?: string + state?: string + body?: string + commit?: { + committedDate: string + author?: { user?: { login: string } | null } | null + } | null +} + +function actorLogin(actor: GqlActor | null | undefined): string | null { + return actor?.login ?? null +} + +function actorIsBot( + actor: GqlActor | null | undefined, + config: ToolsetConfig, +): boolean { + if (actor?.__typename === 'Bot') return true + return isBotLogin(actorLogin(actor), config) +} + +function normalizeTimeline( + nodes: Array, + fallbackAuthor: string | null, + config: ToolsetConfig, +): Array { + const events: Array = [] + for (const node of nodes) { + if (!node) continue + if (node.__typename === 'IssueComment' && node.createdAt) { + events.push({ + kind: 'comment', + actor: actorLogin(node.author), + isBot: actorIsBot(node.author, config), + at: node.createdAt, + body: node.body ?? '', + }) + } else if (node.__typename === 'PullRequestReview' && node.submittedAt) { + events.push({ + kind: 'review', + actor: actorLogin(node.author), + isBot: actorIsBot(node.author, config), + at: node.submittedAt, + reviewState: node.state, + }) + } else if (node.__typename === 'PullRequestCommit' && node.commit) { + const login = node.commit.author?.user?.login ?? fallbackAuthor + events.push({ + kind: 'commit', + actor: login, + isBot: isBotLogin(login, config), + at: node.commit.committedDate, + }) + } + } + return events +} + +function normalizeCIState(state: string | null | undefined): CIState { + switch (state) { + case 'SUCCESS': + return 'success' + case 'FAILURE': + case 'ERROR': + return 'failure' + case 'PENDING': + case 'EXPECTED': + return 'pending' + default: + return 'unknown' + } +} + +const COMMENT_FRAGMENT = ` + __typename + ... on IssueComment { author { login __typename } createdAt body } +` + +const PR_TIMELINE_FRAGMENT = ` + ${COMMENT_FRAGMENT} + ... on PullRequestReview { author { login __typename } submittedAt state } + ... on PullRequestCommit { commit { committedDate author { user { login } } } } +` + +const OPEN_PRS_QUERY = ` +query($owner: String!, $name: String!, $cursor: String) { + repository(owner: $owner, name: $name) { + pullRequests(states: OPEN, first: 20, after: $cursor, orderBy: {field: CREATED_AT, direction: DESC}) { + pageInfo { hasNextPage endCursor } + nodes { + number title url createdAt updatedAt isDraft mergeable reviewDecision + additions deletions changedFiles authorAssociation + author { login __typename ... on User { createdAt } } + labels(first: 20) { nodes { name } } + assignees(first: 10) { nodes { login } } + files(first: 100) { nodes { path } } + closingIssuesReferences(first: 10) { nodes { number } } + commits(last: 1) { nodes { commit { statusCheckRollup { state } } } } + timelineItems(last: 100, itemTypes: [ISSUE_COMMENT, PULL_REQUEST_REVIEW, PULL_REQUEST_COMMIT]) { + nodes { ${PR_TIMELINE_FRAGMENT} } + } + } + } + } +}` + +const OPEN_ISSUES_QUERY = ` +query($owner: String!, $name: String!, $cursor: String) { + repository(owner: $owner, name: $name) { + issues(states: OPEN, first: 40, after: $cursor, orderBy: {field: CREATED_AT, direction: DESC}) { + pageInfo { hasNextPage endCursor } + nodes { + number title url createdAt updatedAt body authorAssociation + author { login __typename } + labels(first: 20) { nodes { name } } + assignees(first: 10) { nodes { login } } + timelineItems(last: 100, itemTypes: [ISSUE_COMMENT]) { + nodes { ${COMMENT_FRAGMENT} } + } + } + } + } +}` + +const DISCUSSIONS_QUERY = ` +query($owner: String!, $name: String!, $cursor: String) { + repository(owner: $owner, name: $name) { + discussions(first: 50, after: $cursor, states: OPEN, orderBy: {field: UPDATED_AT, direction: DESC}) { + pageInfo { hasNextPage endCursor } + nodes { + number title url createdAt updatedAt isAnswered upvoteCount + category { name } + author { login __typename } + comments(last: 20) { + totalCount + nodes { author { login __typename } createdAt } + } + } + } + } +}` + +const CLOSED_SEARCH_QUERY = ` +query($query: String!, $cursor: String) { + search(query: $query, type: ISSUE, first: 50, after: $cursor) { + pageInfo { hasNextPage endCursor } + nodes { + __typename + ... on PullRequest { + number title url createdAt closedAt mergedAt + author { login __typename } + timelineItems(first: 30, itemTypes: [ISSUE_COMMENT, PULL_REQUEST_REVIEW]) { + nodes { ${PR_TIMELINE_FRAGMENT} } + } + } + ... on Issue { + number title url createdAt closedAt + author { login __typename } + timelineItems(first: 30, itemTypes: [ISSUE_COMMENT]) { + nodes { ${COMMENT_FRAGMENT} } + } + } + } + } +}` + +async function paginate( + fetchPage: (cursor: string | null) => Promise<{ + nodes: Array + pageInfo: { hasNextPage: boolean; endCursor: string | null } + }>, + maxPages = 20, +): Promise> { + const all: Array = [] + let cursor: string | null = null + for (let page = 0; page < maxPages; page++) { + const result = await fetchPage(cursor) + for (const node of result.nodes) if (node !== null) all.push(node) + if (!result.pageInfo.hasNextPage) break + cursor = result.pageInfo.endCursor + } + return all +} + +export async function collectSnapshot( + client: GitHubClient, + config: ToolsetConfig, + now: Date = new Date(), +): Promise { + const [owner, repo] = config.repo.split('/') as [string, string] + const vars = { owner, name: repo } + + interface Connection { + nodes: Array | null> + pageInfo: { hasNextPage: boolean; endCursor: string | null } + } + + const prNodes = await paginate((cursor) => + client + .graphql<{ repository: { pullRequests: Connection } }>(OPEN_PRS_QUERY, { + ...vars, + cursor, + }) + .then((d) => d.repository.pullRequests), + ) + + const issueNodes = await paginate((cursor) => + client + .graphql<{ repository: { issues: Connection } }>(OPEN_ISSUES_QUERY, { + ...vars, + cursor, + }) + .then((d) => d.repository.issues), + ) + + const discussionNodes = await paginate( + (cursor) => + client + .graphql<{ + repository: { discussions: Connection } + }>(DISCUSSIONS_QUERY, { ...vars, cursor }) + .then((d) => d.repository.discussions), + 4, + ) + + const sinceDate = new Date( + now.getTime() - CLOSED_LOOKBACK_DAYS * 24 * 60 * 60 * 1000, + ) + .toISOString() + .slice(0, 10) + const closedNodes = await paginate( + (cursor) => + client + .graphql<{ search: Connection }>(CLOSED_SEARCH_QUERY, { + query: `repo:${config.repo} closed:>=${sinceDate}`, + cursor, + }) + .then((d) => d.search), + 10, + ) + + const prs: Array = prNodes.map((n) => { + const author = actorLogin(n.author) + return { + type: 'pr', + number: n.number, + title: n.title, + url: n.url, + author, + authorIsBot: actorIsBot(n.author, config), + authorAccountCreatedAt: n.author?.createdAt ?? null, + authorAssociation: n.authorAssociation ?? 'NONE', + createdAt: n.createdAt, + updatedAt: n.updatedAt, + isDraft: Boolean(n.isDraft), + mergeable: n.mergeable ?? 'UNKNOWN', + reviewDecision: n.reviewDecision ?? '', + additions: n.additions ?? 0, + deletions: n.deletions ?? 0, + changedFiles: n.changedFiles ?? 0, + files: (n.files?.nodes ?? []) + .filter(Boolean) + .map((f: { path: string }) => f.path), + labels: (n.labels?.nodes ?? []) + .filter(Boolean) + .map((l: { name: string }) => l.name), + assignees: (n.assignees?.nodes ?? []) + .filter(Boolean) + .map((a: { login: string }) => a.login), + ciState: normalizeCIState( + n.commits?.nodes?.[0]?.commit?.statusCheckRollup?.state, + ), + linkedIssues: (n.closingIssuesReferences?.nodes ?? []) + .filter(Boolean) + .map((i: { number: number }) => i.number), + timeline: normalizeTimeline(n.timelineItems?.nodes ?? [], author, config), + } + }) + + const issues: Array = issueNodes.map((n) => ({ + type: 'issue', + number: n.number, + title: n.title, + url: n.url, + author: actorLogin(n.author), + authorIsBot: actorIsBot(n.author, config), + authorAssociation: n.authorAssociation ?? 'NONE', + createdAt: n.createdAt, + updatedAt: n.updatedAt, + body: n.body ?? '', + labels: (n.labels?.nodes ?? []) + .filter(Boolean) + .map((l: { name: string }) => l.name), + assignees: (n.assignees?.nodes ?? []) + .filter(Boolean) + .map((a: { login: string }) => a.login), + timeline: normalizeTimeline( + n.timelineItems?.nodes ?? [], + actorLogin(n.author), + config, + ), + })) + + const discussions: Array = discussionNodes.map((n) => ({ + type: 'discussion', + number: n.number, + title: n.title, + url: n.url, + author: actorLogin(n.author), + createdAt: n.createdAt, + updatedAt: n.updatedAt, + isAnswered: Boolean(n.isAnswered), + category: n.category?.name ?? '', + upvotes: n.upvoteCount ?? 0, + commentCount: n.comments?.totalCount ?? 0, + comments: (n.comments?.nodes ?? []) + .filter(Boolean) + .map((c: { author: GqlActor | null; createdAt: string }) => ({ + actor: actorLogin(c.author), + isBot: actorIsBot(c.author, config), + at: c.createdAt, + })), + })) + + const recentlyClosed: Array = closedNodes + .filter((n) => n.closedAt) + .map((n) => ({ + type: + n.__typename === 'PullRequest' ? ('pr' as const) : ('issue' as const), + number: n.number, + title: n.title, + url: n.url, + author: actorLogin(n.author), + authorIsBot: actorIsBot(n.author, config), + createdAt: n.createdAt, + closedAt: n.closedAt, + mergedAt: n.mergedAt ?? null, + timeline: normalizeTimeline( + n.timelineItems?.nodes ?? [], + actorLogin(n.author), + config, + ), + })) + + return { + owner, + repo, + takenAt: now.toISOString(), + prs, + issues, + discussions, + recentlyClosed, + } +} diff --git a/scripts/maintainer/comments.ts b/scripts/maintainer/comments.ts new file mode 100644 index 000000000..b81a36b27 --- /dev/null +++ b/scripts/maintainer/comments.ts @@ -0,0 +1,58 @@ +import type { PRTriage } from './types' + +/** + * HTML markers embedded in bot comments so sweeps stay idempotent: before + * posting, the sweep checks the item's timeline for the marker. + */ +export const ACK_MARKER = '' +export const REPRO_MARKER = '' + +function checklistLine(ok: boolean, okText: string, warnText: string): string { + return ok ? `- ✅ ${okText}` : `- ⚠️ ${warnText}` +} + +export function buildAckComment(triage: PRTriage, assignee: string): string { + const pr = triage.item + const author = pr.author ? `@${pr.author}` : 'there' + const lines = [ + `Thanks for the PR, ${author}! 🙌 @${assignee} will take a look.`, + '', + '**Automated pre-review checks**', + checklistLine( + !triage.ciFailing, + pr.ciState === 'success' ? 'CI passing' : 'CI pending', + 'CI failing — worth a look before review', + ), + checklistLine( + !triage.hasConflicts, + 'No merge conflicts', + 'Merge conflicts with `main` — please rebase', + ), + checklistLine( + !triage.missingChangeset, + 'Changeset present', + 'No changeset found — run `pnpm changeset` if this changes published packages', + ), + checklistLine( + !triage.missingE2E, + 'E2E test changes included', + 'No E2E test changes detected — behavior changes need coverage under `testing/e2e/` (see CONTRIBUTING)', + ), + '', + 'Automated triage — a human review follows.', + ACK_MARKER, + ] + return lines.join('\n') +} + +export function buildReproComment(author: string | null): string { + const mention = author ? `@${author}` : 'there' + return [ + `Thanks for the report, ${mention}! To help us reproduce this, could you add a minimal reproduction?`, + '', + 'A runnable example makes fixes dramatically faster — you can fork one of the [official examples](https://github.com/TanStack/ai/tree/main/examples/), or use [StackBlitz](https://stackblitz.com/) / [CodeSandbox](https://codesandbox.io/). For TypeScript-only issues, a [TS Playground](https://www.typescriptlang.org/play) link works too.', + '', + 'Automated triage — a maintainer will follow up.', + REPRO_MARKER, + ].join('\n') +} diff --git a/scripts/maintainer/config.ts b/scripts/maintainer/config.ts new file mode 100644 index 000000000..61b447e9c --- /dev/null +++ b/scripts/maintainer/config.ts @@ -0,0 +1,106 @@ +import { readFile } from 'node:fs/promises' +import type { ToolsetConfig } from './types' + +export const DEFAULT_CONFIG_PATH = '.github/maintainers.json' + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function fail(message: string): never { + throw new Error(`Invalid maintainers config: ${message}`) +} + +export function parseConfig(raw: unknown): ToolsetConfig { + if (!isRecord(raw)) fail('root must be an object') + const { repo, maintainers, sla, spam, botAllowlist, maxCommentsPerRun } = raw + + if (typeof repo !== 'string' || !repo.includes('/')) { + fail('"repo" must be an "owner/name" string') + } + if (!Array.isArray(maintainers) || maintainers.length === 0) { + fail('"maintainers" must be a non-empty array') + } + const parsedMaintainers = maintainers.map((m, idx) => { + if (!isRecord(m)) fail(`maintainers[${idx}] must be an object`) + if (typeof m.github !== 'string' || m.github.length === 0) { + fail(`maintainers[${idx}].github must be a string`) + } + const areas = m.areas ?? [] + if (!Array.isArray(areas) || areas.some((a) => typeof a !== 'string')) { + fail(`maintainers[${idx}].areas must be an array of glob strings`) + } + return { + github: m.github, + discord: typeof m.discord === 'string' ? m.discord : null, + areas: areas as Array, + maxOpenAssignments: + typeof m.maxOpenAssignments === 'number' ? m.maxOpenAssignments : 10, + } + }) + + if (!isRecord(sla)) fail('"sla" must be an object') + const slaNumber = (key: string): number => { + const v = sla[key] + if (typeof v !== 'number' || v <= 0) + fail(`sla.${key} must be a positive number`) + return v + } + if (!isRecord(spam)) fail('"spam" must be an object') + const spamNumber = (key: string): number => { + const v = spam[key] + if (typeof v !== 'number' || v <= 0) + fail(`spam.${key} must be a positive number`) + return v + } + + return { + repo, + maintainers: parsedMaintainers, + sla: { + firstResponseHours: slaNumber('firstResponseHours'), + followUpResponseHours: slaNumber('followUpResponseHours'), + staleAuthorDays: slaNumber('staleAuthorDays'), + }, + spam: { + maxAccountAgeDays: spamNumber('maxAccountAgeDays'), + maxChangedLines: spamNumber('maxChangedLines'), + }, + botAllowlist: Array.isArray(botAllowlist) + ? botAllowlist.filter((b): b is string => typeof b === 'string') + : [], + maxCommentsPerRun: + typeof maxCommentsPerRun === 'number' ? maxCommentsPerRun : 25, + } +} + +export async function loadConfig( + path: string = DEFAULT_CONFIG_PATH, +): Promise { + const text = await readFile(path, 'utf8') + return parseConfig(JSON.parse(text)) +} + +export function rosterLogins(config: ToolsetConfig): Set { + return new Set(config.maintainers.map((m) => m.github.toLowerCase())) +} + +export function isRosterMaintainer( + login: string | null, + config: ToolsetConfig, +): boolean { + return login !== null && rosterLogins(config).has(login.toLowerCase()) +} + +export function isBotLogin( + login: string | null, + config: ToolsetConfig, +): boolean { + if (login === null) return false + const normalized = login.toLowerCase().replace(/\[bot\]$/, '') + return ( + login.toLowerCase().endsWith('[bot]') || + normalized.startsWith('app/') || + config.botAllowlist.some((b) => b.toLowerCase() === normalized) + ) +} diff --git a/scripts/maintainer/discord.ts b/scripts/maintainer/discord.ts new file mode 100644 index 000000000..579340536 --- /dev/null +++ b/scripts/maintainer/discord.ts @@ -0,0 +1,216 @@ +/** + * Renders a Scorecard into Discord webhook messages (embeds) and posts them. + * Rendering is pure; posting takes an injected fetch. + */ + +import { formatDuration } from './metrics' +import type { Scorecard } from './metrics' + +const MAX_LINES_PER_SECTION = 12 +const MAX_EMBED_DESCRIPTION = 4000 +const MAX_EMBEDS_PER_MESSAGE = 10 +const MAX_MESSAGE_CHARS = 5500 + +export interface DiscordEmbed { + title: string + description: string + color: number +} + +interface TriageLike { + item: { number: number; title: string; url: string; assignees: Array } + unansweredHours?: number | null +} + +/** + * Titles are attacker-controlled; escape Discord markdown so a title like + * `[click me](https://evil.com)` renders as literal text, not a masked link. + */ +function escapeMarkdown(text: string): string { + return text.replace(/[\\`*_~|[\]()<>#-]/g, '\\$&') +} + +function shortTitle(title: string): string { + const truncated = title.length > 60 ? `${title.slice(0, 57)}…` : title + return escapeMarkdown(truncated) +} + +function itemLine(t: TriageLike, note?: string): string { + const assignee = + t.item.assignees.length > 0 ? ` · @${t.item.assignees.join(', @')}` : '' + const suffix = note ? ` — ${note}` : '' + return `[#${t.item.number}](${t.item.url}) ${shortTitle(t.item.title)}${suffix}${assignee}` +} + +function section(lines: Array): string { + if (lines.length === 0) return '_Nothing — all clear!_ 🎉' + const shown = lines.slice(0, MAX_LINES_PER_SECTION) + if (lines.length > shown.length) { + shown.push(`…and ${lines.length - shown.length} more`) + } + return shown.join('\n').slice(0, MAX_EMBED_DESCRIPTION) +} + +function mention(github: string, discord: string | null): string { + return discord ? `<@${discord}>` : `**${github}**` +} + +function delta(n: number): string { + if (n > 0) return `(+${n} vs 7d ago)` + if (n < 0) return `(${n} vs 7d ago)` + return '(±0 vs 7d ago)' +} + +export function buildScorecardEmbeds( + scorecard: Scorecard, + repo: string, +): Array { + const s = scorecard.stats + const embeds: Array = [] + + embeds.push({ + title: `🔴 Answer these — waiting on us past SLA (${scorecard.answerThese.length})`, + color: 0xd93f0b, + description: section( + scorecard.answerThese.map((t) => + itemLine(t, `waiting **${formatDuration(t.unansweredHours ?? 0)}**`), + ), + ), + }) + + embeds.push({ + title: `✅ Ready to merge (${scorecard.readyToMerge.length})`, + color: 0x0e8a16, + description: section(scorecard.readyToMerge.map((t) => itemLine(t))), + }) + + const queueLines = scorecard.perMaintainer.map((q) => { + const fresh = + q.freshReplies.length > 0 + ? ` — **${q.freshReplies.length} with fresh replies**: ${q.freshReplies + .slice(0, 4) + .map((t) => `[#${t.item.number}](${t.item.url})`) + .join(' ')}` + : '' + return `${mention(q.github, q.discord)}: ${q.assignedPRs.length} PRs, ${q.assignedIssues.length} issues${fresh}` + }) + embeds.push({ + title: '👀 Your queue', + color: 0x1d76db, + description: section(queueLines), + }) + + const newLines = [ + ...scorecard.newPRs.map((t) => + itemLine( + t, + `new PR${t.firstTimeContributor ? ' · 🍃 first-time contributor' : ''}`, + ), + ), + ...scorecard.newIssues.map((t) => itemLine(t, 'new issue')), + ...scorecard.newDiscussions.map( + (t) => + `[#${t.item.number}](${t.item.url}) ${shortTitle(t.item.title)} — new discussion`, + ), + ] + const flaggedLines = scorecard.flagged.map((t) => + itemLine(t, `⚠️ needs human judgment: ${t.spamReasons.join(', ')}`), + ) + embeds.push({ + title: `🆕 New in the last 24h (${newLines.length})`, + color: 0x5319e7, + description: section([...newLines, ...flaggedLines]), + }) + + const staleLines = scorecard.stale.map((t) => + itemLine(t, 'waiting on author — nudge or close?'), + ) + const discussionLines = scorecard.discussionsNeedingAttention + .slice(0, 6) + .map( + (t) => + `[#${t.item.number}](${t.item.url}) ${shortTitle(t.item.title)} — unanswered **${formatDuration(t.waitingHours)}**`, + ) + embeds.push({ + title: `💤 Going stale (${scorecard.stale.length}) · 💬 Discussions (${scorecard.discussionsNeedingAttention.length})`, + color: 0xcccccc, + description: section([...staleLines, ...discussionLines]), + }) + + const statLines = [ + `Open PRs: **${s.openPRs}** ${delta(s.openPRsDelta7d)} · Open issues: **${s.openIssues}** ${delta(s.openIssuesDelta7d)}`, + `Merged last 24h: **${s.mergedLast24h}** · Issues closed: **${s.issuesClosedLast24h}**`, + s.medianFirstResponseHours === null + ? 'First response (7d): _no samples yet_' + : `First response (7d, n=${s.firstResponseSampleSize}): median **${formatDuration(s.medianFirstResponseHours)}**, p90 **${formatDuration(s.p90FirstResponseHours ?? 0)}** · ${s.awaitingFirstResponse} still waiting`, + `PRs assigned: **${s.pctPRsAssigned}%** · Bot PRs open: ${s.botPRs} · Pending changesets: ${s.pendingChangesets}`, + ] + if (scorecard.clusters.length > 0) { + statLines.push( + `📚 PR clusters: ${scorecard.clusters + .map((c) => `${c.author} (${c.count})`) + .join(', ')} — consider batch-reviewing`, + ) + } + if (scorecard.unassignable.length > 0) { + statLines.push( + `🤷 Unassignable (everyone at cap): ${scorecard.unassignable + .map((t) => `[#${t.item.number}](${t.item.url})`) + .join(' ')}`, + ) + } + embeds.push({ + title: `📊 Scorecard — ${repo}`, + color: 0x0052cc, + description: section(statLines), + }) + + return embeds +} + +/** Split embeds into webhook-sized messages (count + character limits). */ +export function chunkEmbeds( + embeds: Array, +): Array> { + const messages: Array> = [] + let current: Array = [] + let currentChars = 0 + for (const embed of embeds) { + const size = embed.title.length + embed.description.length + if ( + current.length > 0 && + (current.length >= MAX_EMBEDS_PER_MESSAGE || + currentChars + size > MAX_MESSAGE_CHARS) + ) { + messages.push(current) + current = [] + currentChars = 0 + } + current.push(embed) + currentChars += size + } + if (current.length > 0) messages.push(current) + return messages +} + +export async function postToDiscord( + webhookUrl: string, + embeds: Array, + fetchImpl: typeof fetch = fetch, +): Promise { + for (const chunk of chunkEmbeds(embeds)) { + const response = await fetchImpl(webhookUrl, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + embeds: chunk, + allowed_mentions: { parse: ['users'] }, + }), + }) + if (!response.ok) { + throw new Error( + `Discord webhook HTTP ${response.status}: ${(await response.text()).slice(0, 300)}`, + ) + } + } +} diff --git a/scripts/maintainer/env.ts b/scripts/maintainer/env.ts new file mode 100644 index 000000000..c2c1cbc24 --- /dev/null +++ b/scripts/maintainer/env.ts @@ -0,0 +1,35 @@ +/** Shared entry-point plumbing: token resolution and step-summary output. */ + +import { execFile } from 'node:child_process' +import { appendFile } from 'node:fs/promises' +import { promisify } from 'node:util' + +const execFileAsync = promisify(execFile) + +export async function resolveToken(): Promise { + const fromEnv = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN + if (fromEnv) return fromEnv + try { + const { stdout } = await execFileAsync('gh', ['auth', 'token']) + const token = stdout.trim() + if (token) return token + } catch { + // fall through to the error below + } + throw new Error( + 'No GitHub token: set GITHUB_TOKEN, or authenticate the gh CLI (`gh auth login`).', + ) +} + +export function isDryRun(): boolean { + return ( + process.argv.includes('--dry-run') || + ['1', 'true'].includes((process.env.DRY_RUN ?? '').toLowerCase()) + ) +} + +/** Append markdown to the GitHub Actions step summary when running in CI. */ +export async function writeStepSummary(markdown: string): Promise { + const path = process.env.GITHUB_STEP_SUMMARY + if (path) await appendFile(path, `${markdown}\n`) +} diff --git a/scripts/maintainer/fixtures.ts b/scripts/maintainer/fixtures.ts new file mode 100644 index 000000000..9af2af5ed --- /dev/null +++ b/scripts/maintainer/fixtures.ts @@ -0,0 +1,140 @@ +/** Test fixture builders. NOW is the frozen reference time for all tests. */ + +import type { + DiscussionItem, + IssueItem, + PRItem, + TimelineEvent, + ToolsetConfig, +} from './types' + +export const NOW = new Date('2026-07-16T00:00:00Z') + +export function hoursAgo(h: number): string { + return new Date(NOW.getTime() - h * 60 * 60 * 1000).toISOString() +} + +export function daysAgo(d: number): string { + return hoursAgo(d * 24) +} + +export const config: ToolsetConfig = { + repo: 'TanStack/ai', + maintainers: [ + { + github: 'tom', + discord: '111', + areas: ['packages/ai-sandbox*/**', 'packages/ai-claude-code/**'], + maxOpenAssignments: 5, + }, + { + github: 'alem', + discord: null, + areas: ['packages/ai/**', 'packages/ai-client/**'], + maxOpenAssignments: 5, + }, + { + github: 'jack', + discord: null, + areas: ['docs/**', 'examples/**'], + maxOpenAssignments: 5, + }, + ], + sla: { + firstResponseHours: 24, + followUpResponseHours: 48, + staleAuthorDays: 14, + }, + spam: { maxAccountAgeDays: 30, maxChangedLines: 30 }, + botAllowlist: ['renovate'], + maxCommentsPerRun: 25, +} + +export function comment( + actor: string, + at: string, + body = '', + isBot = false, +): TimelineEvent { + return { kind: 'comment', actor, isBot, at, body } +} + +export function review( + actor: string, + at: string, + reviewState: string, +): TimelineEvent { + return { kind: 'review', actor, isBot: false, at, reviewState } +} + +export function commit(actor: string, at: string): TimelineEvent { + return { kind: 'commit', actor, isBot: false, at } +} + +export function makePR(overrides: Partial = {}): PRItem { + const number = overrides.number ?? 100 + return { + type: 'pr', + number, + title: 'feat: add thing', + url: `https://github.com/TanStack/ai/pull/${number}`, + author: 'contributor', + authorIsBot: false, + authorAccountCreatedAt: daysAgo(400), + authorAssociation: 'CONTRIBUTOR', + createdAt: daysAgo(2), + updatedAt: daysAgo(1), + isDraft: false, + mergeable: 'MERGEABLE', + reviewDecision: '', + additions: 120, + deletions: 30, + changedFiles: 4, + files: ['packages/ai/src/core/chat.ts', '.changeset/nice-fix.md'], + labels: [], + assignees: [], + ciState: 'success', + linkedIssues: [50], + timeline: [], + ...overrides, + } +} + +export function makeIssue(overrides: Partial = {}): IssueItem { + return { + type: 'issue', + number: 200, + title: 'bug: chat streaming breaks', + url: 'https://github.com/TanStack/ai/issues/200', + author: 'reporter', + authorIsBot: false, + authorAssociation: 'NONE', + createdAt: daysAgo(2), + updatedAt: daysAgo(1), + body: 'It breaks. Repro: https://stackblitz.com/x', + labels: [], + assignees: [], + timeline: [], + ...overrides, + } +} + +export function makeDiscussion( + overrides: Partial = {}, +): DiscussionItem { + return { + type: 'discussion', + number: 300, + title: 'How do I use tools?', + url: 'https://github.com/TanStack/ai/discussions/300', + author: 'asker', + createdAt: daysAgo(3), + updatedAt: daysAgo(3), + isAnswered: false, + category: 'Q&A', + upvotes: 2, + commentCount: 0, + comments: [], + ...overrides, + } +} diff --git a/scripts/maintainer/github.ts b/scripts/maintainer/github.ts new file mode 100644 index 000000000..a7cdb8997 --- /dev/null +++ b/scripts/maintainer/github.ts @@ -0,0 +1,77 @@ +/** + * Thin GitHub API client. Token and fetch are injected so tests never touch + * the network and callers control auth. + */ + +export interface GitHubClient { + graphql: (query: string, variables: Record) => Promise + rest: ( + method: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE', + path: string, + body?: unknown, + ) => Promise +} + +export interface GitHubClientOptions { + token: string + fetchImpl?: typeof fetch + baseUrl?: string +} + +export function createGitHubClient(options: GitHubClientOptions): GitHubClient { + const fetchImpl = options.fetchImpl ?? fetch + const baseUrl = options.baseUrl ?? 'https://api.github.com' + const headers = { + authorization: `Bearer ${options.token}`, + accept: 'application/vnd.github+json', + 'content-type': 'application/json', + 'user-agent': 'tanstack-ai-maintainer-toolset', + 'x-github-api-version': '2022-11-28', + } + + return { + async graphql( + query: string, + variables: Record, + ): Promise { + const response = await fetchImpl(`${baseUrl}/graphql`, { + method: 'POST', + headers, + body: JSON.stringify({ query, variables }), + }) + const payload: unknown = await response.json() + if (!response.ok) { + throw new Error( + `GitHub GraphQL HTTP ${response.status}: ${JSON.stringify(payload).slice(0, 500)}`, + ) + } + if ( + typeof payload === 'object' && + payload !== null && + 'errors' in payload && + Array.isArray(payload.errors) && + (payload as { errors: Array }).errors.length > 0 + ) { + throw new Error( + `GitHub GraphQL errors: ${JSON.stringify(payload.errors).slice(0, 800)}`, + ) + } + return (payload as { data: T }).data + }, + + async rest(method, path, body): Promise { + const response = await fetchImpl(`${baseUrl}${path}`, { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + }) + const text = await response.text() + if (!response.ok) { + throw new Error( + `GitHub REST ${method} ${path} → HTTP ${response.status}: ${text.slice(0, 500)}`, + ) + } + return text.length > 0 ? JSON.parse(text) : null + }, + } +} diff --git a/scripts/maintainer/glob.ts b/scripts/maintainer/glob.ts new file mode 100644 index 000000000..5803441ed --- /dev/null +++ b/scripts/maintainer/glob.ts @@ -0,0 +1,42 @@ +/** + * Minimal path glob matcher (no dependency): supports `**`, `*`, and `?`. + * `**` spans path segments; `*` and `?` stay within one segment. + */ + +const REGEX_SPECIALS = /[.+^${}()|[\]\\]/g + +export function globToRegExp(glob: string): RegExp { + let out = '^' + let i = 0 + while (i < glob.length) { + const c = glob[i]! + if (c === '*') { + if (glob[i + 1] === '*') { + // `**` — optionally followed by `/`; matches any depth (including none) + i += 2 + if (glob[i] === '/') i++ + out += out === '^' || out.endsWith('/') ? '(?:.*/)?' : '.*' + // when `**` is the trailing token (e.g. `docs/**`) match everything below + if (i >= glob.length) out += '.*' + } else { + out += '[^/]*' + i++ + } + } else if (c === '?') { + out += '[^/]' + i++ + } else { + out += c.replace(REGEX_SPECIALS, '\\$&') + i++ + } + } + return new RegExp(out + '$') +} + +export function matchesGlob(path: string, glob: string): boolean { + return globToRegExp(glob).test(path) +} + +export function matchesAnyGlob(path: string, globs: Array): boolean { + return globs.some((g) => matchesGlob(path, g)) +} diff --git a/scripts/maintainer/metrics.test.ts b/scripts/maintainer/metrics.test.ts new file mode 100644 index 000000000..041433f9b --- /dev/null +++ b/scripts/maintainer/metrics.test.ts @@ -0,0 +1,258 @@ +import { describe, expect, it } from 'vitest' +import { classifyAll } from './classify' +import { buildScorecardEmbeds, chunkEmbeds } from './discord' +import { computeScorecard, formatDuration, percentile } from './metrics' +import { + NOW, + comment, + config, + daysAgo, + hoursAgo, + makeDiscussion, + makeIssue, + makePR, +} from './fixtures' +import type { RepoSnapshot } from './types' + +function snapshotFixture(): RepoSnapshot { + return { + owner: 'TanStack', + repo: 'ai', + takenAt: NOW.toISOString(), + prs: [ + // breached: no response for 3 days + makePR({ number: 1, createdAt: daysAgo(3) }), + // ready to merge, assigned to tom, created 10 days ago + makePR({ + number: 2, + createdAt: daysAgo(10), + reviewDecision: 'APPROVED', + assignees: ['tom'], + timeline: [comment('alem', daysAgo(9), 'lgtm')], + }), + // assigned to tom with a fresh contributor reply + makePR({ + number: 3, + createdAt: daysAgo(10), + assignees: ['tom'], + timeline: [ + comment('tom', daysAgo(5), 'please change x'), + comment('contributor', daysAgo(1), 'done!'), + ], + }), + // new PR in the last 24h from a first-time contributor + makePR({ + number: 4, + author: 'newbie', + createdAt: hoursAgo(5), + authorAssociation: 'FIRST_TIME_CONTRIBUTOR', + }), + // bot PR — excluded from human metrics + makePR({ number: 5, author: 'renovate[bot]', authorIsBot: true }), + // stale: conflicts, author silent 20 days + makePR({ + number: 6, + createdAt: daysAgo(40), + mergeable: 'CONFLICTING', + author: 'clusterguy', + timeline: [ + comment('alem', daysAgo(25), 'rebase please'), + comment('clusterguy', daysAgo(20), 'ok'), + ], + }), + // cluster: same author, three open PRs + makePR({ number: 7, author: 'clusterguy', createdAt: daysAgo(2) }), + makePR({ number: 8, author: 'clusterguy', createdAt: daysAgo(2) }), + ], + issues: [ + makeIssue({ number: 20, createdAt: hoursAgo(50) }), // breached + ], + discussions: [ + makeDiscussion({ number: 30 }), + makeDiscussion({ + number: 31, + comments: [{ actor: 'jack', isBot: false, at: daysAgo(1) }], + }), + ], + recentlyClosed: [ + { + type: 'pr', + number: 90, + title: 'merged yesterday', + url: 'https://github.com/TanStack/ai/pull/90', + author: 'contributor', + authorIsBot: false, + createdAt: daysAgo(3), + closedAt: hoursAgo(10), + mergedAt: hoursAgo(10), + timeline: [comment('alem', daysAgo(2), 'nice')], + }, + { + type: 'issue', + number: 91, + title: 'closed 5 days ago', + url: 'https://github.com/TanStack/ai/issues/91', + author: 'contributor', + authorIsBot: false, + createdAt: daysAgo(12), + closedAt: daysAgo(5), + mergedAt: null, + timeline: [], + }, + ], + } +} + +describe('computeScorecard', () => { + const snapshot = snapshotFixture() + const triage = classifyAll(snapshot, config, NOW) + const scorecard = computeScorecard(snapshot, triage, config, NOW, 3) + + it('surfaces SLA breaches sorted by longest wait', () => { + const numbers = scorecard.answerThese.map((t) => t.item.number) + expect(numbers).toContain(1) + expect(numbers).toContain(20) + const hours = scorecard.answerThese.map((t) => t.unansweredHours ?? 0) + expect(hours).toEqual([...hours].sort((a, b) => b - a)) + }) + + it('finds ready-to-merge PRs', () => { + expect(scorecard.readyToMerge.map((t) => t.item.number)).toEqual([2]) + }) + + it('builds per-maintainer queues with fresh replies', () => { + const tom = scorecard.perMaintainer.find((q) => q.github === 'tom')! + expect(tom.assignedPRs.map((t) => t.item.number).sort()).toEqual([2, 3]) + expect(tom.freshReplies.map((t) => t.item.number)).toContain(3) + }) + + it('lists new items and flags first-time contributors', () => { + expect(scorecard.newPRs.map((t) => t.item.number)).toEqual([4]) + expect(scorecard.newPRs[0]!.firstTimeContributor).toBe(true) + }) + + it('detects stale items and same-author clusters', () => { + expect(scorecard.stale.map((t) => t.item.number)).toContain(6) + expect(scorecard.clusters).toContainEqual({ + author: 'clusterguy', + count: 3, + }) + }) + + it('excludes bots from open counts but reports them', () => { + expect(scorecard.stats.openPRs).toBe(7) + expect(scorecard.stats.botPRs).toBe(1) + }) + + it('computes 7-day open deltas from created/closed timestamps', () => { + // PRs open 7d ago: #1..#3 no (created <7d? #1 3d→no)… recompute: + // created ≤ 7d ago and still open: #2, #3, #6 → 3; closed-after: none + // (PR #90 created 3d ago). Now open: 7 → delta +4. + expect(scorecard.stats.openPRsDelta7d).toBe(4) + // Issues: open now 1 (created 50h ago). 7d ago: #91 was open → 1. Delta 0. + expect(scorecard.stats.openIssuesDelta7d).toBe(0) + }) + + it('counts merges and closes in the last 24h', () => { + expect(scorecard.stats.mergedLast24h).toBe(1) + expect(scorecard.stats.issuesClosedLast24h).toBe(0) + }) + + it('computes first-response stats over items created in the last 7 days', () => { + // Samples: PR #90 closed (24h response). Open items created <7d with a + // response: none. Awaiting: #1, #4, #7, #8, #20. + expect(scorecard.stats.firstResponseSampleSize).toBe(1) + expect(scorecard.stats.medianFirstResponseHours).toBeCloseTo(24, 0) + expect(scorecard.stats.awaitingFirstResponse).toBe(5) + }) + + it('reports unanswered discussions and pending changesets', () => { + expect(scorecard.stats.unansweredDiscussions).toBe(1) + expect(scorecard.stats.pendingChangesets).toBe(3) + }) + + it('marks PRs unassignable only when routing simulation finds everyone at cap', () => { + // Generous caps: everything unassigned routes somewhere. + expect(scorecard.unassignable).toHaveLength(0) + + const tinyCaps = { + ...config, + maintainers: config.maintainers.map((m) => ({ + ...m, + maxOpenAssignments: 1, + })), + } + const capped = computeScorecard(snapshot, triage, tinyCaps, NOW, 3) + expect(capped.unassignable.length).toBeGreaterThan(0) + }) +}) + +describe('helpers', () => { + it('percentile', () => { + expect(percentile([], 50)).toBeNull() + expect(percentile([5], 90)).toBe(5) + expect(percentile([1, 2, 3, 4], 50)).toBe(2) + expect(percentile([1, 2, 3, 4, 100], 90)).toBe(100) + }) + + it('formatDuration', () => { + expect(formatDuration(0.4)).toBe('<1h') + expect(formatDuration(30)).toBe('30h') + expect(formatDuration(72)).toBe('3d') + }) +}) + +describe('discord rendering', () => { + const snapshot = snapshotFixture() + const triage = classifyAll(snapshot, config, NOW) + const scorecard = computeScorecard(snapshot, triage, config, NOW, 3) + + it('builds all six embeds with links and mentions', () => { + const embeds = buildScorecardEmbeds(scorecard, 'TanStack/ai') + expect(embeds).toHaveLength(6) + const all = embeds.map((e) => `${e.title}\n${e.description}`).join('\n') + expect(all).toContain('https://github.com/TanStack/ai/pull/1') + expect(all).toContain('<@111>') // tom's discord mention + expect(all).toContain('first-time contributor') + expect(all).toContain('PR clusters') + }) + + it('escapes markdown in untrusted titles (no masked links)', () => { + const hostile = structuredClone(scorecard) + hostile.readyToMerge = [ + { + ...hostile.readyToMerge[0]!, + item: { + ...hostile.readyToMerge[0]!.item, + title: '[urgent: click](https://evil.com) `x` **y**', + }, + }, + ] + const embeds = buildScorecardEmbeds(hostile, 'TanStack/ai') + const ready = embeds[1]!.description + expect(ready).not.toContain('[urgent: click](https://evil.com)') + expect(ready).toContain('\\[urgent: click\\]\\(https://evil.com\\)') + expect(ready).toContain('\\`x\\` \\*\\*y\\*\\*') + // the real item link is still a working masked link + expect(ready).toMatch(/\[#\d+\]\(https:\/\/github\.com\//) + }) + + it('chunks embeds under Discord message limits', () => { + const big = Array.from({ length: 13 }, (_, i) => ({ + title: `t${i}`, + description: 'x'.repeat(2000), + color: 0, + })) + const chunks = chunkEmbeds(big) + expect(chunks.length).toBeGreaterThan(1) + for (const chunk of chunks) { + expect(chunk.length).toBeLessThanOrEqual(10) + const chars = chunk.reduce( + (sum, e) => sum + e.title.length + e.description.length, + 0, + ) + expect(chars).toBeLessThanOrEqual(5500) + } + expect(chunks.flat()).toHaveLength(13) + }) +}) diff --git a/scripts/maintainer/metrics.ts b/scripts/maintainer/metrics.ts new file mode 100644 index 000000000..bd40068e7 --- /dev/null +++ b/scripts/maintainer/metrics.ts @@ -0,0 +1,277 @@ +/** + * Scorecard computation: turns a snapshot + triage into the daily digest's + * sections and stats. Pure — no I/O. + */ + +import { deriveResponseState } from './classify' +import { computeLoad, routePR } from './route' +import type { + DiscussionTriage, + IssueTriage, + PRTriage, + RepoSnapshot, + ToolsetConfig, + TriageResult, +} from './types' + +const DAY_MS = 24 * 60 * 60 * 1000 + +export interface MaintainerQueue { + github: string + discord: string | null + assignedPRs: Array + assignedIssues: Array + /** Assigned items where the contributor replied after our last response. */ + freshReplies: Array +} + +export interface ScorecardStats { + openPRs: number + openIssues: number + openPRsDelta7d: number + openIssuesDelta7d: number + mergedLast24h: number + issuesClosedLast24h: number + medianFirstResponseHours: number | null + p90FirstResponseHours: number | null + firstResponseSampleSize: number + awaitingFirstResponse: number + pctPRsAssigned: number + unansweredDiscussions: number + botPRs: number + pendingChangesets: number +} + +export interface Scorecard { + /** Waiting on a maintainer past SLA, oldest wait first. */ + answerThese: Array + /** Approved + green + conflict-free. */ + readyToMerge: Array + perMaintainer: Array + newPRs: Array + newIssues: Array + newDiscussions: Array + /** Suspected drive-by/bounty PRs needing a human call before any ack. */ + flagged: Array + /** Waiting on author past the stale threshold — nudge/close candidates. */ + stale: Array + discussionsNeedingAttention: Array + /** Authors with several open PRs — review as a batch. */ + clusters: Array<{ author: string; count: number }> + unassignable: Array + stats: ScorecardStats +} + +export function percentile(values: Array, p: number): number | null { + if (values.length === 0) return null + const sorted = [...values].sort((a, b) => a - b) + const idx = Math.min( + sorted.length - 1, + Math.ceil((p / 100) * sorted.length) - 1, + ) + return sorted[Math.max(0, idx)]! +} + +export function formatDuration(hours: number): string { + if (hours < 1) return '<1h' + if (hours < 48) return `${Math.round(hours)}h` + return `${Math.round(hours / 24)}d` +} + +function openAtCount( + openCreatedAts: Array, + closed: Array<{ createdAt: string; closedAt: string }>, + at: Date, +): number { + const t = at.getTime() + const stillOpen = openCreatedAts.filter( + (c) => new Date(c).getTime() <= t, + ).length + const closedAfter = closed.filter( + (c) => + new Date(c.createdAt).getTime() <= t && + new Date(c.closedAt).getTime() > t, + ).length + return stillOpen + closedAfter +} + +export function computeScorecard( + snapshot: RepoSnapshot, + triage: TriageResult, + config: ToolsetConfig, + now: Date, + pendingChangesets: number, +): Scorecard { + const humanPRs = triage.prs.filter((t) => !t.bot) + const humanIssues = triage.issues.filter((t) => !t.bot) + const dayAgo = new Date(now.getTime() - DAY_MS) + const weekAgo = new Date(now.getTime() - 7 * DAY_MS) + + const answerThese = [...humanPRs, ...humanIssues] + .filter((t) => t.slaBreached && t.waitingOn === 'maintainer') + .sort((a, b) => (b.unansweredHours ?? 0) - (a.unansweredHours ?? 0)) + + const readyToMerge = humanPRs.filter((t) => t.readyToMerge) + + const perMaintainer: Array = config.maintainers.map((m) => { + const mine = (assignees: Array) => + assignees.some((a) => a.toLowerCase() === m.github.toLowerCase()) + const assignedPRs = humanPRs.filter((t) => mine(t.item.assignees)) + const assignedIssues = humanIssues.filter((t) => mine(t.item.assignees)) + return { + github: m.github, + discord: m.discord ?? null, + assignedPRs, + assignedIssues, + freshReplies: [...assignedPRs, ...assignedIssues].filter( + (t) => + ('freshContributorReply' in t + ? t.freshContributorReply + : t.waitingOn === 'maintainer') && t.waitingOn === 'maintainer', + ), + } + }) + + const isNew = (createdAt: string) => new Date(createdAt) >= dayAgo + const newPRs = humanPRs.filter((t) => isNew(t.item.createdAt)) + const newIssues = humanIssues.filter((t) => isNew(t.item.createdAt)) + const newDiscussions = triage.discussions.filter((t) => + isNew(t.item.createdAt), + ) + + const flagged = humanPRs.filter((t) => t.suspectedSpam) + const stale = [...humanPRs, ...humanIssues].filter((t) => t.staleAuthor) + const discussionsNeedingAttention = triage.discussions + .filter((t) => t.needsAttention) + .sort((a, b) => b.waitingHours - a.waitingHours) + + const clusterCounts = new Map() + for (const t of humanPRs) { + if (t.item.author && !t.rosterAuthor) { + clusterCounts.set( + t.item.author, + (clusterCounts.get(t.item.author) ?? 0) + 1, + ) + } + } + const clusters = [...clusterCounts.entries()] + .filter(([, count]) => count >= 3) + .map(([author, count]) => ({ author, count })) + .sort((a, b) => b.count - a.count) + + // Simulate the sweep's routing over unassigned PRs: only the ones that + // still route nowhere (everyone at cap) are genuinely unassignable. + const load = computeLoad(config, [ + ...snapshot.prs.map((pr) => pr.assignees), + ...snapshot.issues.map((issue) => issue.assignees), + ]) + const unassignable: Array = [] + for (const t of humanPRs) { + if (t.item.isDraft || t.suspectedSpam || t.item.assignees.length > 0) { + continue + } + const routed = routePR( + t.item.files, + t.item.author, + config, + load, + t.item.number, + ) + if (routed === null) unassignable.push(t) + else load.set(routed, (load.get(routed) ?? 0) + 1) + } + + // --- Stats --- + const closedPRs = snapshot.recentlyClosed.filter((c) => c.type === 'pr') + const closedIssues = snapshot.recentlyClosed.filter((c) => c.type === 'issue') + + const openPRs = humanPRs.length + const openIssues = humanIssues.length + const openPRsDelta7d = + openPRs - + openAtCount( + humanPRs.map((t) => t.item.createdAt), + closedPRs.filter((c) => !c.authorIsBot), + weekAgo, + ) + const openIssuesDelta7d = + openIssues - + openAtCount( + humanIssues.map((t) => t.item.createdAt), + closedIssues.filter((c) => !c.authorIsBot), + weekAgo, + ) + + const mergedLast24h = closedPRs.filter( + (c) => c.mergedAt !== null && new Date(c.mergedAt) >= dayAgo, + ).length + const issuesClosedLast24h = closedIssues.filter( + (c) => new Date(c.closedAt) >= dayAgo, + ).length + + // First-response distribution over non-bot items created in the last 7 days + // (both still-open and recently-closed). + const responseSamples: Array = [] + let awaitingFirstResponse = 0 + const openRecent = [...humanPRs, ...humanIssues].filter( + (t) => new Date(t.item.createdAt) >= weekAgo && !t.rosterAuthor, + ) + for (const t of openRecent) { + if (t.firstResponseHours !== null) + responseSamples.push(t.firstResponseHours) + else awaitingFirstResponse++ + } + for (const c of snapshot.recentlyClosed) { + if (c.authorIsBot || new Date(c.createdAt) < weekAgo) continue + const state = deriveResponseState( + c.timeline, + c.author, + c.createdAt, + config, + now, + ) + if (state.firstResponseHours !== null) { + responseSamples.push(state.firstResponseHours) + } + } + + const assignablePRs = humanPRs.filter((t) => !t.item.isDraft) + const pctPRsAssigned = + assignablePRs.length === 0 + ? 100 + : Math.round( + (assignablePRs.filter((t) => t.item.assignees.length > 0).length / + assignablePRs.length) * + 100, + ) + + return { + answerThese, + readyToMerge, + perMaintainer, + newPRs, + newIssues, + newDiscussions, + flagged, + stale, + discussionsNeedingAttention, + clusters, + unassignable, + stats: { + openPRs, + openIssues, + openPRsDelta7d, + openIssuesDelta7d, + mergedLast24h, + issuesClosedLast24h, + medianFirstResponseHours: percentile(responseSamples, 50), + p90FirstResponseHours: percentile(responseSamples, 90), + firstResponseSampleSize: responseSamples.length, + awaitingFirstResponse, + pctPRsAssigned, + unansweredDiscussions: discussionsNeedingAttention.length, + botPRs: triage.prs.length - humanPRs.length, + pendingChangesets, + }, + } +} diff --git a/scripts/maintainer/route.test.ts b/scripts/maintainer/route.test.ts new file mode 100644 index 000000000..977a52b81 --- /dev/null +++ b/scripts/maintainer/route.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from 'vitest' +import { globToRegExp, matchesGlob } from './glob' +import { computeLoad, routeIssue, routePR } from './route' +import { config } from './fixtures' + +describe('glob matcher', () => { + it.each([ + ['packages/ai/src/core/chat.ts', 'packages/ai/**', true], + ['packages/ai-client/src/index.ts', 'packages/ai/**', false], + ['packages/ai-sandbox-docker/src/x.ts', 'packages/ai-sandbox*/**', true], + ['packages/ai-sandbox/src/x.ts', 'packages/ai-sandbox*/**', true], + ['packages/ai-solid/src/x.ts', 'packages/ai-sandbox*/**', false], + ['.changeset/two-dogs.md', '.changeset/*.md', true], + ['.changeset/nested/two-dogs.md', '.changeset/*.md', false], + ['testing/e2e/tests/chat.spec.ts', 'testing/e2e/**', true], + ['docs/a/b/c.md', 'docs/**', true], + ['packages/ai/src/core/chat.ts', 'packages/*/src/**', true], + ['packages/ai/package.json', 'packages/*/src/**', false], + ['deep/path/file.ts', '**/file.ts', true], + ])('%s vs %s → %s', (path, glob, expected) => { + expect(matchesGlob(path, glob)).toBe(expected) + }) + + it('anchors patterns (no substring matches)', () => { + expect(globToRegExp('docs/**').test('notdocs/a.md')).toBe(false) + }) +}) + +describe('routePR', () => { + const noLoad = computeLoad(config, []) + + it('routes by area ownership', () => { + expect( + routePR(['packages/ai-sandbox/src/x.ts'], 'someone', config, noLoad, 1), + ).toBe('tom') + expect(routePR(['docs/guide.md'], 'someone', config, noLoad, 1)).toBe( + 'jack', + ) + }) + + it('never assigns the author to their own PR', () => { + expect( + routePR(['packages/ai-sandbox/src/x.ts'], 'tom', config, noLoad, 1), + ).not.toBe('tom') + }) + + it('falls back to least-loaded rotation when no area matches', () => { + const load = computeLoad(config, [['tom'], ['tom'], ['alem']]) + expect(routePR(['README.md'], 'someone', config, load, 7)).toBe('jack') + }) + + it('rotates deterministically among equally-loaded candidates', () => { + const a = routePR(['README.md'], 'someone', config, noLoad, 1) + const b = routePR(['README.md'], 'someone', config, noLoad, 2) + expect(a).not.toBe(b) + }) + + it('respects assignment caps and returns null when everyone is full', () => { + const full = computeLoad( + config, + Array.from({ length: 5 }, () => ['tom', 'alem', 'jack']), + ) + expect(routePR(['docs/x.md'], 'someone', config, full, 1)).toBeNull() + }) +}) + +describe('routeIssue', () => { + const noLoad = computeLoad(config, []) + + it('routes by package name mentioned in text', () => { + expect( + routeIssue( + 'bug in @tanstack/ai-sandbox process spawn', + 'someone', + config, + noLoad, + 1, + ), + ).toBe('tom') + }) + + it('matches bare package tokens with word boundaries', () => { + expect( + routeIssue('ai-client reconnect loop', 'someone', config, noLoad, 1), + ).toBe('alem') + }) + + it('falls back to rotation when nothing matches', () => { + expect( + routeIssue('something vague', 'someone', config, noLoad, 3), + ).not.toBeNull() + }) +}) diff --git a/scripts/maintainer/route.ts b/scripts/maintainer/route.ts new file mode 100644 index 000000000..0d4e81bcc --- /dev/null +++ b/scripts/maintainer/route.ts @@ -0,0 +1,128 @@ +/** + * Assignment routing: area-glob ownership first, then least-loaded with a + * deterministic rotation (keyed by item number) so ties spread out without + * needing any stored state. + */ + +import { matchesAnyGlob } from './glob' +import type { MaintainerEntry, ToolsetConfig } from './types' + +export type AssignmentLoad = Map + +/** Current open-assignment count per roster maintainer (PRs + issues). */ +export function computeLoad( + config: ToolsetConfig, + assigneeLists: Array>, +): AssignmentLoad { + const load: AssignmentLoad = new Map( + config.maintainers.map((m) => [m.github, 0]), + ) + for (const assignees of assigneeLists) { + for (const login of assignees) { + const match = config.maintainers.find( + (m) => m.github.toLowerCase() === login.toLowerCase(), + ) + if (match) load.set(match.github, (load.get(match.github) ?? 0) + 1) + } + } + return load +} + +function pick( + candidates: Array, + load: AssignmentLoad, + seed: number, +): string | null { + if (candidates.length === 0) return null + const minLoad = Math.min(...candidates.map((m) => load.get(m.github) ?? 0)) + const leastLoaded = candidates.filter( + (m) => (load.get(m.github) ?? 0) === minLoad, + ) + return leastLoaded[Math.abs(seed) % leastLoaded.length]!.github +} + +function eligible( + config: ToolsetConfig, + author: string | null, + load: AssignmentLoad, +): Array { + return config.maintainers.filter( + (m) => + m.github.toLowerCase() !== author?.toLowerCase() && + (load.get(m.github) ?? 0) < (m.maxOpenAssignments ?? 10), + ) +} + +/** + * Route a PR by its changed files. Prefers the maintainer whose area globs + * match the most files; falls back to least-loaded rotation. Returns null + * when everyone eligible is at their assignment cap. + */ +export function routePR( + files: Array, + author: string | null, + config: ToolsetConfig, + load: AssignmentLoad, + seed: number, +): string | null { + const candidates = eligible(config, author, load) + if (candidates.length === 0) return null + const scored = candidates.map((m) => ({ + m, + score: files.filter((f) => matchesAnyGlob(f, m.areas)).length, + })) + const best = Math.max(...scored.map((s) => s.score)) + if (best > 0) { + return pick( + scored.filter((s) => s.score === best).map((s) => s.m), + load, + seed, + ) + } + return pick(candidates, load, seed) +} + +// Extract package-name tokens from a maintainer's area globs, e.g. +// `packages/ai-sandbox*/**` → `ai-sandbox`. +function areaTokens(entry: MaintainerEntry): Array { + const tokens: Array = [] + for (const area of entry.areas) { + const match = /^packages\/([\w-]+)/.exec(area) + if (match) tokens.push(match[1]!.replace(/\*+$/, '')) + } + return tokens +} + +/** + * Route an issue by matching package names mentioned in its title/body + * against maintainer areas; falls back to least-loaded rotation. + */ +export function routeIssue( + text: string, + author: string | null, + config: ToolsetConfig, + load: AssignmentLoad, + seed: number, +): string | null { + const candidates = eligible(config, author, load) + if (candidates.length === 0) return null + const lower = text.toLowerCase() + const scored = candidates.map((m) => ({ + m, + score: areaTokens(m).filter((token) => { + // `@tanstack/ai` must not match inside `@tanstack/ai-sandbox` + if (new RegExp(`@tanstack/${token}(?![\\w-])`).test(lower)) return true + if (token.length <= 3) return false + return new RegExp(`\\b${token.replace(/-/g, '[-\\s]')}\\b`).test(lower) + }).length, + })) + const best = Math.max(...scored.map((s) => s.score)) + if (best > 0) { + return pick( + scored.filter((s) => s.score === best).map((s) => s.m), + load, + seed, + ) + } + return pick(candidates, load, seed) +} diff --git a/scripts/maintainer/scorecard.ts b/scripts/maintainer/scorecard.ts new file mode 100644 index 000000000..1180d5459 --- /dev/null +++ b/scripts/maintainer/scorecard.ts @@ -0,0 +1,71 @@ +/** + * Daily scorecard — computes the maintainer to-do digest and posts it to a + * Discord webhook (DISCORD_WEBHOOK_URL). With --dry-run (or when the webhook + * is not configured) it prints the payload instead of posting. + * + * Usage: tsx scripts/maintainer/scorecard.ts [--dry-run] + */ + +import { readdir } from 'node:fs/promises' +import process from 'node:process' +import { classifyAll } from './classify' +import { collectSnapshot } from './collect' +import { loadConfig } from './config' +import { buildScorecardEmbeds, postToDiscord } from './discord' +import { createGitHubClient } from './github' +import { computeScorecard } from './metrics' +import { isDryRun, resolveToken, writeStepSummary } from './env' + +async function countPendingChangesets(): Promise { + try { + const entries = await readdir('.changeset') + return entries.filter( + (f) => f.endsWith('.md') && f.toLowerCase() !== 'readme.md', + ).length + } catch { + return 0 + } +} + +async function main(): Promise { + const dryRun = isDryRun() + const config = await loadConfig() + const client = createGitHubClient({ token: await resolveToken() }) + + console.log(`Collecting snapshot of ${config.repo}…`) + const now = new Date() + const snapshot = await collectSnapshot(client, config, now) + const triage = classifyAll(snapshot, config, now) + const scorecard = computeScorecard( + snapshot, + triage, + config, + now, + await countPendingChangesets(), + ) + const embeds = buildScorecardEmbeds(scorecard, config.repo) + + const summary = embeds + .map((e) => `### ${e.title}\n\n${e.description}`) + .join('\n\n') + await writeStepSummary(`## Daily scorecard\n\n${summary}`) + + const webhookUrl = process.env.DISCORD_WEBHOOK_URL + if (dryRun || !webhookUrl) { + if (!webhookUrl && !dryRun) { + console.warn( + 'DISCORD_WEBHOOK_URL is not set — printing the digest instead of posting.', + ) + } + console.log(JSON.stringify(embeds, null, 2)) + return + } + + await postToDiscord(webhookUrl, embeds) + console.log(`Posted ${embeds.length} embed(s) to Discord.`) +} + +main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/scripts/maintainer/sweep.test.ts b/scripts/maintainer/sweep.test.ts new file mode 100644 index 000000000..4c3709fbe --- /dev/null +++ b/scripts/maintainer/sweep.test.ts @@ -0,0 +1,214 @@ +import { describe, expect, it } from 'vitest' +import { planLabelChanges } from './actions' +import { ACK_MARKER, REPRO_MARKER, buildAckComment } from './comments' +import { classifyPR } from './classify' +import { planSweep } from './sweep' +import { + NOW, + comment, + config, + daysAgo, + hoursAgo, + makeIssue, + makePR, +} from './fixtures' +import type { RepoSnapshot } from './types' + +function makeSnapshot(overrides: Partial = {}): RepoSnapshot { + return { + owner: 'TanStack', + repo: 'ai', + takenAt: NOW.toISOString(), + prs: [], + issues: [], + discussions: [], + recentlyClosed: [], + ...overrides, + } +} + +describe('planSweep', () => { + it('assigns, acks, and labels a fresh unassigned PR', () => { + const snapshot = makeSnapshot({ + prs: [makePR({ files: ['packages/ai-sandbox/src/x.ts'] })], + }) + const plan = planSweep(snapshot, config) + const kinds = plan.mutations.map((m) => m.kind) + expect(kinds).toContain('assign') + expect(kinds).toContain('comment') + expect(kinds).toContain('add-labels') + + const assign = plan.mutations.find((m) => m.kind === 'assign') + expect(assign).toMatchObject({ number: 100, assignee: 'tom' }) + const ack = plan.mutations.find((m) => m.kind === 'comment') + expect(ack && 'body' in ack && ack.body).toContain(ACK_MARKER) + expect(ack && 'body' in ack && ack.body).toContain('@tom') + }) + + it('is idempotent: assigned + already-acked + correctly-labeled PR → no mutations', () => { + const snapshot = makeSnapshot({ + prs: [ + makePR({ + assignees: ['alem'], + labels: ['waiting-on: maintainer'], + timeline: [ + comment( + 'github-actions[bot]', + daysAgo(1), + `hi\n${ACK_MARKER}`, + true, + ), + ], + }), + ], + }) + expect(planSweep(snapshot, config).mutations).toHaveLength(0) + }) + + it('leaves suspected-spam PRs completely untouched', () => { + const snapshot = makeSnapshot({ + prs: [ + makePR({ + authorAccountCreatedAt: daysAgo(3), + additions: 1, + deletions: 1, + linkedIssues: [], + authorAssociation: 'NONE', + }), + ], + }) + const plan = planSweep(snapshot, config) + expect(plan.mutations).toHaveLength(0) + expect(plan.skippedAsSpam).toEqual([100]) + }) + + it('skips drafts and bot PRs', () => { + const snapshot = makeSnapshot({ + prs: [ + makePR({ number: 1, isDraft: true }), + makePR({ number: 2, author: 'renovate[bot]', authorIsBot: true }), + ], + }) + expect(planSweep(snapshot, config).mutations).toHaveLength(0) + }) + + it('assigns roster-authored PRs to someone else without acking', () => { + const snapshot = makeSnapshot({ + prs: [ + makePR({ + author: 'tom', + authorAssociation: 'COLLABORATOR', + files: ['packages/ai-sandbox/src/x.ts'], + }), + ], + }) + const plan = planSweep(snapshot, config) + const assign = plan.mutations.find((m) => m.kind === 'assign') + expect(assign && 'assignee' in assign && assign.assignee).not.toBe('tom') + expect(plan.mutations.some((m) => m.kind === 'comment')).toBe(false) + }) + + it('asks for a repro once, labels needs-repro and has-pr', () => { + const snapshot = makeSnapshot({ + prs: [makePR({ linkedIssues: [200], assignees: ['alem'] })], + issues: [ + makeIssue({ body: 'no links or code here', assignees: ['alem'] }), + ], + }) + const plan = planSweep(snapshot, config) + const reproComment = plan.mutations.find( + (m) => m.kind === 'comment' && m.body.includes(REPRO_MARKER), + ) + expect(reproComment).toBeDefined() + const labels = plan.mutations.find( + (m) => m.kind === 'add-labels' && m.number === 200, + ) + expect(labels && 'labels' in labels && labels.labels).toEqual( + expect.arrayContaining(['needs-repro', 'has-pr']), + ) + + // second sweep: repro comment already present → no repeat + const acked = makeSnapshot({ + prs: snapshot.prs, + issues: [ + makeIssue({ + body: 'no links or code here', + assignees: ['alem'], + labels: ['needs-repro', 'has-pr', 'waiting-on: maintainer'], + timeline: [ + comment( + 'github-actions[bot]', + hoursAgo(3), + `x\n${REPRO_MARKER}`, + true, + ), + ], + }), + ], + }) + const second = planSweep(acked, config) + expect( + second.mutations.filter((m) => m.kind === 'comment' && m.number === 200), + ).toHaveLength(0) + }) + + it('caps comments per run and reports the overflow', () => { + const tinyCap = { ...config, maxCommentsPerRun: 2 } + const snapshot = makeSnapshot({ + prs: [1, 2, 3, 4].map((n) => makePR({ number: n, assignees: ['alem'] })), + }) + const plan = planSweep(snapshot, tinyCap) + expect(plan.mutations.filter((m) => m.kind === 'comment')).toHaveLength(2) + expect(plan.commentsSuppressedByCap).toBe(2) + }) + + it('spreads fallback assignments across maintainers', () => { + const snapshot = makeSnapshot({ + prs: [11, 12, 13].map((n) => makePR({ number: n, files: ['README.md'] })), + }) + const plan = planSweep(snapshot, config) + const assignees = plan.mutations + .filter((m) => m.kind === 'assign') + .map((m) => (m as { assignee: string }).assignee) + expect(new Set(assignees).size).toBeGreaterThan(1) + }) +}) + +describe('planLabelChanges', () => { + it('only touches managed labels', () => { + const mutations = planLabelChanges( + 1, + ['bug', 'waiting-on: author', 'help wanted'], + ['waiting-on: maintainer'], + ) + expect(mutations).toEqual([ + { kind: 'add-labels', number: 1, labels: ['waiting-on: maintainer'] }, + { kind: 'remove-label', number: 1, label: 'waiting-on: author' }, + ]) + }) + + it('produces nothing when labels already match', () => { + expect( + planLabelChanges(1, ['bug', 'ready-to-merge'], ['ready-to-merge']), + ).toEqual([]) + }) +}) + +describe('buildAckComment', () => { + it('renders warnings for missing changeset/E2E and conflicts', () => { + const t = classifyPR( + makePR({ + files: ['packages/ai/src/core/chat.ts'], + mergeable: 'CONFLICTING', + }), + config, + NOW, + ) + const body = buildAckComment(t, 'alem') + expect(body).toContain('⚠️ No changeset found') + expect(body).toContain('⚠️ No E2E test changes') + expect(body).toContain('⚠️ Merge conflicts') + expect(body).toContain('✅ CI passing') + expect(body).toContain(ACK_MARKER) + }) +}) diff --git a/scripts/maintainer/sweep.ts b/scripts/maintainer/sweep.ts new file mode 100644 index 000000000..4c4f8dd50 --- /dev/null +++ b/scripts/maintainer/sweep.ts @@ -0,0 +1,208 @@ +/** + * Maintainer sweep — runs on a schedule (every few hours). For every open PR + * and issue it: + * - routes and assigns an owner (area globs → least-loaded rotation) + * - posts a one-time ack comment with a deterministic pre-review checklist + * - reconciles `waiting-on:*` / `ready-to-merge` / `needs-repro` / `has-pr` + * labels so triage state is visible in the GitHub UI + * - asks for a reproduction on bug reports that lack one + * + * Suspected drive-by/bounty PRs are left untouched (no ack, no assignment) — + * they surface in the daily Discord digest for a human call instead. + * + * Read/write via the GitHub API only; PR code is never checked out or + * executed. Idempotent: safe to re-run at any time. + * + * Usage: tsx scripts/maintainer/sweep.ts [--dry-run] + */ + +import process from 'node:process' +import { pathToFileURL } from 'node:url' +import { + describeMutation, + ensureManagedLabels, + executeMutations, + planLabelChanges, +} from './actions' +import { classifyAll } from './classify' +import { collectSnapshot } from './collect' +import { buildAckComment, buildReproComment } from './comments' +import { loadConfig } from './config' +import { createGitHubClient } from './github' +import { computeLoad, routeIssue, routePR } from './route' +import { isDryRun, resolveToken, writeStepSummary } from './env' +import type { Mutation } from './actions' +import type { + IssueTriage, + PRTriage, + RepoSnapshot, + ToolsetConfig, +} from './types' + +export interface SweepPlan { + mutations: Array + skippedAsSpam: Array + commentsSuppressedByCap: number +} + +export function planSweep( + snapshot: RepoSnapshot, + config: ToolsetConfig, +): SweepPlan { + const triage = classifyAll(snapshot, config) + const load = computeLoad(config, [ + ...snapshot.prs.map((pr) => pr.assignees), + ...snapshot.issues.map((issue) => issue.assignees), + ]) + + const mutations: Array = [] + const skippedAsSpam: Array = [] + let comments = 0 + let commentsSuppressedByCap = 0 + const addComment = (m: Mutation & { kind: 'comment' }) => { + if (comments < config.maxCommentsPerRun) { + mutations.push(m) + comments++ + } else { + commentsSuppressedByCap++ + } + } + + for (const t of triage.prs) { + const pr = t.item + if (t.bot) continue + if (t.suspectedSpam) { + skippedAsSpam.push(pr.number) + continue + } + if (pr.isDraft) continue + + let assignee = pr.assignees[0] ?? null + if (assignee === null) { + const routed = routePR(pr.files, pr.author, config, load, pr.number) + if (routed !== null) { + mutations.push({ kind: 'assign', number: pr.number, assignee: routed }) + load.set(routed, (load.get(routed) ?? 0) + 1) + t.suggestedAssignee = routed + assignee = routed + } + } + + if (!t.hasAckComment && !t.rosterAuthor && assignee !== null) { + addComment({ + kind: 'comment', + number: pr.number, + body: buildAckComment(t, assignee), + note: 'ack + pre-review checklist', + }) + } + + const desired: Array = [] + if (t.readyToMerge) desired.push('ready-to-merge') + if (t.waitingOn === 'maintainer') desired.push('waiting-on: maintainer') + if (t.waitingOn === 'author') desired.push('waiting-on: author') + mutations.push(...planLabelChanges(pr.number, pr.labels, desired)) + } + + for (const t of triage.issues) { + const issue = t.item + if (t.bot) continue + + if (issue.assignees.length === 0) { + const routed = routeIssue( + `${issue.title}\n${issue.body}`, + issue.author, + config, + load, + issue.number, + ) + if (routed !== null) { + mutations.push({ + kind: 'assign', + number: issue.number, + assignee: routed, + }) + load.set(routed, (load.get(routed) ?? 0) + 1) + t.suggestedAssignee = routed + } + } + + if (t.needsRepro && !t.hasReproComment && !t.rosterAuthor) { + addComment({ + kind: 'comment', + number: issue.number, + body: buildReproComment(issue.author), + note: 'reproduction request', + }) + } + + const desired: Array = [] + if (t.waitingOn === 'maintainer') desired.push('waiting-on: maintainer') + if (t.waitingOn === 'author') desired.push('waiting-on: author') + if (t.needsRepro) desired.push('needs-repro') + if (t.hasOpenPR) desired.push('has-pr') + mutations.push(...planLabelChanges(issue.number, issue.labels, desired)) + } + + return { mutations, skippedAsSpam, commentsSuppressedByCap } +} + +function summarize(plan: SweepPlan, dryRun: boolean): string { + const lines = [ + `## Maintainer sweep ${dryRun ? '(dry run)' : ''}`, + '', + `Planned mutations: **${plan.mutations.length}**`, + '', + ...plan.mutations.map((m) => `- ${describeMutation(m)}`), + ] + if (plan.skippedAsSpam.length > 0) { + lines.push( + '', + `⚠️ Left untouched pending human judgment (suspected drive-by): ${plan.skippedAsSpam.map((n) => `#${n}`).join(', ')}`, + ) + } + if (plan.commentsSuppressedByCap > 0) { + lines.push( + '', + `${plan.commentsSuppressedByCap} comment(s) deferred to the next run by the per-run cap.`, + ) + } + return lines.join('\n') +} + +async function main(): Promise { + const dryRun = isDryRun() + const config = await loadConfig() + const client = createGitHubClient({ token: await resolveToken() }) + + console.log(`Collecting snapshot of ${config.repo}…`) + const snapshot = await collectSnapshot(client, config) + console.log( + `Open: ${snapshot.prs.length} PRs, ${snapshot.issues.length} issues, ${snapshot.discussions.length} discussions.`, + ) + + const plan = planSweep(snapshot, config) + const summary = summarize(plan, dryRun) + console.log(summary) + await writeStepSummary(summary) + + if (dryRun) { + console.log('\nDry run — nothing executed.') + return + } + + await ensureManagedLabels(client, config.repo) + await executeMutations(client, config.repo, plan.mutations) + console.log(`Executed ${plan.mutations.length} mutation(s).`) +} + +const isDirectRun = + process.argv[1] !== undefined && + import.meta.url === pathToFileURL(process.argv[1]).href + +if (isDirectRun) { + main().catch((error) => { + console.error(error) + process.exitCode = 1 + }) +} diff --git a/scripts/maintainer/types.ts b/scripts/maintainer/types.ts new file mode 100644 index 000000000..71f97729f --- /dev/null +++ b/scripts/maintainer/types.ts @@ -0,0 +1,221 @@ +/** + * Shared types for the maintainer toolset (sweep + scorecard). + * + * The data layer (collect.ts) normalizes GitHub GraphQL responses into these + * shapes; everything downstream (classify.ts, route.ts, metrics.ts) is pure + * and operates only on these types, so it can be unit-tested with fixtures. + */ + +export type CIState = 'success' | 'failure' | 'pending' | 'unknown' + +export interface TimelineEvent { + kind: 'comment' | 'review' | 'commit' + /** GitHub login, or null when the account was deleted / unresolvable. */ + actor: string | null + isBot: boolean + /** ISO timestamp. */ + at: string + /** Only present for comments; used for bot-marker detection. */ + body?: string + /** Only present for reviews: APPROVED | CHANGES_REQUESTED | COMMENTED | DISMISSED. */ + reviewState?: string +} + +export interface PRItem { + type: 'pr' + number: number + title: string + url: string + author: string | null + authorIsBot: boolean + /** ISO timestamp of the author account creation, when resolvable. */ + authorAccountCreatedAt: string | null + authorAssociation: string + createdAt: string + updatedAt: string + isDraft: boolean + /** MERGEABLE | CONFLICTING | UNKNOWN */ + mergeable: string + /** '' | APPROVED | CHANGES_REQUESTED | REVIEW_REQUIRED */ + reviewDecision: string + additions: number + deletions: number + changedFiles: number + /** Paths of changed files (capped at 100 by the API query). */ + files: Array + labels: Array + assignees: Array + ciState: CIState + /** Issue numbers this PR closes (via closingIssuesReferences). */ + linkedIssues: Array + timeline: Array +} + +export interface IssueItem { + type: 'issue' + number: number + title: string + url: string + author: string | null + authorIsBot: boolean + authorAssociation: string + createdAt: string + updatedAt: string + body: string + labels: Array + assignees: Array + timeline: Array +} + +export interface DiscussionItem { + type: 'discussion' + number: number + title: string + url: string + author: string | null + createdAt: string + updatedAt: string + isAnswered: boolean + category: string + upvotes: number + commentCount: number + /** Most recent comments (login + timestamp), newest last. */ + comments: Array<{ actor: string | null; isBot: boolean; at: string }> +} + +/** Minimal record of a recently closed PR/issue, for scorecard trend metrics. */ +export interface ClosedItem { + type: 'pr' | 'issue' + number: number + title: string + url: string + author: string | null + authorIsBot: boolean + createdAt: string + closedAt: string + /** Only for PRs; null when closed without merging. */ + mergedAt: string | null + /** Comment/review events, enough to compute first-response time. */ + timeline: Array +} + +export interface RepoSnapshot { + owner: string + repo: string + /** ISO timestamp the snapshot was taken. */ + takenAt: string + prs: Array + issues: Array + discussions: Array + /** PRs and issues closed within the lookback window (14 days). */ + recentlyClosed: Array +} + +// --- Configuration (.github/maintainers.json) --- + +export interface MaintainerEntry { + github: string + /** Discord user id (snowflake) for @-mentions in the digest; null → plain text. */ + discord?: string | null + /** File globs this maintainer owns; used to route PR assignment. */ + areas: Array + /** Routing skips this maintainer once they have this many open assignments. */ + maxOpenAssignments?: number +} + +export interface SlaConfig { + /** Hours before an item with no maintainer response ever counts as breached. */ + firstResponseHours: number + /** Hours to answer a follow-up message on an already-touched item. */ + followUpResponseHours: number + /** Days of author silence before a waiting-on-author item counts as stale. */ + staleAuthorDays: number +} + +export interface SpamConfig { + /** Author accounts younger than this are eligible for the spam flag. */ + maxAccountAgeDays: number + /** Diffs at or under this many changed lines are eligible for the spam flag. */ + maxChangedLines: number +} + +export interface ToolsetConfig { + /** owner/repo, e.g. "TanStack/ai". */ + repo: string + maintainers: Array + sla: SlaConfig + spam: SpamConfig + /** Logins (without [bot] suffix) always treated as bots, e.g. "renovate". */ + botAllowlist: Array + /** Safety cap on comments posted per sweep run. */ + maxCommentsPerRun: number +} + +// --- Classification output --- + +export type WaitingOn = 'maintainer' | 'author' | 'nobody' + +export interface PRTriage { + item: PRItem + bot: boolean + rosterAuthor: boolean + firstTimeContributor: boolean + suspectedSpam: boolean + spamReasons: Array + waitingOn: WaitingOn + waitingReason: string + readyToMerge: boolean + hasConflicts: boolean + ciFailing: boolean + missingChangeset: boolean + missingE2E: boolean + /** Last comment/review by a roster maintainer who isn't the PR author. */ + lastMaintainerResponseAt: string | null + /** Last non-bot comment/commit by the author or a third party. */ + lastContributorActivityAt: string | null + /** Oldest contributor message (or PR creation) still awaiting a maintainer reply. */ + unansweredSince: string | null + /** Hours the oldest unanswered message has been waiting. */ + unansweredHours: number | null + slaBreached: boolean + /** Hours from creation to first maintainer response; null if none yet. */ + firstResponseHours: number | null + /** Contributor activity newer than the last maintainer response. */ + freshContributorReply: boolean + staleAuthor: boolean + hasAckComment: boolean + /** Set by the sweep when the PR is unassigned and routable. */ + suggestedAssignee: string | null +} + +export interface IssueTriage { + item: IssueItem + bot: boolean + rosterAuthor: boolean + waitingOn: WaitingOn + needsRepro: boolean + hasOpenPR: boolean + lastMaintainerResponseAt: string | null + lastContributorActivityAt: string | null + unansweredSince: string | null + unansweredHours: number | null + slaBreached: boolean + firstResponseHours: number | null + staleAuthor: boolean + hasReproComment: boolean + suggestedAssignee: string | null +} + +export interface DiscussionTriage { + item: DiscussionItem + /** True when unanswered and no roster maintainer has commented. */ + needsAttention: boolean + /** Hours since the last comment (or creation) on a needs-attention discussion. */ + waitingHours: number +} + +export interface TriageResult { + prs: Array + issues: Array + discussions: Array +}