diff --git a/.github/workflows/stale-discovered.yml b/.github/workflows/stale-discovered.yml new file mode 100644 index 0000000000..cbfae8a5f0 --- /dev/null +++ b/.github/workflows/stale-discovered.yml @@ -0,0 +1,62 @@ +# Auto-discovered pull requests whose test failed long ago and that nobody looked +# at, in three steps, one per pull request and run (code/stale-discovered.sh): +# re-test (labeled retested-stale); if it still fails, ping the application's +# author once (labeled pinged-author); if nobody answers in 60 days, close +# (labeled auto-closed; discover-apps.yml proposes the repository again when it +# publishes a new release). A pull request with a human comment is never +# touched. Manual only, a dry run by default. Nothing from any pull request is run. + +name: Stale auto-discovered pull requests + +on: + workflow_dispatch: + inputs: + limit: + description: 'Maximum number of pull requests to re-test, ping or close in this run (oldest first)' + type: number + default: 20 + days: + description: 'A test result older than this many days counts as stale (step 1, re-test)' + type: number + default: 90 + close_days: + description: 'Close this many days after the ping went unanswered (step 3)' + type: number + default: 60 + dry_run: + description: 'Dry run (only list what would be re-tested, pinged or closed)' + type: boolean + default: true + +jobs: + stale: + if: github.repository == 'AppImage/appimage.github.io' + runs-on: ubuntu-latest + timeout-minutes: 90 + concurrency: + group: stale-discovered + cancel-in-progress: false + permissions: + contents: read # the entry's link, to find the author to mention + issues: write + pull-requests: write + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PAT: ${{ secrets.SCREENSHOT_UPLOAD_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + sparse-checkout: code + + - name: Re-test, ping, close + env: + LIMIT: ${{ github.event.inputs.limit || '20' }} + DAYS: ${{ github.event.inputs.days || '90' }} + CLOSE_DAYS: ${{ github.event.inputs.close_days || '60' }} + DRY_RUN: ${{ github.event.inputs.dry_run }} + run: | + ARGS=(--limit "$LIMIT" --days "$DAYS" --close-days "$CLOSE_DAYS") + [ "$DRY_RUN" != "false" ] && ARGS+=(--dry-run) + bash code/stale-discovered.sh "${ARGS[@]}" diff --git a/AGENTS.md b/AGENTS.md index 050b19b7b0..ed493be670 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -169,6 +169,24 @@ GitHub Pages (Jekyll) from this repository. ``). A maintainer who reopens it on purpose is respected. Only the list of changed files is read, via the API. + `.github/workflows/stale-discovered.yml` (`workflow_dispatch` only; inputs + `limit` (default 20), `days` (default 90), `close_days` (default 60), `dry_run` + (default true); `code/stale-discovered.sh`) walks auto-discovered PRs whose test + failed and that no human commented on up a ladder, one step per PR and run: + (1) a last result older than `days` → re-test (`retest-pr.sh`, with + `SCREENSHOT_UPLOAD_TOKEN`), label `retested-stale`; (2) the re-test's result is + in and still failed → one comment that @mentions the GitHub account the AppImage + comes from (`upstream-owner.sh`; never a maintainer or bot), asks for `/retest` + when fixed and announces the closing, label `pinged-author` (marker + ``); (3) `close_days` after the ping with no human + comment → closed with a comment, label `auto-closed` (the other two removed). + A PR that passes the re-test keeps `screenshot-ok` and waits for a maintainer. + `discover-apps.sh` treats an `auto-closed` PR like a repository checked without + success on its closing day (proposed again when pushed to later), not like one + rejected by a maintainer. API budget: the comments of a PR are only fetched + when it is on the ladder or had no activity (`updated_at`) for `days`; the run + stops with exit 4 when fewer than `RESERVE` (200) requests of the hour are left, + and prints how many it used (see "GitHub API budget" below). `.github/workflows/health-report.yml` (weekly, Monday, and `workflow_dispatch`; `code/health-report.sh`) writes one issue, updated in place (marker ``, label `health-report`): what needs a diff --git a/code/discover-apps.sh b/code/discover-apps.sh index 4a22df1c0a..3f947e8d03 100755 --- a/code/discover-apps.sh +++ b/code/discover-apps.sh @@ -224,13 +224,24 @@ while : ; do PAGE_JSON=$(api GET "repos/$REPO/issues?labels=$LABEL&state=all&per_page=100&page=$LABELED_PAGE") N_THIS_PAGE=$(jq 'if type == "array" then length else 0 end' <<<"$PAGE_JSON" 2>/dev/null || echo 0) [ "${N_THIS_PAGE:-0}" -gt 0 ] || break - jq -r '.[] | select(.pull_request) | .body // ""' <<<"$PAGE_JSON" 2>/dev/null \ + # A pull request closed by close-stale-discovered.yml (label auto-closed) was + # not rejected by a maintainer: its repository counts as checked without + # success on the day it was closed (proposed again when it is pushed to later), + # not as proposed before + jq -r '.[] | select(.pull_request) | select([.labels[]?.name] | index("auto-closed") | not) | .body // ""' <<<"$PAGE_JSON" 2>/dev/null \ | sed -n 's/^Repository: *\(https:\/\/github\.com\/[^ ]*\).*/\1/p' | tr -d '\r' \ | while read -r LINE ; do parse_owner_repo "$LINE" ; done >> "$WORKDIR/labeled.txt" + jq -r '.[] | select(.pull_request) | select([.labels[]?.name] | index("auto-closed")) | [(.closed_at // .updated_at // "")[0:10], (.body // "")] | @tsv' <<<"$PAGE_JSON" 2>/dev/null \ + | while IFS=$'\t' read -r DAY BODY ; do + OR=$(sed -n 's/^Repository: *\(https:\/\/github\.com\/[^ ]*\).*/\1/p' <<<"$BODY" | head -n 1 | tr -d '\r') + OR=$(parse_owner_repo "$OR") && printf '%s\t%s\n' "$OR" "$DAY" + done >> "$WORKDIR/autoclosed.tsv" [ "$N_THIS_PAGE" -lt 100 ] && break LABELED_PAGE=$((LABELED_PAGE + 1)) done [ -f "$WORKDIR/labeled.txt" ] && while read -r OR ; do [ -n "$OR" ] && SKIP_LABELED["$OR"]=1 ; done < "$WORKDIR/labeled.txt" +# (the auto-closed ones join the recently-checked below, with their closing day) +[ -f "$WORKDIR/autoclosed.tsv" ] && echo "$(grep -c . "$WORKDIR/autoclosed.tsv") repositories were auto-closed before (proposed again when pushed to)" >&2 echo "$(sort -u "$WORKDIR/labeled.txt" 2>/dev/null | grep -c .) repositories were proposed before (label $LABEL)" >&2 # --- 3b. Apps a maintainer opted out (label opt-out on any PR) ------------- @@ -323,6 +334,15 @@ done > "$WORKDIR/recent.txt" # that, at least a week later, is checked again (it may publish an AppImage now) declare -A SKIP_RECENT_ON=() while IFS=$'\t' read -r OR DATE ; do [ -n "$OR" ] && { SKIP_RECENT["$OR"]=1 ; SKIP_RECENT_ON["$OR"]="$DATE" ; } ; done < "$WORKDIR/recent.txt" +# Auto-closed pull requests: like checked without success on the closing day +# (skipped for 90 days, unless the repository is pushed to in the meantime) +if [ -f "$WORKDIR/autoclosed.tsv" ] ; then + while IFS=$'\t' read -r OR DATE ; do + [ -n "$OR" ] || continue + [ -n "${SKIP_KNOWN[$OR]:-}${SKIP_LABELED[$OR]:-}" ] && continue + if [[ "$DATE" > "$CUTOFF" ]] ; then SKIP_RECENT["$OR"]=1 ; SKIP_RECENT_ON["$OR"]="$DATE" ; fi + done < "$WORKDIR/autoclosed.tsv" +fi # --- 9. Persist the state file ------------------------------------------- # Called after every opened PR, every 10 slices and at the end, so that a diff --git a/code/stale-discovered.sh b/code/stale-discovered.sh new file mode 100755 index 0000000000..40144e168b --- /dev/null +++ b/code/stale-discovered.sh @@ -0,0 +1,209 @@ +#!/bin/bash + +# Handle the auto-discovered pull requests whose test failed long ago and that +# nobody looked at: re-test, then ping the application's author, then (much +# later) close. Used by .github/workflows/stale-discovered.yml (workflow_dispatch +# only; a dry run by default); runnable by hand. +# +# Usage: stale-discovered.sh [--dry-run] [--limit N] [--days D] [--close-days C] +# --dry-run Only print what would be done. +# --limit N Handle at most N pull requests per run (default 20; a +# re-test costs a Test run, a ping is a mention). +# --days D A test result older than D days counts as old (default 90). +# --close-days C Close C days after the ping went unanswered (default 60). +# +# The ladder, one step per pull request and run, each step on a fresh result: +# 1. Re-test: an open pull request labeled auto-discovered with an error-* +# label, no comment by anyone but github-actions, none of the labels +# below, and a last test result older than D days: tested once more +# (closed and reopened with PAT, as /retest does: current workflow files, +# newest release), labeled retested-stale. +# 2. Ping: labeled retested-stale, the re-test's result is in and still +# failed: one comment that @mentions the GitHub account the AppImage +# comes from (code/upstream-owner.sh on the entry's link; the first test +# result mentioned them once already, months ago), says what fails (the +# result above), asks for /retest when fixed, and says that the pull +# request is closed in C days otherwise; labeled pinged-author. +# 3. Close: labeled pinged-author, the ping is C days old, still no comment +# by a human: closed with a comment, labeled auto-closed (the other two +# labels removed). discover-apps.sh proposes the repository again when it +# is pushed to later. +# A pull request with a comment by anyone but github-actions is never touched +# (the author or a maintainer is looking at it); one that passes the re-test +# keeps screenshot-ok and waits for a maintainer (the health report lists it). +# +# Needs GH_TOKEN; PAT (SCREENSHOT_UPLOAD_TOKEN) for step 1; GITHUB_REPOSITORY. +# +# API budget: the list of open pull requests costs one request per 100; the +# comments are fetched only for pull requests on the ladder or without any +# activity in D days (updated_at from the list), one request each; every +# action costs 2-5. The run stops (exit 4) when fewer than RESERVE (default +# 200) requests of the hourly quota are left, and prints how many it used. + +set -uo pipefail + +REPO="${GITHUB_REPOSITORY:-AppImage/appimage.github.io}" +DRY_RUN=false +LIMIT=20 +DAYS=90 +CLOSE_DAYS=60 +while [ $# -gt 0 ] ; do + case "$1" in + --dry-run) DRY_RUN=true ;; + --limit) LIMIT="$2" ; shift ;; + --limit=*) LIMIT="${1#*=}" ;; + --days) DAYS="$2" ; shift ;; + --days=*) DAYS="${1#*=}" ;; + --close-days) CLOSE_DAYS="$2" ; shift ;; + --close-days=*) CLOSE_DAYS="${1#*=}" ;; + *) echo "Unknown argument: $1" >&2 ; exit 2 ;; + esac + shift +done +[[ "$LIMIT" =~ ^[0-9]+$ ]] && [ "$LIMIT" -ge 1 ] || LIMIT=20 +[[ "$DAYS" =~ ^[0-9]+$ ]] || DAYS=90 +[[ "$CLOSE_DAYS" =~ ^[0-9]+$ ]] || CLOSE_DAYS=60 +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +WORKDIR=$(mktemp -d) +trap 'rm -rf "$WORKDIR"' EXIT +iso_ago() { date -u -d "$1 days ago" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-"$1"d +%Y-%m-%dT%H:%M:%SZ ; } +CUTOFF=$(iso_ago "$DAYS") +CLOSE_CUTOFF=$(iso_ago "$CLOSE_DAYS") +MARKER='' +PING_MARKER='' +L_RETESTED=retested-stale +L_PINGED=pinged-author +L_CLOSED=auto-closed + +# Every request goes through api(): it counts them and, every 50 requests, +# stops the run when fewer than RESERVE requests of the hourly quota are left +# (GITHUB_TOKEN has 1000 an hour per repository, a personal token 5000). +RESERVE=${RESERVE:-200} +N_REQUESTS=0 +api() { + N_REQUESTS=$((N_REQUESTS + 1)) + if [ $((N_REQUESTS % 50)) -eq 0 ] ; then + LEFT=$(gh api rate_limit --jq .resources.core.remaining 2>/dev/null || echo 0) + if [ "${LEFT:-0}" -lt "$RESERVE" ] ; then + echo "Only $LEFT API requests left this hour (reserve $RESERVE); stopping after $N_REQUESTS requests" >&2 + exit 4 + fi + fi + api "$@" +} +ensure_label() { api "repos/$REPO/labels/$1" >/dev/null 2>&1 || api -X POST "repos/$REPO/labels" -f name="$1" -f color="$2" -f description="$3" >/dev/null ; } +lower() { tr 'A-Z' 'a-z' <<<"$1" ; } + +# Open auto-discovered pull requests with an error label, oldest first +: > "$WORKDIR/prs.jsonl" +PAGE=1 +while [ "$PAGE" -le 100 ] ; do + api "repos/$REPO/pulls?state=open&per_page=100&page=$PAGE&sort=created&direction=asc" > "$WORKDIR/page.json" 2>/dev/null || break + [ "$(jq 'length' "$WORKDIR/page.json" 2>/dev/null || echo 0)" -gt 0 ] || break + jq -c '.[] | select([.labels[].name] | index("auto-discovered")) | select([.labels[].name | select(startswith("error-"))] | length > 0) + | select([.labels[].name] | index("screenshot-ok") | not) | select([.labels[].name] | index("manual-check-needed") | not) + | {n:.number, sha:.head.sha, user:.user.login, updated:.updated_at, labels:[.labels[].name]}' "$WORKDIR/page.json" >> "$WORKDIR/prs.jsonl" + PAGE=$((PAGE + 1)) +done +echo "$(wc -l < "$WORKDIR/prs.jsonl") open auto-discovered pull requests with a failed test" >&2 + +N_RETESTED=0 ; N_PINGED=0 ; N_CLOSED=0 ; N_HUMAN=0 ; N_FRESH=0 ; N_WAIT=0 +label_time() { # label_time PR LABEL -> when it was last applied + api --paginate "repos/$REPO/issues/$1/events?per_page=100" --jq '.[] | select(.event == "labeled" and .label.name == "'"$2"'") | .created_at' 2>/dev/null | tail -n 1 +} +while IFS= read -r ROW ; do + [ $((N_RETESTED + N_PINGED + N_CLOSED)) -lt "$LIMIT" ] || break + N=$(jq -r .n <<<"$ROW") + has_label() { jq -e --arg l "$1" '.labels | index($l)' <<<"$ROW" >/dev/null ; } + # Cheap first: a pull request not yet on the ladder whose last activity (a + # comment, a label, a push) is newer than D days cannot have a result older + # than that, so its comments are not fetched at all. Thousands of pull + # requests fail this test; the comments of the few hundred that pass it are + # one request each. + if ! has_label "$L_PINGED" && ! has_label "$L_RETESTED" && [[ "$(jq -r .updated <<<"$ROW")" > "$CUTOFF" ]] ; then + N_FRESH=$((N_FRESH + 1)) ; continue + fi + api --paginate "repos/$REPO/issues/$N/comments?per_page=100" --jq '.[] | [.user.login, .created_at, (.body | contains("'"$MARKER"'")), (.body | contains("'"$PING_MARKER"'"))] | @tsv' > "$WORKDIR/c.tsv" 2>/dev/null || continue + if awk -F'\t' '$1 != "github-actions[bot]" { f = 1 } END { exit !f }' "$WORKDIR/c.tsv" ; then + N_HUMAN=$((N_HUMAN + 1)) ; continue + fi + LAST_RESULT=$(awk -F'\t' '$3 == "true" { t = $2 } END { print t }' "$WORKDIR/c.tsv") + PING_AT=$(awk -F'\t' '$4 == "true" { t = $2 } END { print t }' "$WORKDIR/c.tsv") + + if has_label "$L_PINGED" ; then + # Step 3: close, once the ping is old enough and still unanswered + [ -n "$PING_AT" ] || PING_AT=$(label_time "$N" "$L_PINGED") + if [ -z "$PING_AT" ] || [[ "$PING_AT" > "$CLOSE_CUTOFF" ]] ; then N_WAIT=$((N_WAIT + 1)) ; continue ; fi + if [ "$DRY_RUN" = true ] ; then + echo "#$N: would close (pinged on ${PING_AT%%T*}, no answer)" + else + ensure_label "$L_CLOSED" ededed "Auto-discovered, failed its test for months, author pinged, no answer: closed; proposed again when the repository publishes a new release" + { + echo "" + echo "No answer in $CLOSE_DAYS days, so this automatically opened pull request is closed to keep the queue manageable. Nothing is lost: the application is proposed again automatically once its repository publishes a new release, and a pull request that adds \`data/\` is welcome at any time. Thank you!" + } > "$WORKDIR/c.md" + api -X POST "repos/$REPO/issues/$N/comments" -F body=@"$WORKDIR/c.md" >/dev/null + api -X POST "repos/$REPO/issues/$N/labels" -f "labels[]=$L_CLOSED" >/dev/null + api -X DELETE "repos/$REPO/issues/$N/labels/$L_PINGED" >/dev/null 2>&1 || true + api -X DELETE "repos/$REPO/issues/$N/labels/$L_RETESTED" >/dev/null 2>&1 || true + api -X PATCH "repos/$REPO/pulls/$N" -f state=closed >/dev/null + echo "#$N: closed" + fi + N_CLOSED=$((N_CLOSED + 1)) + elif has_label "$L_RETESTED" ; then + # Step 2: ping, once the re-test's result is in and still failed + LABELED_AT=$(label_time "$N" "$L_RETESTED") + if [ -z "$LAST_RESULT" ] || [ -z "$LABELED_AT" ] || [[ "$LAST_RESULT" < "$LABELED_AT" ]] ; then N_WAIT=$((N_WAIT + 1)) ; continue ; fi + # The GitHub account the AppImage comes from (never the bot, never a maintainer) + SHA=$(jq -r .sha <<<"$ROW") + FILE=$(api --paginate "repos/$REPO/pulls/$N/files" --jq '.[] | select(.filename | startswith("data/")) | .filename' 2>/dev/null | head -n 1) + OWNER="" + if [ -n "$FILE" ] ; then + URL=$(api -H "Accept: application/vnd.github.raw" "repos/$REPO/contents/$FILE?ref=$SHA" 2>/dev/null | head -n 1 | tr -d '\r') + OWNER=$(bash "$SCRIPT_DIR/upstream-owner.sh" "$URL" 2>/dev/null | tr -cd 'A-Za-z0-9-') + fi + case "$(lower "$OWNER")" in ""|*"[bot]"|probonopd|github-actions) OWNER="" ;; esac + if [ "$DRY_RUN" = true ] ; then + echo "#$N: would ping ${OWNER:+@$OWNER }(re-tested on ${LAST_RESULT%%T*}, still failing)" + else + ensure_label "$L_PINGED" fef2c0 "Auto-discovered, still failing after a re-test; the application's author was asked (stale-discovered.yml); closed if unanswered" + { + echo "$PING_MARKER" + echo "${OWNER:+@$OWNER: }This pull request was opened automatically to add the application to the [AppImage catalog](https://appimage.github.io), and we would like to list it, but its AppImage does not pass the test (the result above says what fails and how to fix it; the test was just run again on the newest release). Could you have a look? Once a fixed release is out, comment \`/retest\` here and it is tested again. If we do not hear anything in $CLOSE_DAYS days, this pull request is closed to keep the queue manageable; the application is proposed again automatically when the repository publishes a new release. Thank you!" + } > "$WORKDIR/c.md" + api -X POST "repos/$REPO/issues/$N/comments" -F body=@"$WORKDIR/c.md" >/dev/null + api -X POST "repos/$REPO/issues/$N/labels" -f "labels[]=$L_PINGED" >/dev/null + echo "#$N: pinged ${OWNER:+@$OWNER}" + sleep 3 + fi + N_PINGED=$((N_PINGED + 1)) + else + # Step 1: re-test, if the last result is old + if [ -z "$LAST_RESULT" ] || [[ "$LAST_RESULT" > "$CUTOFF" ]] ; then N_FRESH=$((N_FRESH + 1)) ; continue ; fi + if [ "$DRY_RUN" = true ] ; then + echo "#$N: would re-test (last result ${LAST_RESULT%%T*}) and label $L_RETESTED" + else + [ -n "${PAT:-}" ] || { echo "PAT is needed to re-test; stopping" >&2 ; break ; } + ensure_label "$L_RETESTED" fef2c0 "Auto-discovered, failed months ago, tested once more (stale-discovered.yml); the author is pinged if it still fails" + if PAT="$PAT" REPO="$REPO" bash "$SCRIPT_DIR/retest-pr.sh" "$N" >/dev/null 2>&1 ; then + api -X POST "repos/$REPO/issues/$N/labels" -f "labels[]=$L_RETESTED" >/dev/null + echo "#$N: re-tested (closed and reopened), labeled $L_RETESTED" + else + echo "#$N: could not re-test; skipped" >&2 ; continue + fi + sleep 3 + fi + N_RETESTED=$((N_RETESTED + 1)) + fi +done < "$WORKDIR/prs.jsonl" + +{ + echo "## Stale auto-discovered pull requests" + echo + W=$([ "$DRY_RUN" = true ] && echo " (would)") + echo "Re-tested$W: $N_RETESTED; pinged$W: $N_PINGED; closed$W: $N_CLOSED (limit $LIMIT; results older than $DAYS days; closed $CLOSE_DAYS days after an unanswered ping)." + echo "Left alone: $N_HUMAN with a comment by a human, $N_FRESH with a result (or other activity) newer than $DAYS days, $N_WAIT waiting (for a re-test result, or for the $CLOSE_DAYS days after the ping)." + echo "API requests: $N_REQUESTS." + [ "$DRY_RUN" = true ] && echo && echo "(dry run: nothing was changed)" +} | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}" +exit 0