diff --git a/.github/workflows/flaky_fix_sync.yml b/.github/workflows/flaky_fix_sync.yml new file mode 100644 index 000000000000..75cde5d3dcfb --- /dev/null +++ b/.github/workflows/flaky_fix_sync.yml @@ -0,0 +1,93 @@ +name: Flaky Fix Backport Check + +on: + workflow_dispatch: + inputs: + since_days: + description: 'How many days back to scan upstream master (default: 7)' + required: false + type: string + default: '7' + pattern: + description: 'Commit subject pattern to match (case-insensitive)' + required: false + type: string + default: 'fix flaky test' + upstream_ref: + description: 'Upstream branch/ref to scan' + required: false + type: string + default: 'master' + create_pr: + description: 'Cherry-pick missing commits and open a backport PR' + required: false + type: boolean + default: false + + schedule: + - cron: '0 12 * * 1' # Every Monday at 08:00 EDT (12:00 UTC) + +jobs: + check: + runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Check for missing flaky-fix backports + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 tests/ci/check_for_flaky_fixes.py \ + --since-days "${{ inputs.since_days || '7' }}" \ + --pattern "${{ inputs.pattern || 'fix flaky test' }}" \ + --upstream-ref "${{ inputs.upstream_ref || 'master' }}" \ + --md-output flaky_fix_report.md \ + --json-output flaky_fix.json + + - name: Write summary + if: always() + run: cat flaky_fix_report.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload check results + uses: actions/upload-artifact@v4 + with: + name: flaky-fix-check + path: | + flaky_fix_report.md + flaky_fix.json + + backport: + needs: check + if: ${{ github.event_name == 'schedule' || inputs.create_pr == true }} + runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Download check results + uses: actions/download-artifact@v4 + with: + name: flaky-fix-check + + - name: Cherry-pick and open PR + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 tests/ci/backport_flaky_fixes.py \ + --json-file flaky_fix.json \ + --repo "${{ github.repository }}" diff --git a/tests/ci/backport_flaky_fixes.py b/tests/ci/backport_flaky_fixes.py new file mode 100755 index 000000000000..3fd528494d2f --- /dev/null +++ b/tests/ci/backport_flaky_fixes.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +""" +Read the JSON output from check_for_flaky_fix_backports.py, cherry-pick each +missing commit onto a new branch, push it, and open a single PR. +""" + +import argparse +import json +import subprocess +import sys +from datetime import datetime, timezone + + +UPSTREAM_GIT_URL = "https://github.com/{repo}.git" +UPSTREAM_COMMIT_URL = "https://github.com/{repo}/commit/{sha}" + + +def run_git(*args, check=True, capture=True) -> subprocess.CompletedProcess: + result = subprocess.run( + ["git"] + list(args), + text=True, + capture_output=capture, + ) + if check and result.returncode != 0: + print(f"git {' '.join(args)} failed:\n{result.stderr}", file=sys.stderr) + sys.exit(1) + return result + + +def git_out(*args) -> str: + return run_git(*args).stdout.strip() + + +def fetch_commit(upstream_repo: str, sha: str) -> bool: + """Fetch a single commit object by SHA from the upstream repo. + Returns True on success, False on failure.""" + url = UPSTREAM_GIT_URL.format(repo=upstream_repo) + result = run_git("fetch", "--no-tags", "--no-recurse-submodules", url, sha, check=False) + if result.returncode != 0: + print(f"Failed to fetch {sha[:12]} from upstream:\n{result.stderr}", file=sys.stderr) + return False + return True + + +def cherry_pick(sha: str) -> bool: + """Attempt cherry-pick. Returns True on success, False on conflict.""" + result = run_git("cherry-pick", "-x", sha, check=False) + if result.returncode == 0: + return True + print(f"Cherry-pick of {sha[:12]} failed (conflict), aborting.", file=sys.stderr) + run_git("cherry-pick", "--abort", check=False) + return False + + +def build_pr_body( + upstream_repo: str, + applied: list, + conflicted: list, + fetch_failed: list, +) -> str: + lines = [] + lines.append("Automated backport of upstream flaky-fix commits.") + lines.append("") + lines.append("- [x] Exclude all Regression") + lines.append("") + + if applied: + lines.append("### Cherry-picked") + lines.append("") + for sha, subject, date in applied: + url = UPSTREAM_COMMIT_URL.format(repo=upstream_repo, sha=sha) + lines.append(f"- [`{sha[:12]}`]({url}) {subject} _(committed {date})_") + lines.append("") + + if conflicted: + lines.append("### Skipped (cherry-pick conflict — manual backport needed)") + lines.append("") + for sha, subject, date in conflicted: + url = UPSTREAM_COMMIT_URL.format(repo=upstream_repo, sha=sha) + lines.append(f"- [`{sha[:12]}`]({url}) {subject} _(committed {date})_") + lines.append("") + + if fetch_failed: + lines.append("### Skipped (fetch failed — commit may not be reachable)") + lines.append("") + for sha, subject, date in fetch_failed: + url = UPSTREAM_COMMIT_URL.format(repo=upstream_repo, sha=sha) + lines.append(f"- [`{sha[:12]}`]({url}) {subject} _(committed {date})_") + lines.append("") + + return "\n".join(lines) + + +def create_pr(repo: str, branch: str, base: str, title: str, body: str, dry_run: bool) -> None: + if dry_run: + print(f"DRY RUN: would open PR in {repo}:", file=sys.stderr) + print(f" title: {title}", file=sys.stderr) + print(f" base: {base}", file=sys.stderr) + print(f" head: {branch}", file=sys.stderr) + return + + result = subprocess.run( + [ + "gh", "pr", "create", + "--repo", repo, + "--base", base, + "--head", branch, + "--title", title, + "--body", body, + ], + text=True, + capture_output=False, + ) + if result.returncode != 0: + print("gh pr create failed", file=sys.stderr) + sys.exit(1) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Cherry-pick missing flaky-fix commits and open a backport PR.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--json-file", + required=True, + metavar="PATH", + help="JSON output file produced by check_flaky_fix_backports.py", + ) + parser.add_argument( + "--repo", + default="Altinity/ClickHouse", + metavar="OWNER/REPO", + help="target GitHub repository where the PR will be opened", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="fetch and cherry-pick locally but do not push or open a PR", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + with open(args.json_file, encoding="utf-8") as f: + data = json.load(f) + + missing = data.get("missing", []) + if not missing: + print("No missing commits in JSON file. Nothing to do.", file=sys.stderr) + return + + upstream_repo = data["upstream_repo"] + base_branch = data["branch"] + + # Sort oldest-first so cherry-picks apply in chronological order. + missing.sort(key=lambda c: c["date"]) + + date_tag = datetime.now(timezone.utc).strftime("%Y-%m-%d") + backport_branch = f"flaky-fix-backport/{base_branch}/{date_tag}" + + existing = run_git("branch", "--list", backport_branch).stdout.strip() + if existing: + print(f"Branch {backport_branch} already exists. Delete it first.", file=sys.stderr) + sys.exit(1) + + run_git("checkout", "-b", backport_branch) + + applied = [] + conflicted = [] + fetch_failed = [] + + for commit in missing: + sha = commit["sha"] + subject = commit["subject"] + date = commit["date"] + print(f"Fetching {sha[:12]}: {subject}", file=sys.stderr) + if not fetch_commit(upstream_repo, sha): + fetch_failed.append((sha, subject, date)) + print(f" Skipped {sha[:12]} (fetch failed)", file=sys.stderr) + continue + if cherry_pick(sha): + applied.append((sha, subject, date)) + print(f" Applied {sha[:12]}", file=sys.stderr) + else: + conflicted.append((sha, subject, date)) + print(f" Skipped {sha[:12]} (conflict)", file=sys.stderr) + + if not applied: + print("No commits could be applied (all conflicted or failed to fetch). Cleaning up.", file=sys.stderr) + run_git("checkout", base_branch) + run_git("branch", "-D", backport_branch) + sys.exit(0) + + pr_title = f"Backport flaky-fix commits from upstream ({date_tag})" + pr_body = build_pr_body(upstream_repo, applied, conflicted, fetch_failed) + print(f"\n--- PR title ---\n{pr_title}\n--- PR body ---\n{pr_body}---") + + if args.dry_run: + print( + f"DRY RUN: would push {backport_branch} and open PR against {base_branch}.", + file=sys.stderr, + ) + print(f" Applied: {len(applied)} Conflicted: {len(conflicted)} Fetch failed: {len(fetch_failed)}", file=sys.stderr) + run_git("checkout", base_branch) + run_git("branch", "-D", backport_branch) + return + + run_git("push", "origin", backport_branch, capture=False) + + create_pr( + repo=args.repo, + branch=backport_branch, + base=base_branch, + title=pr_title, + body=pr_body, + dry_run=False, + ) + + +if __name__ == "__main__": + main() diff --git a/tests/ci/check_flaky_fixes.py b/tests/ci/check_flaky_fixes.py new file mode 100755 index 000000000000..f3f9da129c19 --- /dev/null +++ b/tests/ci/check_flaky_fixes.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +""" +Check upstream master for recent "fixed flaky test" commits and report +which ones are not yet ported to the current branch. + +Uses the GitHub API to list upstream commits (avoids shallow-fetch issues +with the ClickHouse repo), then checks each candidate against the local +git log by subject line. +""" + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request +from datetime import datetime, timedelta, timezone + + +UPSTREAM_BASE_URL = "https://github.com/{repo}/commit/{sha}" +GITHUB_API = "https://api.github.com" + + +def since_iso(days: int) -> str: + cutoff = datetime.now(timezone.utc) - timedelta(days=days) + return cutoff.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def github_commits(repo: str, ref: str, since: str) -> list: + """Fetch all commits on repo/ref since the given ISO timestamp, paginating as needed.""" + token = os.environ.get("GITHUB_TOKEN") + headers = {"User-Agent": "check_flaky_fix_backports"} + if token: + headers["Authorization"] = f"Bearer {token}" + else: + print("Warning: GITHUB_TOKEN not set; using unauthenticated API (60 req/hour limit)", file=sys.stderr) + + commits = [] + url = ( + f"{GITHUB_API}/repos/{repo}/commits" + f"?sha={ref}&since={since}&per_page=100" + ) + while url: + print(f"Fetching {url} ...", file=sys.stderr) + req = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(req) as resp: + commits.extend(json.loads(resp.read())) + link = resp.headers.get("Link", "") + url = None + for part in link.split(","): + if 'rel="next"' in part: + url = part.split(";")[0].strip().strip("<>") + except urllib.error.HTTPError as e: + print(f"GitHub API error: {e}", file=sys.stderr) + sys.exit(1) + return commits + + +def flaky_commits(commits: list, pattern: str) -> list: + """Filter to commits whose subject matches the pattern (case-insensitive). + Returns list of (sha, subject, date) tuples.""" + results = [] + for c in commits: + subject = c["commit"]["message"].splitlines()[0] + if pattern.lower() in subject.lower(): + date = c["commit"]["committer"]["date"] + results.append((c["sha"], subject, date)) + return results + + +def run_git(*args) -> str: + import subprocess + result = subprocess.run( + ["git"] + list(args), text=True, capture_output=True + ) + if result.returncode != 0: + print(f"git {' '.join(args)} failed:\n{result.stderr}", file=sys.stderr) + sys.exit(1) + return result.stdout.strip() + + +def is_ported(subject: str) -> bool: + """Return True if a commit with this subject exists on the current branch.""" + out = run_git("log", "--fixed-strings", f"--grep={subject}", "--format=%H", "HEAD") + return bool(out) + + +def missing_commits(candidates: list) -> list: + return [(sha, subj, date) for sha, subj, date in candidates if not is_ported(subj)] + + +def current_branch() -> str: + branch = run_git("rev-parse", "--abbrev-ref", "HEAD") + return branch if branch != "HEAD" else run_git("rev-parse", "HEAD")[:12] + + +def render_markdown( + branch: str, + upstream_repo: str, + upstream_ref: str, + days: int, + pattern: str, + all_upstream: list, + missing: list, +) -> str: + lines = [] + lines.append("## Flaky Fix Backport Check") + lines.append("") + lines.append(f"**Branch:** `{branch}` ") + lines.append(f"**Upstream:** `{upstream_repo}` @ `{upstream_ref}` ") + lines.append(f"**Window:** last {days} day(s) ") + lines.append(f"**Pattern:** `{pattern}` ") + lines.append("") + lines.append(f"Upstream commits matching pattern: **{len(all_upstream)}** ") + lines.append(f"Missing from current branch: **{len(missing)}** ") + lines.append("") + + if not missing: + lines.append(":white_check_mark: No missing flaky-fix backports found.") + else: + lines.append(":warning: The following upstream fixes are not yet ported:") + lines.append("") + for sha, subject, date in missing: + url = UPSTREAM_BASE_URL.format(repo=upstream_repo, sha=sha) + short = sha[:12] + lines.append(f"- [`{short}`]({url}) {subject} _(committed {date})_") + + lines.append("") + return "\n".join(lines) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Report upstream flaky-fix commits not yet ported to the current branch.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--since-days", + type=int, + default=7, + metavar="N", + help="how many days back to scan upstream", + ) + parser.add_argument( + "--pattern", + default="fix flaky test", + help="case-insensitive substring to match in commit subjects", + ) + parser.add_argument( + "--upstream-repo", + default="ClickHouse/ClickHouse", + metavar="OWNER/REPO", + help="upstream GitHub repository", + ) + parser.add_argument( + "--upstream-ref", + default="master", + help="upstream branch or ref to scan", + ) + parser.add_argument( + "--md-output", + metavar="PATH", + help="write markdown report to this file (default: stdout)", + ) + parser.add_argument( + "--json-output", + metavar="PATH", + help="write machine-readable JSON report to this file", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + cutoff = since_iso(args.since_days) + branch = current_branch() + + commits = github_commits(args.upstream_repo, args.upstream_ref, cutoff) + all_upstream = flaky_commits(commits, args.pattern) + missing = missing_commits(all_upstream) + + report = render_markdown( + branch=branch, + upstream_repo=args.upstream_repo, + upstream_ref=args.upstream_ref, + days=args.since_days, + pattern=args.pattern, + all_upstream=all_upstream, + missing=missing, + ) + + if args.md_output: + with open(args.md_output, "w", encoding="utf-8") as f: + f.write(report) + print(f"Markdown report written to {args.md_output}", file=sys.stderr) + else: + print(report) + + if args.json_output: + payload = { + "branch": branch, + "upstream_repo": args.upstream_repo, + "upstream_ref": args.upstream_ref, + "missing": [ + {"sha": sha, "subject": subj, "date": date} + for sha, subj, date in missing + ], + } + with open(args.json_output, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + print(f"JSON report written to {args.json_output}", file=sys.stderr) + + if missing: + print(f"\n{len(missing)} missing flaky-fix commit(s) found.", file=sys.stderr) + + +if __name__ == "__main__": + main()