From f4be4330b71e85b9c73cc97d2d2c08bfb046f430 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Mon, 20 Jul 2026 22:14:39 -0400 Subject: [PATCH 1/3] New workflow to identify flaky test fixes --- .../workflows/flaky_fix_backport_check.yml | 49 +++++ tests/ci/check_flaky_fix_backports.py | 193 ++++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 .github/workflows/flaky_fix_backport_check.yml create mode 100755 tests/ci/check_flaky_fix_backports.py diff --git a/.github/workflows/flaky_fix_backport_check.yml b/.github/workflows/flaky_fix_backport_check.yml new file mode 100644 index 000000000000..04659adedfa9 --- /dev/null +++ b/.github/workflows/flaky_fix_backport_check.yml @@ -0,0 +1,49 @@ +name: Flaky Fix Backport Check + +on: + workflow_dispatch: + inputs: + since_days: + description: 'How many days back to scan upstream master (default: 7)' + required: false + type: string + default: '7' + pattern: + description: 'Commit subject pattern to match (case-insensitive)' + required: false + type: string + default: 'fix flaky test' + upstream_ref: + description: 'Upstream branch/ref to scan' + required: false + type: string + default: 'master' + + schedule: + - cron: '0 12 * * 1' # Every Monday at 08:00 EDT (12:00 UTC) + +jobs: + check: + runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Check for missing flaky-fix backports + run: | + python3 tests/ci/check_flaky_fix_backports.py \ + --since-days "${{ inputs.since_days || '7' }}" \ + --pattern "${{ inputs.pattern || 'fix flaky test' }}" \ + --upstream-ref "${{ inputs.upstream_ref || 'master' }}" \ + --output-file flaky_fix_report.md + + - name: Write summary + if: always() + run: cat flaky_fix_report.md >> "$GITHUB_STEP_SUMMARY" diff --git a/tests/ci/check_flaky_fix_backports.py b/tests/ci/check_flaky_fix_backports.py new file mode 100755 index 000000000000..2ed4f657b4d5 --- /dev/null +++ b/tests/ci/check_flaky_fix_backports.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +""" +Check upstream master for recent "fixed flaky test" commits and report +which ones are not yet ported to the current branch. + +Uses the GitHub API to list upstream commits (avoids shallow-fetch issues +with the ClickHouse repo), then checks each candidate against the local +git log by subject line. +""" + +import argparse +import json +import sys +import urllib.error +import urllib.request +from datetime import datetime, timedelta, timezone + + +UPSTREAM_BASE_URL = "https://github.com/{repo}/commit/{sha}" +GITHUB_API = "https://api.github.com" + + +def since_iso(days: int) -> str: + cutoff = datetime.now(timezone.utc) - timedelta(days=days) + return cutoff.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def github_commits(repo: str, ref: str, since: str) -> list: + """Fetch all commits on repo/ref since the given ISO timestamp, paginating as needed.""" + commits = [] + url = ( + f"{GITHUB_API}/repos/{repo}/commits" + f"?sha={ref}&since={since}&per_page=100" + ) + while url: + print(f"Fetching {url} ...", file=sys.stderr) + req = urllib.request.Request(url, headers={"User-Agent": "check_flaky_fix_backports"}) + try: + with urllib.request.urlopen(req) as resp: + commits.extend(json.loads(resp.read())) + link = resp.headers.get("Link", "") + url = None + for part in link.split(","): + if 'rel="next"' in part: + url = part.split(";")[0].strip().strip("<>") + except urllib.error.HTTPError as e: + print(f"GitHub API error: {e}", file=sys.stderr) + sys.exit(1) + return commits + + +def flaky_commits(commits: list, pattern: str) -> list: + """Filter to commits whose subject matches the pattern (case-insensitive). + Returns list of (sha, subject, date) tuples.""" + results = [] + for c in commits: + subject = c["commit"]["message"].splitlines()[0] + if pattern.lower() in subject.lower(): + date = c["commit"]["committer"]["date"] + results.append((c["sha"], subject, date)) + return results + + +def run_git(*args) -> str: + import subprocess + result = subprocess.run( + ["git"] + list(args), text=True, capture_output=True + ) + if result.returncode != 0: + print(f"git {' '.join(args)} failed:\n{result.stderr}", file=sys.stderr) + sys.exit(1) + return result.stdout.strip() + + +def is_ported(subject: str) -> bool: + """Return True if a commit with this subject exists on the current branch.""" + out = run_git("log", "--fixed-strings", f"--grep={subject}", "--format=%H", "HEAD") + return bool(out) + + +def missing_commits(candidates: list) -> list: + return [(sha, subj, date) for sha, subj, date in candidates if not is_ported(subj)] + + +def current_branch() -> str: + branch = run_git("rev-parse", "--abbrev-ref", "HEAD") + return branch if branch != "HEAD" else run_git("rev-parse", "HEAD")[:12] + + +def render_markdown( + branch: str, + upstream_repo: str, + upstream_ref: str, + days: int, + pattern: str, + all_upstream: list, + missing: list, +) -> str: + lines = [] + lines.append("## Flaky Fix Backport Check") + lines.append("") + lines.append(f"**Branch:** `{branch}` ") + lines.append(f"**Upstream:** `{upstream_repo}` @ `{upstream_ref}` ") + lines.append(f"**Window:** last {days} day(s) ") + lines.append(f"**Pattern:** `{pattern}` ") + lines.append("") + lines.append(f"Upstream commits matching pattern: **{len(all_upstream)}** ") + lines.append(f"Missing from current branch: **{len(missing)}** ") + lines.append("") + + if not missing: + lines.append(":white_check_mark: No missing flaky-fix backports found.") + else: + lines.append(":warning: The following upstream fixes are not yet ported:") + lines.append("") + for sha, subject, date in missing: + url = UPSTREAM_BASE_URL.format(repo=upstream_repo, sha=sha) + short = sha[:12] + lines.append(f"- [`{short}`]({url}) {subject} _(committed {date})_") + + lines.append("") + return "\n".join(lines) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Report upstream flaky-fix commits not yet ported to the current branch.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--since-days", + type=int, + default=7, + metavar="N", + help="how many days back to scan upstream", + ) + parser.add_argument( + "--pattern", + default="fix flaky test", + help="case-insensitive substring to match in commit subjects", + ) + parser.add_argument( + "--upstream-repo", + default="ClickHouse/ClickHouse", + metavar="OWNER/REPO", + help="upstream GitHub repository", + ) + parser.add_argument( + "--upstream-ref", + default="master", + help="upstream branch or ref to scan", + ) + parser.add_argument( + "--output-file", + metavar="PATH", + help="write markdown report to this file (default: stdout)", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + cutoff = since_iso(args.since_days) + branch = current_branch() + + commits = github_commits(args.upstream_repo, args.upstream_ref, cutoff) + all_upstream = flaky_commits(commits, args.pattern) + missing = missing_commits(all_upstream) + + report = render_markdown( + branch=branch, + upstream_repo=args.upstream_repo, + upstream_ref=args.upstream_ref, + days=args.since_days, + pattern=args.pattern, + all_upstream=all_upstream, + missing=missing, + ) + + if args.output_file: + with open(args.output_file, "w", encoding="utf-8") as f: + f.write(report) + print(f"Report written to {args.output_file}", file=sys.stderr) + else: + print(report) + + if missing: + print(f"\n{len(missing)} missing flaky-fix commit(s) found.", file=sys.stderr) + + +if __name__ == "__main__": + main() From 56f0453c05fa877f6c2173e00b500c407dc66704 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Mon, 20 Jul 2026 22:24:26 -0400 Subject: [PATCH 2/3] add gh token to flaky backport script --- .github/workflows/flaky_fix_backport_check.yml | 2 ++ tests/ci/check_flaky_fix_backports.py | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/flaky_fix_backport_check.yml b/.github/workflows/flaky_fix_backport_check.yml index 04659adedfa9..c6e3b1880ce2 100644 --- a/.github/workflows/flaky_fix_backport_check.yml +++ b/.github/workflows/flaky_fix_backport_check.yml @@ -37,6 +37,8 @@ jobs: python-version: '3.x' - name: Check for missing flaky-fix backports + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | python3 tests/ci/check_flaky_fix_backports.py \ --since-days "${{ inputs.since_days || '7' }}" \ diff --git a/tests/ci/check_flaky_fix_backports.py b/tests/ci/check_flaky_fix_backports.py index 2ed4f657b4d5..8d0a7f8356e3 100755 --- a/tests/ci/check_flaky_fix_backports.py +++ b/tests/ci/check_flaky_fix_backports.py @@ -10,6 +10,7 @@ import argparse import json +import os import sys import urllib.error import urllib.request @@ -27,6 +28,13 @@ def since_iso(days: int) -> str: def github_commits(repo: str, ref: str, since: str) -> list: """Fetch all commits on repo/ref since the given ISO timestamp, paginating as needed.""" + token = os.environ.get("GITHUB_TOKEN") + headers = {"User-Agent": "check_flaky_fix_backports"} + if token: + headers["Authorization"] = f"Bearer {token}" + else: + print("Warning: GITHUB_TOKEN not set; using unauthenticated API (60 req/hour limit)", file=sys.stderr) + commits = [] url = ( f"{GITHUB_API}/repos/{repo}/commits" @@ -34,7 +42,7 @@ def github_commits(repo: str, ref: str, since: str) -> list: ) while url: print(f"Fetching {url} ...", file=sys.stderr) - req = urllib.request.Request(url, headers={"User-Agent": "check_flaky_fix_backports"}) + req = urllib.request.Request(url, headers=headers) try: with urllib.request.urlopen(req) as resp: commits.extend(json.loads(resp.read())) From d9931923ac6c9fe881c483da275f9c72ce06ad3f Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 21 Jul 2026 09:00:26 -0400 Subject: [PATCH 3/3] Add script to backport flaky test fixes --- ..._backport_check.yml => flaky_fix_sync.yml} | 46 +++- tests/ci/backport_flaky_fixes.py | 224 ++++++++++++++++++ ..._fix_backports.py => check_flaky_fixes.py} | 27 ++- 3 files changed, 291 insertions(+), 6 deletions(-) rename .github/workflows/{flaky_fix_backport_check.yml => flaky_fix_sync.yml} (51%) create mode 100755 tests/ci/backport_flaky_fixes.py rename tests/ci/{check_flaky_fix_backports.py => check_flaky_fixes.py} (87%) diff --git a/.github/workflows/flaky_fix_backport_check.yml b/.github/workflows/flaky_fix_sync.yml similarity index 51% rename from .github/workflows/flaky_fix_backport_check.yml rename to .github/workflows/flaky_fix_sync.yml index c6e3b1880ce2..75cde5d3dcfb 100644 --- a/.github/workflows/flaky_fix_backport_check.yml +++ b/.github/workflows/flaky_fix_sync.yml @@ -18,6 +18,11 @@ on: required: false type: string default: 'master' + create_pr: + description: 'Cherry-pick missing commits and open a backport PR' + required: false + type: boolean + default: false schedule: - cron: '0 12 * * 1' # Every Monday at 08:00 EDT (12:00 UTC) @@ -40,12 +45,49 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - python3 tests/ci/check_flaky_fix_backports.py \ + python3 tests/ci/check_for_flaky_fixes.py \ --since-days "${{ inputs.since_days || '7' }}" \ --pattern "${{ inputs.pattern || 'fix flaky test' }}" \ --upstream-ref "${{ inputs.upstream_ref || 'master' }}" \ - --output-file flaky_fix_report.md + --md-output flaky_fix_report.md \ + --json-output flaky_fix.json - name: Write summary if: always() run: cat flaky_fix_report.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload check results + uses: actions/upload-artifact@v4 + with: + name: flaky-fix-check + path: | + flaky_fix_report.md + flaky_fix.json + + backport: + needs: check + if: ${{ github.event_name == 'schedule' || inputs.create_pr == true }} + runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Download check results + uses: actions/download-artifact@v4 + with: + name: flaky-fix-check + + - name: Cherry-pick and open PR + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 tests/ci/backport_flaky_fixes.py \ + --json-file flaky_fix.json \ + --repo "${{ github.repository }}" diff --git a/tests/ci/backport_flaky_fixes.py b/tests/ci/backport_flaky_fixes.py new file mode 100755 index 000000000000..3fd528494d2f --- /dev/null +++ b/tests/ci/backport_flaky_fixes.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +""" +Read the JSON output from check_for_flaky_fix_backports.py, cherry-pick each +missing commit onto a new branch, push it, and open a single PR. +""" + +import argparse +import json +import subprocess +import sys +from datetime import datetime, timezone + + +UPSTREAM_GIT_URL = "https://github.com/{repo}.git" +UPSTREAM_COMMIT_URL = "https://github.com/{repo}/commit/{sha}" + + +def run_git(*args, check=True, capture=True) -> subprocess.CompletedProcess: + result = subprocess.run( + ["git"] + list(args), + text=True, + capture_output=capture, + ) + if check and result.returncode != 0: + print(f"git {' '.join(args)} failed:\n{result.stderr}", file=sys.stderr) + sys.exit(1) + return result + + +def git_out(*args) -> str: + return run_git(*args).stdout.strip() + + +def fetch_commit(upstream_repo: str, sha: str) -> bool: + """Fetch a single commit object by SHA from the upstream repo. + Returns True on success, False on failure.""" + url = UPSTREAM_GIT_URL.format(repo=upstream_repo) + result = run_git("fetch", "--no-tags", "--no-recurse-submodules", url, sha, check=False) + if result.returncode != 0: + print(f"Failed to fetch {sha[:12]} from upstream:\n{result.stderr}", file=sys.stderr) + return False + return True + + +def cherry_pick(sha: str) -> bool: + """Attempt cherry-pick. Returns True on success, False on conflict.""" + result = run_git("cherry-pick", "-x", sha, check=False) + if result.returncode == 0: + return True + print(f"Cherry-pick of {sha[:12]} failed (conflict), aborting.", file=sys.stderr) + run_git("cherry-pick", "--abort", check=False) + return False + + +def build_pr_body( + upstream_repo: str, + applied: list, + conflicted: list, + fetch_failed: list, +) -> str: + lines = [] + lines.append("Automated backport of upstream flaky-fix commits.") + lines.append("") + lines.append("- [x] Exclude all Regression") + lines.append("") + + if applied: + lines.append("### Cherry-picked") + lines.append("") + for sha, subject, date in applied: + url = UPSTREAM_COMMIT_URL.format(repo=upstream_repo, sha=sha) + lines.append(f"- [`{sha[:12]}`]({url}) {subject} _(committed {date})_") + lines.append("") + + if conflicted: + lines.append("### Skipped (cherry-pick conflict — manual backport needed)") + lines.append("") + for sha, subject, date in conflicted: + url = UPSTREAM_COMMIT_URL.format(repo=upstream_repo, sha=sha) + lines.append(f"- [`{sha[:12]}`]({url}) {subject} _(committed {date})_") + lines.append("") + + if fetch_failed: + lines.append("### Skipped (fetch failed — commit may not be reachable)") + lines.append("") + for sha, subject, date in fetch_failed: + url = UPSTREAM_COMMIT_URL.format(repo=upstream_repo, sha=sha) + lines.append(f"- [`{sha[:12]}`]({url}) {subject} _(committed {date})_") + lines.append("") + + return "\n".join(lines) + + +def create_pr(repo: str, branch: str, base: str, title: str, body: str, dry_run: bool) -> None: + if dry_run: + print(f"DRY RUN: would open PR in {repo}:", file=sys.stderr) + print(f" title: {title}", file=sys.stderr) + print(f" base: {base}", file=sys.stderr) + print(f" head: {branch}", file=sys.stderr) + return + + result = subprocess.run( + [ + "gh", "pr", "create", + "--repo", repo, + "--base", base, + "--head", branch, + "--title", title, + "--body", body, + ], + text=True, + capture_output=False, + ) + if result.returncode != 0: + print("gh pr create failed", file=sys.stderr) + sys.exit(1) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Cherry-pick missing flaky-fix commits and open a backport PR.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--json-file", + required=True, + metavar="PATH", + help="JSON output file produced by check_flaky_fix_backports.py", + ) + parser.add_argument( + "--repo", + default="Altinity/ClickHouse", + metavar="OWNER/REPO", + help="target GitHub repository where the PR will be opened", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="fetch and cherry-pick locally but do not push or open a PR", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + with open(args.json_file, encoding="utf-8") as f: + data = json.load(f) + + missing = data.get("missing", []) + if not missing: + print("No missing commits in JSON file. Nothing to do.", file=sys.stderr) + return + + upstream_repo = data["upstream_repo"] + base_branch = data["branch"] + + # Sort oldest-first so cherry-picks apply in chronological order. + missing.sort(key=lambda c: c["date"]) + + date_tag = datetime.now(timezone.utc).strftime("%Y-%m-%d") + backport_branch = f"flaky-fix-backport/{base_branch}/{date_tag}" + + existing = run_git("branch", "--list", backport_branch).stdout.strip() + if existing: + print(f"Branch {backport_branch} already exists. Delete it first.", file=sys.stderr) + sys.exit(1) + + run_git("checkout", "-b", backport_branch) + + applied = [] + conflicted = [] + fetch_failed = [] + + for commit in missing: + sha = commit["sha"] + subject = commit["subject"] + date = commit["date"] + print(f"Fetching {sha[:12]}: {subject}", file=sys.stderr) + if not fetch_commit(upstream_repo, sha): + fetch_failed.append((sha, subject, date)) + print(f" Skipped {sha[:12]} (fetch failed)", file=sys.stderr) + continue + if cherry_pick(sha): + applied.append((sha, subject, date)) + print(f" Applied {sha[:12]}", file=sys.stderr) + else: + conflicted.append((sha, subject, date)) + print(f" Skipped {sha[:12]} (conflict)", file=sys.stderr) + + if not applied: + print("No commits could be applied (all conflicted or failed to fetch). Cleaning up.", file=sys.stderr) + run_git("checkout", base_branch) + run_git("branch", "-D", backport_branch) + sys.exit(0) + + pr_title = f"Backport flaky-fix commits from upstream ({date_tag})" + pr_body = build_pr_body(upstream_repo, applied, conflicted, fetch_failed) + print(f"\n--- PR title ---\n{pr_title}\n--- PR body ---\n{pr_body}---") + + if args.dry_run: + print( + f"DRY RUN: would push {backport_branch} and open PR against {base_branch}.", + file=sys.stderr, + ) + print(f" Applied: {len(applied)} Conflicted: {len(conflicted)} Fetch failed: {len(fetch_failed)}", file=sys.stderr) + run_git("checkout", base_branch) + run_git("branch", "-D", backport_branch) + return + + run_git("push", "origin", backport_branch, capture=False) + + create_pr( + repo=args.repo, + branch=backport_branch, + base=base_branch, + title=pr_title, + body=pr_body, + dry_run=False, + ) + + +if __name__ == "__main__": + main() diff --git a/tests/ci/check_flaky_fix_backports.py b/tests/ci/check_flaky_fixes.py similarity index 87% rename from tests/ci/check_flaky_fix_backports.py rename to tests/ci/check_flaky_fixes.py index 8d0a7f8356e3..f3f9da129c19 100755 --- a/tests/ci/check_flaky_fix_backports.py +++ b/tests/ci/check_flaky_fixes.py @@ -159,10 +159,15 @@ def parse_args() -> argparse.Namespace: help="upstream branch or ref to scan", ) parser.add_argument( - "--output-file", + "--md-output", metavar="PATH", help="write markdown report to this file (default: stdout)", ) + parser.add_argument( + "--json-output", + metavar="PATH", + help="write machine-readable JSON report to this file", + ) return parser.parse_args() @@ -186,13 +191,27 @@ def main() -> None: missing=missing, ) - if args.output_file: - with open(args.output_file, "w", encoding="utf-8") as f: + if args.md_output: + with open(args.md_output, "w", encoding="utf-8") as f: f.write(report) - print(f"Report written to {args.output_file}", file=sys.stderr) + print(f"Markdown report written to {args.md_output}", file=sys.stderr) else: print(report) + if args.json_output: + payload = { + "branch": branch, + "upstream_repo": args.upstream_repo, + "upstream_ref": args.upstream_ref, + "missing": [ + {"sha": sha, "subject": subj, "date": date} + for sha, subj, date in missing + ], + } + with open(args.json_output, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + print(f"JSON report written to {args.json_output}", file=sys.stderr) + if missing: print(f"\n{len(missing)} missing flaky-fix commit(s) found.", file=sys.stderr)