Antalya 26.3 - New workflow to identify flaky test fixes

.github/workflows/flaky_fix_sync.yml

@@ -0,0 +1,93 @@
+name: Flaky Fix Backport Check
+
+on:
+  workflow_dispatch:
+    inputs:
+      since_days:
+        description: 'How many days back to scan upstream master (default: 7)'
+        required: false
+        type: string
+        default: '7'
+      pattern:
+        description: 'Commit subject pattern to match (case-insensitive)'
+        required: false
+        type: string
+        default: 'fix flaky test'
+      upstream_ref:
+        description: 'Upstream branch/ref to scan'
+        required: false
+        type: string
+        default: 'master'
+      create_pr:
+        description: 'Cherry-pick missing commits and open a backport PR'
+        required: false
+        type: boolean
+        default: false
+
+  schedule:
+    - cron: '0 12 * * 1'  # Every Monday at 08:00 EDT (12:00 UTC)
+
+jobs:
+  check:
+    runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Check for missing flaky-fix backports
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 tests/ci/check_for_flaky_fixes.py \
+            --since-days "${{ inputs.since_days || '7' }}" \
+            --pattern "${{ inputs.pattern || 'fix flaky test' }}" \
+            --upstream-ref "${{ inputs.upstream_ref || 'master' }}" \
+            --md-output flaky_fix_report.md \
+            --json-output flaky_fix.json
+
+      - name: Write summary
+        if: always()
+        run: cat flaky_fix_report.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload check results
+        uses: actions/upload-artifact@v4
+        with:
+          name: flaky-fix-check
+          path: |
+            flaky_fix_report.md
+            flaky_fix.json
+
+  backport:
+    needs: check
+    if: ${{ github.event_name == 'schedule' || inputs.create_pr == true }}
+    runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Download check results
+        uses: actions/download-artifact@v4
+        with:
+          name: flaky-fix-check
+
+      - name: Cherry-pick and open PR
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 tests/ci/backport_flaky_fixes.py \
+            --json-file flaky_fix.json \
+            --repo "${{ github.repository }}"

tests/ci/backport_flaky_fixes.py

@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+"""
+Read the JSON output from check_for_flaky_fix_backports.py, cherry-pick each
+missing commit onto a new branch, push it, and open a single PR.
+"""
+
+import argparse
+import json
+import subprocess
+import sys
+from datetime import datetime, timezone
+
+
+UPSTREAM_GIT_URL = "https://github.com/{repo}.git"
+UPSTREAM_COMMIT_URL = "https://github.com/{repo}/commit/{sha}"
+
+
+def run_git(*args, check=True, capture=True) -> subprocess.CompletedProcess:
+    result = subprocess.run(
+        ["git"] + list(args),
+        text=True,
+        capture_output=capture,
+    )
+    if check and result.returncode != 0:
+        print(f"git {' '.join(args)} failed:\n{result.stderr}", file=sys.stderr)
+        sys.exit(1)
+    return result
+
+
+def git_out(*args) -> str:
+    return run_git(*args).stdout.strip()
+
+
+def fetch_commit(upstream_repo: str, sha: str) -> bool:
+    """Fetch a single commit object by SHA from the upstream repo.
+    Returns True on success, False on failure."""
+    url = UPSTREAM_GIT_URL.format(repo=upstream_repo)
+    result = run_git("fetch", "--no-tags", "--no-recurse-submodules", url, sha, check=False)
+    if result.returncode != 0:
+        print(f"Failed to fetch {sha[:12]} from upstream:\n{result.stderr}", file=sys.stderr)
+        return False
+    return True
+
+
+def cherry_pick(sha: str) -> bool:
+    """Attempt cherry-pick. Returns True on success, False on conflict."""
+    result = run_git("cherry-pick", "-x", sha, check=False)
+    if result.returncode == 0:
+        return True
+    print(f"Cherry-pick of {sha[:12]} failed (conflict), aborting.", file=sys.stderr)
+    run_git("cherry-pick", "--abort", check=False)
+    return False
+
+
+def build_pr_body(
+    upstream_repo: str,
+    applied: list,
+    conflicted: list,
+    fetch_failed: list,
+) -> str:
+    lines = []
+    lines.append("Automated backport of upstream flaky-fix commits.")
+    lines.append("")
+    lines.append("- [x] <!---ci_exclude_regression--> Exclude all Regression")
+    lines.append("")
+
+    if applied:
+        lines.append("### Cherry-picked")
+        lines.append("")
+        for sha, subject, date in applied:
+            url = UPSTREAM_COMMIT_URL.format(repo=upstream_repo, sha=sha)
+            lines.append(f"- [`{sha[:12]}`]({url}) {subject}  _(committed {date})_")
+        lines.append("")
+
+    if conflicted:
+        lines.append("### Skipped (cherry-pick conflict — manual backport needed)")
+        lines.append("")
+        for sha, subject, date in conflicted:
+            url = UPSTREAM_COMMIT_URL.format(repo=upstream_repo, sha=sha)
+            lines.append(f"- [`{sha[:12]}`]({url}) {subject}  _(committed {date})_")
+        lines.append("")
+
+    if fetch_failed:
+        lines.append("### Skipped (fetch failed — commit may not be reachable)")
+        lines.append("")
+        for sha, subject, date in fetch_failed:
+            url = UPSTREAM_COMMIT_URL.format(repo=upstream_repo, sha=sha)
+            lines.append(f"- [`{sha[:12]}`]({url}) {subject}  _(committed {date})_")
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def create_pr(repo: str, branch: str, base: str, title: str, body: str, dry_run: bool) -> None:
+    if dry_run:
+        print(f"DRY RUN: would open PR in {repo}:", file=sys.stderr)
+        print(f"  title: {title}", file=sys.stderr)
+        print(f"  base:  {base}", file=sys.stderr)
+        print(f"  head:  {branch}", file=sys.stderr)
+        return
+
+    result = subprocess.run(
+        [
+            "gh", "pr", "create",
+            "--repo", repo,
+            "--base", base,
+            "--head", branch,
+            "--title", title,
+            "--body", body,
+        ],
+        text=True,
+        capture_output=False,
+    )
+    if result.returncode != 0:
+        print("gh pr create failed", file=sys.stderr)
+        sys.exit(1)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Cherry-pick missing flaky-fix commits and open a backport PR.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--json-file",
+        required=True,
+        metavar="PATH",
+        help="JSON output file produced by check_flaky_fix_backports.py",
+    )
+    parser.add_argument(
+        "--repo",
+        default="Altinity/ClickHouse",
+        metavar="OWNER/REPO",
+        help="target GitHub repository where the PR will be opened",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="fetch and cherry-pick locally but do not push or open a PR",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+
+    with open(args.json_file, encoding="utf-8") as f:
+        data = json.load(f)
+
+    missing = data.get("missing", [])
+    if not missing:
+        print("No missing commits in JSON file. Nothing to do.", file=sys.stderr)
+        return
+
+    upstream_repo = data["upstream_repo"]
+    base_branch = data["branch"]
+
+    # Sort oldest-first so cherry-picks apply in chronological order.
+    missing.sort(key=lambda c: c["date"])
+
+    date_tag = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    backport_branch = f"flaky-fix-backport/{base_branch}/{date_tag}"
+
+    existing = run_git("branch", "--list", backport_branch).stdout.strip()
+    if existing:
+        print(f"Branch {backport_branch} already exists. Delete it first.", file=sys.stderr)
+        sys.exit(1)
+
+    run_git("checkout", "-b", backport_branch)
+
+    applied = []
+    conflicted = []
+    fetch_failed = []
+
+    for commit in missing:
+        sha = commit["sha"]
+        subject = commit["subject"]
+        date = commit["date"]
+        print(f"Fetching {sha[:12]}: {subject}", file=sys.stderr)
+        if not fetch_commit(upstream_repo, sha):
+            fetch_failed.append((sha, subject, date))
+            print(f"  Skipped {sha[:12]} (fetch failed)", file=sys.stderr)
+            continue
+        if cherry_pick(sha):
+            applied.append((sha, subject, date))
+            print(f"  Applied {sha[:12]}", file=sys.stderr)
+        else:
+            conflicted.append((sha, subject, date))
+            print(f"  Skipped {sha[:12]} (conflict)", file=sys.stderr)
+
+    if not applied:
+        print("No commits could be applied (all conflicted or failed to fetch). Cleaning up.", file=sys.stderr)
+        run_git("checkout", base_branch)
+        run_git("branch", "-D", backport_branch)
+        sys.exit(0)
+
+    pr_title = f"Backport flaky-fix commits from upstream ({date_tag})"
+    pr_body = build_pr_body(upstream_repo, applied, conflicted, fetch_failed)
+    print(f"\n--- PR title ---\n{pr_title}\n--- PR body ---\n{pr_body}---")
+
+    if args.dry_run:
+        print(
+            f"DRY RUN: would push {backport_branch} and open PR against {base_branch}.",
+            file=sys.stderr,
+        )
+        print(f"  Applied: {len(applied)}  Conflicted: {len(conflicted)}  Fetch failed: {len(fetch_failed)}", file=sys.stderr)
+        run_git("checkout", base_branch)
+        run_git("branch", "-D", backport_branch)
+        return
+
+    run_git("push", "origin", backport_branch, capture=False)
+
+    create_pr(
+        repo=args.repo,
+        branch=backport_branch,
+        base=base_branch,
+        title=pr_title,
+        body=pr_body,
+        dry_run=False,
+    )
+
+
+if __name__ == "__main__":
+    main()