diff --git a/.github/scripts/mirror_tdvt.py b/.github/scripts/mirror_tdvt.py new file mode 100644 index 000000000..9e69c1326 --- /dev/null +++ b/.github/scripts/mirror_tdvt.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Dispatch Tableau TDVT and mirror its result. Used by nightly.yml and release.yml.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import time +from datetime import datetime, timedelta, timezone + +TDVT_REPO = "ClickHouse/clickhouse-tableau-tdvt" +WORKFLOW = "tableau-tdvt.yml" +POLL_ATTEMPTS = 24 +POLL_INTERVAL_SEC = 15 +WATCH_INTERVAL_SEC = 30 +CREATED_FILTER_SKEW_SEC = 120 + + +def fail(message: str) -> None: + print(f"::error::{message}", file=sys.stderr) + sys.exit(1) + + +def gh_json(*args: str) -> dict | list: + result = subprocess.run(["gh", *args], check=True, capture_output=True, text=True) + return json.loads(result.stdout) + + +def gh_api_call(path: str, *fields: str) -> None: + args = ["gh", "api", path] + for field in fields: + flag = "-F" if field.startswith("client_payload[") else "-f" + args.extend([flag, field]) + subprocess.run(args, check=True) + + +def latest_run_id(repo: str) -> int: + data = gh_json( + "api", + f"repos/{repo}/actions/workflows/{WORKFLOW}/runs?event=repository_dispatch&per_page=1", + ) + runs = data.get("workflow_runs", []) + return runs[0]["id"] if runs else 0 + + +def list_runs(repo: str, created_since: str) -> list[dict]: + data = gh_json( + "api", + f"repos/{repo}/actions/workflows/{WORKFLOW}/runs" + f"?event=repository_dispatch&created=>={created_since}&per_page=20", + ) + return data.get("workflow_runs", []) + + +def title_has_version(display_title: str, version: str) -> bool: + # TDVT run-name is "TDVT (JDBC {version})" — require the closing ")" boundary. + return re.search(r"\(JDBC " + re.escape(version) + r"\)", display_title) is not None + + +def version_matches(runs: list[dict], threshold: int, version: str) -> list[dict]: + return [ + run + for run in runs + if run.get("id", 0) > threshold + and title_has_version(run.get("display_title", ""), version) + ] + + +def parse_github_time(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def find_run_id( + runs: list[dict], + threshold: int, + version: str, + correlation_id: str, + *, + final: bool = False, + dispatch_after: datetime | None = None, +) -> int | None: + matches = version_matches(runs, threshold, version) + + corr = [run for run in matches if correlation_id in run.get("display_title", "")] + if len(corr) == 1: + return corr[0]["id"] + if len(corr) > 1: + return None + if len(matches) > 1: + return None + if len(matches) == 1: + new_runs = [run for run in runs if run.get("id", 0) > threshold] + if len(new_runs) > 1: + return matches[0]["id"] + if final and dispatch_after is not None: + created = parse_github_time(matches[0]["created_at"]) + if created >= dispatch_after - timedelta(seconds=10): + return matches[0]["id"] + return None + + +def write_summary(version: str, run_url: str, info: dict) -> None: + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + return + lines = [ + f"## Tableau TDVT - clickhouse-jdbc `{version}`", + "", + f"[Full TDVT run]({run_url})", + "", + f"**Conclusion: {info.get('conclusion', '')}**", + "", + "| Job | Result |", + "|---|---|", + ] + for job in info.get("jobs", []): + result = job.get("conclusion") or job.get("status") or "" + lines.append(f"| {job.get('name', '')} | {result} |") + with open(path, "a", encoding="utf-8") as summary: + summary.write("\n".join(lines) + "\n") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--event-type", required=True, choices=("jdbc-snapshot", "jdbc-release")) + parser.add_argument("--version", required=True) + parser.add_argument("--correlation-id", required=True) + parser.add_argument("--tdvt-repo", default=TDVT_REPO) + args = parser.parse_args() + + if not os.environ.get("GH_TOKEN"): + fail("GH_TOKEN is required") + + print( + f"Dispatching TDVT for clickhouse-jdbc {args.version} " + f"(correlation: {args.correlation_id})" + ) + + prev_id = latest_run_id(args.tdvt_repo) + created_since = ( + datetime.now(timezone.utc) - timedelta(seconds=CREATED_FILTER_SKEW_SEC) + ).strftime("%Y-%m-%dT%H:%M:%SZ") + dispatch_after = datetime.now(timezone.utc) + gh_api_call( + f"repos/{args.tdvt_repo}/dispatches", + f"event_type={args.event_type}", + f"client_payload[jdbc_version]={args.version}", + f"client_payload[correlation_id]={args.correlation_id}", + ) + + run_id = None + for _ in range(POLL_ATTEMPTS): + run_id = find_run_id( + list_runs(args.tdvt_repo, created_since), + prev_id, + args.version, + args.correlation_id, + ) + if run_id is not None: + break + time.sleep(POLL_INTERVAL_SEC) + + if run_id is None: + runs = list_runs(args.tdvt_repo, created_since) + run_id = find_run_id( + runs, prev_id, args.version, args.correlation_id, + final=True, dispatch_after=dispatch_after, + ) + if run_id is None: + if len(version_matches(runs, prev_id, args.version)) > 1: + fail( + f"Multiple TDVT runs match JDBC {args.version} after dispatch; " + "cannot pick the correct run safely" + ) + fail("TDVT run did not appear after dispatch") + + run_url = f"https://github.com/{args.tdvt_repo}/actions/runs/{run_id}" + print(f"TDVT run: {run_url}") + + subprocess.run( + ["gh", "run", "watch", str(run_id), "--repo", args.tdvt_repo, "--interval", str(WATCH_INTERVAL_SEC)], + check=False, + ) + info = gh_json("run", "view", str(run_id), "--repo", args.tdvt_repo, "--json", "conclusion,jobs") + conclusion = info.get("conclusion") or "" + print(f"TDVT concluded: {conclusion}") + write_summary(args.version, run_url, info) + + if conclusion != "success": + fail(f"Tableau TDVT did not pass (conclusion: {conclusion}) - {run_url}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1846891dd..3345314ca 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -21,6 +21,8 @@ jobs: name: "Build and Publish Nightly Snapshot" runs-on: "ubuntu-latest" timeout-minutes: 20 + outputs: + jdbc_version: ${{ steps.publish-version.outputs.jdbc_version }} steps: - name: Check out Git repository uses: actions/checkout@v4 @@ -63,6 +65,12 @@ jobs: # -e 's|^\( \).*\(\)$|\1${{ env.CHC_VERSION }}-SNAPSHOT\2|' \ # -e 's|${parent.groupId}|com.clickhouse|g' -e 's|${project.parent.groupId}|com.clickhouse|g' '{}' \; find . -type f -name "simplelogger.*" -exec rm -fv '{}' \; + - name: Record published snapshot version + id: publish-version + run: | + VERSION=$(sed -n 's|.*\(.*\).*|\1|p' pom.xml | head -1) + echo "Published snapshot version: $VERSION" + echo "jdbc_version=$VERSION" >> "$GITHUB_OUTPUT" - name: Release Snapshot uses: samuelmeuli/action-maven-publish@v1 with: @@ -84,3 +92,29 @@ jobs: gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }} nexus_username: ${{ secrets.SONATYPE_TOKEN_USER }} nexus_password: ${{ secrets.SONATYPE_TOKEN }} + # Mirror Tableau TDVT after publish (non-blocking). + tdvt: + name: "Tableau TDVT" + needs: nightly + if: ${{ needs.nightly.result == 'success' && startsWith(github.repository, 'ClickHouse/') }} + runs-on: "ubuntu-latest" + timeout-minutes: 150 # the TDVT matrix (latest/head/cloud, single-threaded) runs ~1h; allow for a queue + steps: + - name: Check out Git repository + uses: actions/checkout@v4 + - name: Mint a token for the TDVT repo (dispatch + read the run) + id: tdvt-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ secrets.WORKFLOW_AUTH_PUBLIC_APP_ID }} + private-key: ${{ secrets.WORKFLOW_AUTH_PUBLIC_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "clickhouse-tableau-tdvt" + - name: Run Tableau TDVT and mirror its result + env: + GH_TOKEN: ${{ steps.tdvt-token.outputs.token }} + run: | + python3 .github/scripts/mirror_tdvt.py \ + --event-type jdbc-snapshot \ + --version "${{ needs.nightly.outputs.jdbc_version }}" \ + --correlation-id "${{ github.repository }}#${{ github.run_id }}-${{ github.run_attempt }}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 022d27155..c14eb01ce 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -88,3 +88,29 @@ jobs: clickhouse-r2dbc/target/clickhouse*.jar client-v2/target/client-v2*.jar jdbc-v2/target/jdbc-v2*.jar + # Mirror Tableau TDVT after release (non-blocking). + tdvt: + name: "Tableau TDVT" + needs: release + if: ${{ needs.release.result == 'success' && startsWith(github.repository, 'ClickHouse/') }} + runs-on: "ubuntu-latest" + timeout-minutes: 150 # the TDVT matrix (latest/head/cloud, single-threaded) runs ~1h; allow for a queue + steps: + - name: Check out Git repository + uses: actions/checkout@v4 + - name: Mint a token for the TDVT repo (dispatch + read the run) + id: tdvt-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ secrets.WORKFLOW_AUTH_PUBLIC_APP_ID }} + private-key: ${{ secrets.WORKFLOW_AUTH_PUBLIC_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "clickhouse-tableau-tdvt" + - name: Run Tableau TDVT and mirror its result + env: + GH_TOKEN: ${{ steps.tdvt-token.outputs.token }} + run: | + python3 .github/scripts/mirror_tdvt.py \ + --event-type jdbc-release \ + --version "${{ github.event.inputs.version }}" \ + --correlation-id "${{ github.repository }}#${{ github.run_id }}-${{ github.run_attempt }}" diff --git a/.gitignore b/.gitignore index bb8e8575f..5b82fa60e 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,7 @@ performance/jmh-simple-results.json *.csv *.sql *.json + +# Python +__pycache__/ +*.pyc