diff --git a/.github/scripts/mirror_tdvt.py b/.github/scripts/mirror_tdvt.py
new file mode 100644
index 000000000..9e69c1326
--- /dev/null
+++ b/.github/scripts/mirror_tdvt.py
@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""Dispatch Tableau TDVT and mirror its result. Used by nightly.yml and release.yml."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+from datetime import datetime, timedelta, timezone
+
+TDVT_REPO = "ClickHouse/clickhouse-tableau-tdvt"
+WORKFLOW = "tableau-tdvt.yml"
+POLL_ATTEMPTS = 24
+POLL_INTERVAL_SEC = 15
+WATCH_INTERVAL_SEC = 30
+CREATED_FILTER_SKEW_SEC = 120
+
+
+def fail(message: str) -> None:
+ print(f"::error::{message}", file=sys.stderr)
+ sys.exit(1)
+
+
+def gh_json(*args: str) -> dict | list:
+ result = subprocess.run(["gh", *args], check=True, capture_output=True, text=True)
+ return json.loads(result.stdout)
+
+
+def gh_api_call(path: str, *fields: str) -> None:
+ args = ["gh", "api", path]
+ for field in fields:
+ flag = "-F" if field.startswith("client_payload[") else "-f"
+ args.extend([flag, field])
+ subprocess.run(args, check=True)
+
+
+def latest_run_id(repo: str) -> int:
+ data = gh_json(
+ "api",
+ f"repos/{repo}/actions/workflows/{WORKFLOW}/runs?event=repository_dispatch&per_page=1",
+ )
+ runs = data.get("workflow_runs", [])
+ return runs[0]["id"] if runs else 0
+
+
+def list_runs(repo: str, created_since: str) -> list[dict]:
+ data = gh_json(
+ "api",
+ f"repos/{repo}/actions/workflows/{WORKFLOW}/runs"
+ f"?event=repository_dispatch&created=>={created_since}&per_page=20",
+ )
+ return data.get("workflow_runs", [])
+
+
+def title_has_version(display_title: str, version: str) -> bool:
+ # TDVT run-name is "TDVT (JDBC {version})" — require the closing ")" boundary.
+ return re.search(r"\(JDBC " + re.escape(version) + r"\)", display_title) is not None
+
+
+def version_matches(runs: list[dict], threshold: int, version: str) -> list[dict]:
+ return [
+ run
+ for run in runs
+ if run.get("id", 0) > threshold
+ and title_has_version(run.get("display_title", ""), version)
+ ]
+
+
+def parse_github_time(value: str) -> datetime:
+ return datetime.fromisoformat(value.replace("Z", "+00:00"))
+
+
+def find_run_id(
+ runs: list[dict],
+ threshold: int,
+ version: str,
+ correlation_id: str,
+ *,
+ final: bool = False,
+ dispatch_after: datetime | None = None,
+) -> int | None:
+ matches = version_matches(runs, threshold, version)
+
+ corr = [run for run in matches if correlation_id in run.get("display_title", "")]
+ if len(corr) == 1:
+ return corr[0]["id"]
+ if len(corr) > 1:
+ return None
+ if len(matches) > 1:
+ return None
+ if len(matches) == 1:
+ new_runs = [run for run in runs if run.get("id", 0) > threshold]
+ if len(new_runs) > 1:
+ return matches[0]["id"]
+ if final and dispatch_after is not None:
+ created = parse_github_time(matches[0]["created_at"])
+ if created >= dispatch_after - timedelta(seconds=10):
+ return matches[0]["id"]
+ return None
+
+
+def write_summary(version: str, run_url: str, info: dict) -> None:
+ path = os.environ.get("GITHUB_STEP_SUMMARY")
+ if not path:
+ return
+ lines = [
+ f"## Tableau TDVT - clickhouse-jdbc `{version}`",
+ "",
+ f"[Full TDVT run]({run_url})",
+ "",
+ f"**Conclusion: {info.get('conclusion', '')}**",
+ "",
+ "| Job | Result |",
+ "|---|---|",
+ ]
+ for job in info.get("jobs", []):
+ result = job.get("conclusion") or job.get("status") or ""
+ lines.append(f"| {job.get('name', '')} | {result} |")
+ with open(path, "a", encoding="utf-8") as summary:
+ summary.write("\n".join(lines) + "\n")
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--event-type", required=True, choices=("jdbc-snapshot", "jdbc-release"))
+ parser.add_argument("--version", required=True)
+ parser.add_argument("--correlation-id", required=True)
+ parser.add_argument("--tdvt-repo", default=TDVT_REPO)
+ args = parser.parse_args()
+
+ if not os.environ.get("GH_TOKEN"):
+ fail("GH_TOKEN is required")
+
+ print(
+ f"Dispatching TDVT for clickhouse-jdbc {args.version} "
+ f"(correlation: {args.correlation_id})"
+ )
+
+ prev_id = latest_run_id(args.tdvt_repo)
+ created_since = (
+ datetime.now(timezone.utc) - timedelta(seconds=CREATED_FILTER_SKEW_SEC)
+ ).strftime("%Y-%m-%dT%H:%M:%SZ")
+ dispatch_after = datetime.now(timezone.utc)
+ gh_api_call(
+ f"repos/{args.tdvt_repo}/dispatches",
+ f"event_type={args.event_type}",
+ f"client_payload[jdbc_version]={args.version}",
+ f"client_payload[correlation_id]={args.correlation_id}",
+ )
+
+ run_id = None
+ for _ in range(POLL_ATTEMPTS):
+ run_id = find_run_id(
+ list_runs(args.tdvt_repo, created_since),
+ prev_id,
+ args.version,
+ args.correlation_id,
+ )
+ if run_id is not None:
+ break
+ time.sleep(POLL_INTERVAL_SEC)
+
+ if run_id is None:
+ runs = list_runs(args.tdvt_repo, created_since)
+ run_id = find_run_id(
+ runs, prev_id, args.version, args.correlation_id,
+ final=True, dispatch_after=dispatch_after,
+ )
+ if run_id is None:
+ if len(version_matches(runs, prev_id, args.version)) > 1:
+ fail(
+ f"Multiple TDVT runs match JDBC {args.version} after dispatch; "
+ "cannot pick the correct run safely"
+ )
+ fail("TDVT run did not appear after dispatch")
+
+ run_url = f"https://github.com/{args.tdvt_repo}/actions/runs/{run_id}"
+ print(f"TDVT run: {run_url}")
+
+ subprocess.run(
+ ["gh", "run", "watch", str(run_id), "--repo", args.tdvt_repo, "--interval", str(WATCH_INTERVAL_SEC)],
+ check=False,
+ )
+ info = gh_json("run", "view", str(run_id), "--repo", args.tdvt_repo, "--json", "conclusion,jobs")
+ conclusion = info.get("conclusion") or ""
+ print(f"TDVT concluded: {conclusion}")
+ write_summary(args.version, run_url, info)
+
+ if conclusion != "success":
+ fail(f"Tableau TDVT did not pass (conclusion: {conclusion}) - {run_url}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 1846891dd..3345314ca 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -21,6 +21,8 @@ jobs:
name: "Build and Publish Nightly Snapshot"
runs-on: "ubuntu-latest"
timeout-minutes: 20
+ outputs:
+ jdbc_version: ${{ steps.publish-version.outputs.jdbc_version }}
steps:
- name: Check out Git repository
uses: actions/checkout@v4
@@ -63,6 +65,12 @@ jobs:
# -e 's|^\( \).*\(\)$|\1${{ env.CHC_VERSION }}-SNAPSHOT\2|' \
# -e 's|${parent.groupId}|com.clickhouse|g' -e 's|${project.parent.groupId}|com.clickhouse|g' '{}' \;
find . -type f -name "simplelogger.*" -exec rm -fv '{}' \;
+ - name: Record published snapshot version
+ id: publish-version
+ run: |
+ VERSION=$(sed -n 's|.*\(.*\).*|\1|p' pom.xml | head -1)
+ echo "Published snapshot version: $VERSION"
+ echo "jdbc_version=$VERSION" >> "$GITHUB_OUTPUT"
- name: Release Snapshot
uses: samuelmeuli/action-maven-publish@v1
with:
@@ -84,3 +92,29 @@ jobs:
gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }}
nexus_username: ${{ secrets.SONATYPE_TOKEN_USER }}
nexus_password: ${{ secrets.SONATYPE_TOKEN }}
+ # Mirror Tableau TDVT after publish (non-blocking).
+ tdvt:
+ name: "Tableau TDVT"
+ needs: nightly
+ if: ${{ needs.nightly.result == 'success' && startsWith(github.repository, 'ClickHouse/') }}
+ runs-on: "ubuntu-latest"
+ timeout-minutes: 150 # the TDVT matrix (latest/head/cloud, single-threaded) runs ~1h; allow for a queue
+ steps:
+ - name: Check out Git repository
+ uses: actions/checkout@v4
+ - name: Mint a token for the TDVT repo (dispatch + read the run)
+ id: tdvt-token
+ uses: actions/create-github-app-token@v3
+ with:
+ app-id: ${{ secrets.WORKFLOW_AUTH_PUBLIC_APP_ID }}
+ private-key: ${{ secrets.WORKFLOW_AUTH_PUBLIC_PRIVATE_KEY }}
+ owner: ${{ github.repository_owner }}
+ repositories: "clickhouse-tableau-tdvt"
+ - name: Run Tableau TDVT and mirror its result
+ env:
+ GH_TOKEN: ${{ steps.tdvt-token.outputs.token }}
+ run: |
+ python3 .github/scripts/mirror_tdvt.py \
+ --event-type jdbc-snapshot \
+ --version "${{ needs.nightly.outputs.jdbc_version }}" \
+ --correlation-id "${{ github.repository }}#${{ github.run_id }}-${{ github.run_attempt }}"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 022d27155..c14eb01ce 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -88,3 +88,29 @@ jobs:
clickhouse-r2dbc/target/clickhouse*.jar
client-v2/target/client-v2*.jar
jdbc-v2/target/jdbc-v2*.jar
+ # Mirror Tableau TDVT after release (non-blocking).
+ tdvt:
+ name: "Tableau TDVT"
+ needs: release
+ if: ${{ needs.release.result == 'success' && startsWith(github.repository, 'ClickHouse/') }}
+ runs-on: "ubuntu-latest"
+ timeout-minutes: 150 # the TDVT matrix (latest/head/cloud, single-threaded) runs ~1h; allow for a queue
+ steps:
+ - name: Check out Git repository
+ uses: actions/checkout@v4
+ - name: Mint a token for the TDVT repo (dispatch + read the run)
+ id: tdvt-token
+ uses: actions/create-github-app-token@v3
+ with:
+ app-id: ${{ secrets.WORKFLOW_AUTH_PUBLIC_APP_ID }}
+ private-key: ${{ secrets.WORKFLOW_AUTH_PUBLIC_PRIVATE_KEY }}
+ owner: ${{ github.repository_owner }}
+ repositories: "clickhouse-tableau-tdvt"
+ - name: Run Tableau TDVT and mirror its result
+ env:
+ GH_TOKEN: ${{ steps.tdvt-token.outputs.token }}
+ run: |
+ python3 .github/scripts/mirror_tdvt.py \
+ --event-type jdbc-release \
+ --version "${{ github.event.inputs.version }}" \
+ --correlation-id "${{ github.repository }}#${{ github.run_id }}-${{ github.run_attempt }}"
diff --git a/.gitignore b/.gitignore
index bb8e8575f..5b82fa60e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -60,3 +60,7 @@ performance/jmh-simple-results.json
*.csv
*.sql
*.json
+
+# Python
+__pycache__/
+*.pyc