From 1e1c938903826a903a5a2c67b65b4e6a58465853 Mon Sep 17 00:00:00 2001
From: 3for <287494524@qq.com>
Date: Wed, 9 Sep 2026 16:05:02 +0800
Subject: [PATCH 01/13] ci: slim down Rocky Linux build dependencies
- Remove Development Tools and unnecessary packages
- Install only JDK 8, git-core and zstd with weak dependencies disabled
- Use C.utf8 to avoid installing glibc-langpack-en
- Resolve and validate JAVA_HOME so Gradle no longer requires which
---
.github/workflows/pr-build.yml | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml
index c7a5a6f4160..625ed783fab 100644
--- a/.github/workflows/pr-build.yml
+++ b/.github/workflows/pr-build.yml
@@ -113,15 +113,26 @@ jobs:
env:
GRADLE_USER_HOME: /github/home/.gradle
- LANG: en_US.UTF-8
- LC_ALL: en_US.UTF-8
+ LANG: C.utf8
+ LC_ALL: C.utf8
steps:
- name: Install dependencies (Rocky 8 + JDK8)
run: |
set -euxo pipefail
- dnf -y install java-1.8.0-openjdk-devel git wget unzip which jq bc curl glibc-langpack-en
- dnf -y groupinstall "Development Tools"
+ # Rocky 8 already provides CA certificates, JNI runtime libraries, tar and gzip.
+ # Its built-in C.utf8 locale provides UTF-8 without an extra language pack.
+ # git-core provides checkout commands; zstd supports actions/cache compression.
+ dnf -y --setopt=install_weak_deps=False install \
+ java-1.8.0-openjdk-devel git-core zstd
+ # Set JAVA_HOME so the Gradle wrapper does not need which.
+ javac_path=$(command -v javac)
+ javac_real=$(readlink -f "$javac_path")
+ jdk_bin=$(dirname "$javac_real")
+ jdk_home=$(dirname "$jdk_bin")
+ test -x "$jdk_home/bin/java"
+ test -x "$jdk_home/bin/javac"
+ printf 'JAVA_HOME=%s\n' "$jdk_home" >> "$GITHUB_ENV"
- name: Checkout code
uses: actions/checkout@v5
From 5707ef1c455b1a4ce496603e3c1bcb4d73218c8b Mon Sep 17 00:00:00 2001
From: 3for <287494524@qq.com>
Date: Wed, 9 Sep 2026 16:30:28 +0800
Subject: [PATCH 02/13] ci: upload unit test logs and JUnit reports for failure
diagnosis
- Collect tron-test.log, rotated logs and JUnit XML across five PR build jobs
- Run artifact uploads even when preceding steps fail
- Use distinct artifact names per job and matrix configuration
- Retain artifacts for 7 days and warn when no files are found
---
.github/workflows/pr-build.yml | 60 ++++++++++++++++++++++++++++++++++
1 file changed, 60 insertions(+)
diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml
index 625ed783fab..8189006452c 100644
--- a/.github/workflows/pr-build.yml
+++ b/.github/workflows/pr-build.yml
@@ -63,6 +63,18 @@ jobs:
java -jar "$JAR" db archive -h
java -jar "$JAR" keystore --help
+ - name: Upload unit test logs
+ if: always()
+ uses: actions/upload-artifact@v6
+ with:
+ name: tron-test-logs-${{ github.job }}-jdk${{ matrix.java }}-${{ matrix.arch }}
+ path: |
+ **/logs/tron-test.log
+ **/logs/tron-test-*.log.zip
+ **/build/test-results/**/TEST-*.xml
+ if-no-files-found: warn
+ retention-days: 7
+
build-ubuntu:
name: Build ubuntu24 (JDK 17 / aarch64)
if: ${{ github.event_name == 'pull_request' || inputs.job == 'all' || inputs.job == 'ubuntu' }}
@@ -102,6 +114,18 @@ jobs:
java -jar "$JAR" db archive -h
java -jar "$JAR" keystore --help
+ - name: Upload unit test logs
+ if: always()
+ uses: actions/upload-artifact@v6
+ with:
+ name: tron-test-logs-${{ github.job }}
+ path: |
+ **/logs/tron-test.log
+ **/logs/tron-test-*.log.zip
+ **/build/test-results/**/TEST-*.xml
+ if-no-files-found: warn
+ retention-days: 7
+
docker-build-rockylinux:
name: Build rockylinux (JDK 8 / x86_64)
if: ${{ github.event_name == 'pull_request' || inputs.job == 'all' || inputs.job == 'rockylinux' }}
@@ -168,6 +192,18 @@ jobs:
- name: Test with RocksDB engine
run: ./gradlew :framework:testWithRocksDb --no-daemon
+ - name: Upload unit test logs
+ if: always()
+ uses: actions/upload-artifact@v6
+ with:
+ name: tron-test-logs-${{ github.job }}
+ path: |
+ **/logs/tron-test.log
+ **/logs/tron-test-*.log.zip
+ **/build/test-results/**/TEST-*.xml
+ if-no-files-found: warn
+ retention-days: 7
+
docker-build-debian11:
name: Build debian11 (JDK 8 / x86_64)
if: ${{ github.event_name == 'pull_request' || inputs.job == 'all' || inputs.job == 'debian11' }}
@@ -233,6 +269,18 @@ jobs:
**/build/reports/jacoco/test/jacocoTestReport.xml
if-no-files-found: error
+ - name: Upload unit test logs
+ if: always()
+ uses: actions/upload-artifact@v6
+ with:
+ name: tron-test-logs-${{ github.job }}
+ path: |
+ **/logs/tron-test.log
+ **/logs/tron-test-*.log.zip
+ **/build/test-results/**/TEST-*.xml
+ if-no-files-found: warn
+ retention-days: 7
+
coverage-base:
name: Coverage Base (JDK 8 / x86_64)
if: ${{ github.event_name == 'pull_request' }}
@@ -293,6 +341,18 @@ jobs:
**/build/reports/jacoco/test/jacocoTestReport.xml
if-no-files-found: warn
+ - name: Upload unit test logs
+ if: always()
+ uses: actions/upload-artifact@v6
+ with:
+ name: tron-test-logs-${{ github.job }}
+ path: |
+ **/logs/tron-test.log
+ **/logs/tron-test-*.log.zip
+ **/build/test-results/**/TEST-*.xml
+ if-no-files-found: warn
+ retention-days: 7
+
coverage-gate:
name: Coverage Gate
needs: [docker-build-debian11, coverage-base]
From 4b6cbf63a14705f6aa76fc5995fa16e218d3f230 Mon Sep 17 00:00:00 2001
From: 3for <287494524@qq.com>
Date: Wed, 9 Sep 2026 16:35:22 +0800
Subject: [PATCH 03/13] ci: remove deleted multinode workflow from PR
cancellation list
Remove the stale integration-test-multinode.yml reference to avoid
unnecessary API requests when cancelling workflows for closed PRs.
---
.github/workflows/pr-cancel.yml | 1 -
1 file changed, 1 deletion(-)
diff --git a/.github/workflows/pr-cancel.yml b/.github/workflows/pr-cancel.yml
index 3213026d3f9..a4d46153d47 100644
--- a/.github/workflows/pr-cancel.yml
+++ b/.github/workflows/pr-cancel.yml
@@ -21,7 +21,6 @@ jobs:
'pr-build.yml',
'codeql.yml',
'integration-test-single-node.yml',
- 'integration-test-multinode.yml',
];
const headSha = context.payload.pull_request.head.sha;
const prNumber = context.payload.pull_request.number;
From a22a2c7ad3ee3c458efc34b742baccae7f586e04 Mon Sep 17 00:00:00 2001
From: 3for <287494524@qq.com>
Date: Wed, 9 Sep 2026 17:31:10 +0800
Subject: [PATCH 04/13] ci: capture Gradle console logs and upload HTML test
reports
- Save build, RocksDB test and coverage output with tee and plain console mode
- Use Bash pipefail to preserve failures when capturing stdout and stderr
- Include console logs and HTML test reports in diagnostic artifacts
- Preserve existing test retry and base coverage failure policies
---
.github/workflows/pr-build.yml | 80 +++++++++++++++++++++++++++-------
1 file changed, 65 insertions(+), 15 deletions(-)
diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml
index 8189006452c..b8056094ee1 100644
--- a/.github/workflows/pr-build.yml
+++ b/.github/workflows/pr-build.yml
@@ -52,7 +52,11 @@ jobs:
restore-keys: macos26-${{ matrix.arch }}-gradle-
- name: Build
- run: ./gradlew clean build --no-daemon
+ shell: bash
+ run: |
+ set -euo pipefail
+ mkdir -p ci-logs
+ ./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log
- name: Toolkit jar smoke test
run: |
@@ -63,7 +67,7 @@ jobs:
java -jar "$JAR" db archive -h
java -jar "$JAR" keystore --help
- - name: Upload unit test logs
+ - name: Upload test diagnostics
if: always()
uses: actions/upload-artifact@v6
with:
@@ -72,6 +76,8 @@ jobs:
**/logs/tron-test.log
**/logs/tron-test-*.log.zip
**/build/test-results/**/TEST-*.xml
+ **/build/reports/tests/**
+ ci-logs/*.log
if-no-files-found: warn
retention-days: 7
@@ -103,7 +109,11 @@ jobs:
restore-keys: ubuntu24-aarch64-gradle-
- name: Build
- run: ./gradlew clean build --no-daemon
+ shell: bash
+ run: |
+ set -euo pipefail
+ mkdir -p ci-logs
+ ./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log
- name: Toolkit jar smoke test
run: |
@@ -114,7 +124,7 @@ jobs:
java -jar "$JAR" db archive -h
java -jar "$JAR" keystore --help
- - name: Upload unit test logs
+ - name: Upload test diagnostics
if: always()
uses: actions/upload-artifact@v6
with:
@@ -123,6 +133,8 @@ jobs:
**/logs/tron-test.log
**/logs/tron-test-*.log.zip
**/build/test-results/**/TEST-*.xml
+ **/build/reports/tests/**
+ ci-logs/*.log
if-no-files-found: warn
retention-days: 7
@@ -178,7 +190,11 @@ jobs:
run: ./gradlew --stop || true
- name: Build
- run: ./gradlew clean build --no-daemon
+ shell: bash
+ run: |
+ set -euo pipefail
+ mkdir -p ci-logs
+ ./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log
- name: Toolkit jar smoke test
run: |
@@ -190,9 +206,13 @@ jobs:
java -jar "$JAR" keystore --help
- name: Test with RocksDB engine
- run: ./gradlew :framework:testWithRocksDb --no-daemon
+ shell: bash
+ run: |
+ set -euo pipefail
+ mkdir -p ci-logs
+ ./gradlew :framework:testWithRocksDb --no-daemon --console=plain 2>&1 | tee ci-logs/rocksdb-test.log
- - name: Upload unit test logs
+ - name: Upload test diagnostics
if: always()
uses: actions/upload-artifact@v6
with:
@@ -201,6 +221,8 @@ jobs:
**/logs/tron-test.log
**/logs/tron-test-*.log.zip
**/build/test-results/**/TEST-*.xml
+ **/build/reports/tests/**
+ ci-logs/*.log
if-no-files-found: warn
retention-days: 7
@@ -244,7 +266,11 @@ jobs:
debian11-x86_64-gradle-
- name: Build
- run: ./gradlew clean build --no-daemon
+ shell: bash
+ run: |
+ set -euo pipefail
+ mkdir -p ci-logs
+ ./gradlew clean build --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/build.log
- name: Toolkit jar smoke test
run: |
@@ -256,10 +282,18 @@ jobs:
java -jar "$JAR" keystore --help
- name: Test with RocksDB engine
- run: ./gradlew :framework:testWithRocksDb --no-daemon
+ shell: bash
+ run: |
+ set -euo pipefail
+ mkdir -p ci-logs
+ ./gradlew :framework:testWithRocksDb --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/rocksdb-test.log
- name: Generate module coverage reports
- run: ./gradlew jacocoTestReport --no-daemon
+ shell: bash
+ run: |
+ set -euo pipefail
+ mkdir -p ci-logs
+ ./gradlew jacocoTestReport --no-daemon --console=plain 2>&1 | tee ci-logs/coverage.log
- name: Upload PR coverage reports
uses: actions/upload-artifact@v6
@@ -269,7 +303,7 @@ jobs:
**/build/reports/jacoco/test/jacocoTestReport.xml
if-no-files-found: error
- - name: Upload unit test logs
+ - name: Upload test diagnostics
if: always()
uses: actions/upload-artifact@v6
with:
@@ -278,6 +312,8 @@ jobs:
**/logs/tron-test.log
**/logs/tron-test-*.log.zip
**/build/test-results/**/TEST-*.xml
+ **/build/reports/tests/**
+ ci-logs/*.log
if-no-files-found: warn
retention-days: 7
@@ -324,14 +360,26 @@ jobs:
# this PR. The only output we need from this job is the jacoco XML for
# coverage diffing, so we must not let a stale test failure block it.
continue-on-error: true
- run: ./gradlew clean build --no-daemon
+ shell: bash
+ run: |
+ set -euo pipefail
+ mkdir -p ci-logs
+ ./gradlew clean build --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/build.log
- name: Test with RocksDB engine (base)
continue-on-error: true
- run: ./gradlew :framework:testWithRocksDb --no-daemon
+ shell: bash
+ run: |
+ set -euo pipefail
+ mkdir -p ci-logs
+ ./gradlew :framework:testWithRocksDb --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/rocksdb-test.log
- name: Generate module coverage reports (base)
- run: ./gradlew jacocoTestReport --no-daemon
+ shell: bash
+ run: |
+ set -euo pipefail
+ mkdir -p ci-logs
+ ./gradlew jacocoTestReport --no-daemon --console=plain 2>&1 | tee ci-logs/coverage.log
- name: Upload base coverage reports
uses: actions/upload-artifact@v6
@@ -341,7 +389,7 @@ jobs:
**/build/reports/jacoco/test/jacocoTestReport.xml
if-no-files-found: warn
- - name: Upload unit test logs
+ - name: Upload test diagnostics
if: always()
uses: actions/upload-artifact@v6
with:
@@ -350,6 +398,8 @@ jobs:
**/logs/tron-test.log
**/logs/tron-test-*.log.zip
**/build/test-results/**/TEST-*.xml
+ **/build/reports/tests/**
+ ci-logs/*.log
if-no-files-found: warn
retention-days: 7
From 376780b00d570ac3343988f6c3a3bfadf62bf30f Mon Sep 17 00:00:00 2001
From: 3for <287494524@qq.com>
Date: Wed, 9 Sep 2026 17:34:29 +0800
Subject: [PATCH 05/13] test: remove automatic retries from framework tests
Remove the test-retry plugin and retry configuration shared by test and
testWithRocksDb so test failures fail the task without retrying.
---
framework/build.gradle | 5 -----
1 file changed, 5 deletions(-)
diff --git a/framework/build.gradle b/framework/build.gradle
index 8edd6714f95..f090b9646db 100644
--- a/framework/build.gradle
+++ b/framework/build.gradle
@@ -1,5 +1,4 @@
plugins {
- id "org.gradle.test-retry" version "1.5.9"
id "org.sonarqube" version "2.6"
id "com.gorylenko.gradle-git-properties" version "2.4.1"
}
@@ -119,10 +118,6 @@ run {
}
def configureTestTask = { Task t ->
- t.retry {
- maxRetries = 5
- maxFailures = 20
- }
t.testLogging {
exceptionFormat = 'full'
}
From 679efc4f9e46fb585eb65c7f259e0fabb5f87283 Mon Sep 17 00:00:00 2001
From: 3for <287494524@qq.com>
Date: Wed, 9 Sep 2026 19:44:35 +0800
Subject: [PATCH 06/13] ci: upload only essential logs on failure
- Limit diagnostic artifacts to **/logs/tron-test.log and ci-logs/*.log
- Upload logs only when a preceding step fails
- Include base test failures tolerated by continue-on-error
- Keep the existing 7-day retention period
---
.github/workflows/pr-build.yml | 28 ++++++++--------------------
1 file changed, 8 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml
index b8056094ee1..ec301b26514 100644
--- a/.github/workflows/pr-build.yml
+++ b/.github/workflows/pr-build.yml
@@ -68,15 +68,12 @@ jobs:
java -jar "$JAR" keystore --help
- name: Upload test diagnostics
- if: always()
+ if: failure()
uses: actions/upload-artifact@v6
with:
name: tron-test-logs-${{ github.job }}-jdk${{ matrix.java }}-${{ matrix.arch }}
path: |
**/logs/tron-test.log
- **/logs/tron-test-*.log.zip
- **/build/test-results/**/TEST-*.xml
- **/build/reports/tests/**
ci-logs/*.log
if-no-files-found: warn
retention-days: 7
@@ -125,15 +122,12 @@ jobs:
java -jar "$JAR" keystore --help
- name: Upload test diagnostics
- if: always()
+ if: failure()
uses: actions/upload-artifact@v6
with:
name: tron-test-logs-${{ github.job }}
path: |
**/logs/tron-test.log
- **/logs/tron-test-*.log.zip
- **/build/test-results/**/TEST-*.xml
- **/build/reports/tests/**
ci-logs/*.log
if-no-files-found: warn
retention-days: 7
@@ -213,15 +207,12 @@ jobs:
./gradlew :framework:testWithRocksDb --no-daemon --console=plain 2>&1 | tee ci-logs/rocksdb-test.log
- name: Upload test diagnostics
- if: always()
+ if: failure()
uses: actions/upload-artifact@v6
with:
name: tron-test-logs-${{ github.job }}
path: |
**/logs/tron-test.log
- **/logs/tron-test-*.log.zip
- **/build/test-results/**/TEST-*.xml
- **/build/reports/tests/**
ci-logs/*.log
if-no-files-found: warn
retention-days: 7
@@ -304,15 +295,12 @@ jobs:
if-no-files-found: error
- name: Upload test diagnostics
- if: always()
+ if: failure()
uses: actions/upload-artifact@v6
with:
name: tron-test-logs-${{ github.job }}
path: |
**/logs/tron-test.log
- **/logs/tron-test-*.log.zip
- **/build/test-results/**/TEST-*.xml
- **/build/reports/tests/**
ci-logs/*.log
if-no-files-found: warn
retention-days: 7
@@ -355,6 +343,7 @@ jobs:
coverage-base-x86_64-gradle-
- name: Build (base)
+ id: base_build
# Test failures on the base branch are tolerated: merge-order races can
# leave the base with a pre-existing failing test that is unrelated to
# this PR. The only output we need from this job is the jacoco XML for
@@ -367,6 +356,7 @@ jobs:
./gradlew clean build --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/build.log
- name: Test with RocksDB engine (base)
+ id: base_rocksdb_test
continue-on-error: true
shell: bash
run: |
@@ -390,15 +380,13 @@ jobs:
if-no-files-found: warn
- name: Upload test diagnostics
- if: always()
+ # Preserve logs for test failures tolerated by continue-on-error above.
+ if: ${{ failure() || steps.base_build.outcome == 'failure' || steps.base_rocksdb_test.outcome == 'failure' }}
uses: actions/upload-artifact@v6
with:
name: tron-test-logs-${{ github.job }}
path: |
**/logs/tron-test.log
- **/logs/tron-test-*.log.zip
- **/build/test-results/**/TEST-*.xml
- **/build/reports/tests/**
ci-logs/*.log
if-no-files-found: warn
retention-days: 7
From 0ecbf2d2171e1cafa235ac7bc2c61ae4b10430c8 Mon Sep 17 00:00:00 2001
From: 3for <287494524@qq.com>
Date: Wed, 9 Sep 2026 22:13:07 +0800
Subject: [PATCH 07/13] ci: increase Rocky download concurrency and retain
retry config comments
- Set DNF max_parallel_downloads to 10 for Rocky dependency installation
- Restore the test-retry plugin and configuration as comments, keeping retries disabled
---
.github/workflows/pr-build.yml | 3 ++-
framework/build.gradle | 5 +++++
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml
index ec301b26514..9ba1cdf55fc 100644
--- a/.github/workflows/pr-build.yml
+++ b/.github/workflows/pr-build.yml
@@ -153,7 +153,8 @@ jobs:
# Rocky 8 already provides CA certificates, JNI runtime libraries, tar and gzip.
# Its built-in C.utf8 locale provides UTF-8 without an extra language pack.
# git-core provides checkout commands; zstd supports actions/cache compression.
- dnf -y --setopt=install_weak_deps=False install \
+ # Download more packages concurrently when mirror requests are slow.
+ dnf -y --setopt=install_weak_deps=False --setopt=max_parallel_downloads=10 install \
java-1.8.0-openjdk-devel git-core zstd
# Set JAVA_HOME so the Gradle wrapper does not need which.
javac_path=$(command -v javac)
diff --git a/framework/build.gradle b/framework/build.gradle
index f090b9646db..5e181f2f26d 100644
--- a/framework/build.gradle
+++ b/framework/build.gradle
@@ -1,4 +1,5 @@
plugins {
+ // id "org.gradle.test-retry" version "1.5.9"
id "org.sonarqube" version "2.6"
id "com.gorylenko.gradle-git-properties" version "2.4.1"
}
@@ -118,6 +119,10 @@ run {
}
def configureTestTask = { Task t ->
+ // t.retry {
+ // maxRetries = 5
+ // maxFailures = 20
+ // }
t.testLogging {
exceptionFormat = 'full'
}
From c1309b003f15e358c25e8595a1e2fa4af8beeaf8 Mon Sep 17 00:00:00 2001
From: 3for <287494524@qq.com>
Date: Fri, 18 Sep 2026 19:09:01 +0800
Subject: [PATCH 08/13] ci: abandon slow mirrors during Rocky dependency
installation
Set DNF minrate=256k and timeout=30 to abort persistently slow
connections and allow fallback to another mirror.
---
.github/workflows/pr-build.yml | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml
index 9ba1cdf55fc..e6063a3bf3a 100644
--- a/.github/workflows/pr-build.yml
+++ b/.github/workflows/pr-build.yml
@@ -153,9 +153,13 @@ jobs:
# Rocky 8 already provides CA certificates, JNI runtime libraries, tar and gzip.
# Its built-in C.utf8 locale provides UTF-8 without an extra language pack.
# git-core provides checkout commands; zstd supports actions/cache compression.
- # Download more packages concurrently when mirror requests are slow.
- dnf -y --setopt=install_weak_deps=False --setopt=max_parallel_downloads=10 install \
- java-1.8.0-openjdk-devel git-core zstd
+ # Abandon connections below 256 KiB/s for 30 seconds so DNF can try another mirror.
+ dnf -y \
+ --setopt=install_weak_deps=False \
+ --setopt=max_parallel_downloads=10 \
+ --setopt=minrate=256k \
+ --setopt=timeout=30 \
+ install java-1.8.0-openjdk-devel git-core zstd
# Set JAVA_HOME so the Gradle wrapper does not need which.
javac_path=$(command -v javac)
javac_real=$(readlink -f "$javac_path")
From e6a0c8610591201af39cd6322806695897ea7282 Mon Sep 17 00:00:00 2001
From: 3for <287494524@qq.com>
Date: Wed, 23 Sep 2026 11:50:15 +0800
Subject: [PATCH 09/13] ci: respect Gradle build cache configuration
Remove --no-build-cache from Debian and baseline coverage jobs
to preserve the cache behavior introduced in #6938.
---
.github/workflows/pr-build.yml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml
index e6063a3bf3a..8d951ae4abe 100644
--- a/.github/workflows/pr-build.yml
+++ b/.github/workflows/pr-build.yml
@@ -266,7 +266,7 @@ jobs:
run: |
set -euo pipefail
mkdir -p ci-logs
- ./gradlew clean build --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/build.log
+ ./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log
- name: Toolkit jar smoke test
run: |
@@ -282,7 +282,7 @@ jobs:
run: |
set -euo pipefail
mkdir -p ci-logs
- ./gradlew :framework:testWithRocksDb --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/rocksdb-test.log
+ ./gradlew :framework:testWithRocksDb --no-daemon --console=plain 2>&1 | tee ci-logs/rocksdb-test.log
- name: Generate module coverage reports
shell: bash
@@ -358,7 +358,7 @@ jobs:
run: |
set -euo pipefail
mkdir -p ci-logs
- ./gradlew clean build --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/build.log
+ ./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log
- name: Test with RocksDB engine (base)
id: base_rocksdb_test
@@ -367,7 +367,7 @@ jobs:
run: |
set -euo pipefail
mkdir -p ci-logs
- ./gradlew :framework:testWithRocksDb --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/rocksdb-test.log
+ ./gradlew :framework:testWithRocksDb --no-daemon --console=plain 2>&1 | tee ci-logs/rocksdb-test.log
- name: Generate module coverage reports (base)
shell: bash
From 430e431c546e72d792e906d05d0d6806ba48f940 Mon Sep 17 00:00:00 2001
From: 3for <287494524@qq.com>
Date: Sat, 10 Oct 2026 20:14:46 +0800
Subject: [PATCH 10/13] ci: improve diagnostic uploads on failure and
cancellation
- Upload diagnostics on failure or cancellation with a 4-minute timeout.
- Add step timeouts to reserve time for diagnostic uploads.
- Keep Gradle daemon shutdown failures non-fatal.
- Retain job-level timeouts for base tests to avoid accepting a partial coverage baseline after a tolerated step timeout.
- Preserve outcome checks for tolerated base test failures.
---
.github/workflows/pr-build.yml | 38 +++++++++++++++++++++++++++++-----
1 file changed, 33 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml
index 8d951ae4abe..dc64727a0c4 100644
--- a/.github/workflows/pr-build.yml
+++ b/.github/workflows/pr-build.yml
@@ -20,6 +20,8 @@ concurrency:
jobs:
+ # Bound build/test/report steps to leave time for diagnostics before the job limit.
+ # Uploads after job cancellation are best-effort only.
build-macos:
name: Build macos26 (JDK ${{ matrix.java }} / ${{ matrix.arch }})
if: ${{ github.event_name == 'pull_request' || inputs.job == 'all' || inputs.job == 'macos' }}
@@ -52,6 +54,7 @@ jobs:
restore-keys: macos26-${{ matrix.arch }}-gradle-
- name: Build
+ timeout-minutes: 45
shell: bash
run: |
set -euo pipefail
@@ -59,6 +62,7 @@ jobs:
./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log
- name: Toolkit jar smoke test
+ timeout-minutes: 2
run: |
set -e
JAR=plugins/build/libs/Toolkit.jar
@@ -68,7 +72,8 @@ jobs:
java -jar "$JAR" keystore --help
- name: Upload test diagnostics
- if: failure()
+ if: ${{ failure() || cancelled() }}
+ timeout-minutes: 4
uses: actions/upload-artifact@v6
with:
name: tron-test-logs-${{ github.job }}-jdk${{ matrix.java }}-${{ matrix.arch }}
@@ -106,6 +111,7 @@ jobs:
restore-keys: ubuntu24-aarch64-gradle-
- name: Build
+ timeout-minutes: 45
shell: bash
run: |
set -euo pipefail
@@ -113,6 +119,7 @@ jobs:
./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log
- name: Toolkit jar smoke test
+ timeout-minutes: 2
run: |
set -e
JAR=plugins/build/libs/Toolkit.jar
@@ -122,7 +129,8 @@ jobs:
java -jar "$JAR" keystore --help
- name: Upload test diagnostics
- if: failure()
+ if: ${{ failure() || cancelled() }}
+ timeout-minutes: 4
uses: actions/upload-artifact@v6
with:
name: tron-test-logs-${{ github.job }}
@@ -186,9 +194,12 @@ jobs:
rockylinux-x86_64-gradle-
- name: Stop Gradle daemon
+ timeout-minutes: 2
+ continue-on-error: true
run: ./gradlew --stop || true
- name: Build
+ timeout-minutes: 30
shell: bash
run: |
set -euo pipefail
@@ -196,6 +207,7 @@ jobs:
./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log
- name: Toolkit jar smoke test
+ timeout-minutes: 2
run: |
set -e
JAR=plugins/build/libs/Toolkit.jar
@@ -205,6 +217,7 @@ jobs:
java -jar "$JAR" keystore --help
- name: Test with RocksDB engine
+ timeout-minutes: 10
shell: bash
run: |
set -euo pipefail
@@ -212,7 +225,8 @@ jobs:
./gradlew :framework:testWithRocksDb --no-daemon --console=plain 2>&1 | tee ci-logs/rocksdb-test.log
- name: Upload test diagnostics
- if: failure()
+ if: ${{ failure() || cancelled() }}
+ timeout-minutes: 4
uses: actions/upload-artifact@v6
with:
name: tron-test-logs-${{ github.job }}
@@ -262,6 +276,7 @@ jobs:
debian11-x86_64-gradle-
- name: Build
+ timeout-minutes: 30
shell: bash
run: |
set -euo pipefail
@@ -269,6 +284,7 @@ jobs:
./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log
- name: Toolkit jar smoke test
+ timeout-minutes: 2
run: |
set -e
JAR=plugins/build/libs/Toolkit.jar
@@ -278,6 +294,7 @@ jobs:
java -jar "$JAR" keystore --help
- name: Test with RocksDB engine
+ timeout-minutes: 10
shell: bash
run: |
set -euo pipefail
@@ -285,6 +302,7 @@ jobs:
./gradlew :framework:testWithRocksDb --no-daemon --console=plain 2>&1 | tee ci-logs/rocksdb-test.log
- name: Generate module coverage reports
+ timeout-minutes: 5
shell: bash
run: |
set -euo pipefail
@@ -292,6 +310,7 @@ jobs:
./gradlew jacocoTestReport --no-daemon --console=plain 2>&1 | tee ci-logs/coverage.log
- name: Upload PR coverage reports
+ timeout-minutes: 2
uses: actions/upload-artifact@v6
with:
name: jacoco-coverage-pr
@@ -300,7 +319,8 @@ jobs:
if-no-files-found: error
- name: Upload test diagnostics
- if: failure()
+ if: ${{ failure() || cancelled() }}
+ timeout-minutes: 4
uses: actions/upload-artifact@v6
with:
name: tron-test-logs-${{ github.job }}
@@ -347,6 +367,8 @@ jobs:
restore-keys: |
coverage-base-x86_64-gradle-
+ # Use the job timeout for both base test steps. A tolerated step timeout
+ # could let incomplete coverage become the comparison baseline.
- name: Build (base)
id: base_build
# Test failures on the base branch are tolerated: merge-order races can
@@ -370,6 +392,7 @@ jobs:
./gradlew :framework:testWithRocksDb --no-daemon --console=plain 2>&1 | tee ci-logs/rocksdb-test.log
- name: Generate module coverage reports (base)
+ timeout-minutes: 5
shell: bash
run: |
set -euo pipefail
@@ -377,6 +400,7 @@ jobs:
./gradlew jacocoTestReport --no-daemon --console=plain 2>&1 | tee ci-logs/coverage.log
- name: Upload base coverage reports
+ timeout-minutes: 2
uses: actions/upload-artifact@v6
with:
name: jacoco-coverage-base
@@ -386,7 +410,11 @@ jobs:
- name: Upload test diagnostics
# Preserve logs for test failures tolerated by continue-on-error above.
- if: ${{ failure() || steps.base_build.outcome == 'failure' || steps.base_rocksdb_test.outcome == 'failure' }}
+ if: >-
+ failure() || cancelled() ||
+ steps.base_build.outcome == 'failure' ||
+ steps.base_rocksdb_test.outcome == 'failure'
+ timeout-minutes: 4
uses: actions/upload-artifact@v6
with:
name: tron-test-logs-${{ github.job }}
From e91b6d2ac9575f9d96988c194fd452f57f1ca27c Mon Sep 17 00:00:00 2001
From: 3for <287494524@qq.com>
Date: Tue, 13 Oct 2026 18:04:25 +0800
Subject: [PATCH 11/13] chore(build): remove unused test-retry verification
metadata
Remove verification entries for the disabled test-retry plugin
and its plugin marker artifact.
---
gradle/verification-metadata.xml | 16 ----------------
1 file changed, 16 deletions(-)
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
index a73c4715fe1..24e08e5cdd0 100644
--- a/gradle/verification-metadata.xml
+++ b/gradle/verification-metadata.xml
@@ -2238,22 +2238,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
From 8a0dd2c939eaedab23f13005d556c42a63315d70 Mon Sep 17 00:00:00 2001
From: 3for <287494524@qq.com>
Date: Mon, 12 Oct 2026 22:21:58 +0800
Subject: [PATCH 12/13] ci: add diagnostics for Gradle and JVM failures
- Wrap PR build, test, coverage and Toolkit stages with a shared runner
- Collect JVM crash logs, daemon logs, memory samples and OOM counters
- Record test classes and methods associated with each test executor
- Show failure evidence in Actions annotations and job summaries
- Upload stage-specific diagnostics and available JUnit XML on failure
- Preserve command exit codes when diagnostic logging fails
- Use PR diagnostic scripts for base coverage builds and retain logs
for tolerated failures
(cherry picked from commit 2f61ae1a7f28f0d935c22627be41748ded6cf6ac)
---
.github/ci/ci-run.sh | 607 +++++++++++++++++++++++++++
.github/ci/test-progress.init.gradle | 72 ++++
.github/workflows/pr-build.yml | 119 +++---
3 files changed, 739 insertions(+), 59 deletions(-)
create mode 100755 .github/ci/ci-run.sh
create mode 100644 .github/ci/test-progress.init.gradle
diff --git a/.github/ci/ci-run.sh b/.github/ci/ci-run.sh
new file mode 100755
index 00000000000..d8878ddfb54
--- /dev/null
+++ b/.github/ci/ci-run.sh
@@ -0,0 +1,607 @@
+#!/usr/bin/env bash
+# Run one CI stage and keep enough evidence to explain failures that have no
+# failing test case: JVM crashes, Java or kernel OOM, killed Gradle workers,
+# non-test Gradle task failures, timeouts and cancellations.
+#
+# Usage: ci-run.sh [args...]
+#
+# Evidence for each stage goes to its own directory, $CI_DIAG_ROOT/
+# (default: ./ci-logs/), so a later stage never reports an earlier
+# stage's crash files. Set CI_DIAG_ANNOTATION=warning for steps whose failures
+# are tolerated.
+#
+# The command's exit status is preserved: diagnostics never turn a failure
+# into a success or the other way around. The report separates what was
+# observed from what it may mean. Compatible with bash 3.2 (macOS runners) and
+# minimal images without findutils or procps.
+
+set -uo pipefail
+
+if [ "$#" -lt 2 ]; then
+ echo "usage: $0 [args...]" >&2
+ exit 2
+fi
+
+STAGE=$1
+shift
+case "$STAGE" in
+ *[!A-Za-z0-9_.-]* | '' | .*)
+ echo "invalid stage name: $STAGE" >&2
+ exit 2 ;;
+esac
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+ROOT=${CI_DIAG_ROOT:-$PWD/ci-logs}
+DIAG_DIR=$ROOT/$STAGE
+n=2
+while [ -e "$DIAG_DIR" ]; do
+ DIAG_DIR=$ROOT/$STAGE-$n
+ n=$((n + 1))
+done
+mkdir -p "$DIAG_DIR"
+DIAG_DIR=$(cd "$DIAG_DIR" && pwd)
+export CI_DIAG_DIR=$DIAG_DIR
+
+CONSOLE_LOG=$DIAG_DIR/console.log
+MEM_LOG=$DIAG_DIR/memory.log
+JVM_LOG=$DIAG_DIR/jvms.log
+JVM_LAST=$DIAG_DIR/jvms-last.txt
+STATUS_FILE=$DIAG_DIR/status.txt
+REPORT=$DIAG_DIR/diagnosis.md
+MARKER=$DIAG_DIR/.start
+SAMPLE_SECONDS=${CI_MEM_SAMPLE_SECONDS:-15}
+OS=$(uname -s)
+
+# Write HotSpot fatal error logs to this stage's directory. JAVA_TOOL_OPTIONS
+# reaches every JVM started below, including the single-use Gradle daemon,
+# Gradle test workers and compiler daemons.
+export JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:+$JAVA_TOOL_OPTIONS }-XX:ErrorFile=$DIAG_DIR/hs_err_pid%p.log"
+
+log() {
+ echo "[ci-diag] $*"
+}
+
+warn_diagnostics() {
+ local message=$1
+ log "warning: $message" >&2
+ if [ -n "${GITHUB_ACTIONS:-}" ]; then
+ printf '::warning title=CI diagnostics (%s)::%s\n' "$STAGE" "$message"
+ fi
+}
+
+now() {
+ date -u +%H:%M:%S
+}
+
+# ------------------------------------------------------------------ cgroups
+
+# Resolve this shell's memory cgroup through /proc/self/mountinfo so nested
+# cgroups and non-root mounts map to the right directory. Leaves CG_DIR empty
+# when the mapping is ambiguous; the report then says "unavailable" rather
+# than reading an unrelated cgroup.
+CG_DIR=""
+CG_VERSION=""
+
+cgroup_path_under_mount() {
+ local path=$1 root=$2 mnt=$3
+ if [ "$path" = "$root" ]; then
+ echo "$mnt"
+ elif [ "$root" = / ]; then
+ echo "$mnt$path"
+ else
+ case "$path" in
+ "$root"/*) echo "$mnt${path#"$root"}" ;;
+ esac
+ fi
+}
+
+resolve_cgroup() {
+ [ -r /proc/self/cgroup ] && [ -r /proc/self/mountinfo ] || return 0
+ local path mount dir
+ path=$(awk -F: '$1 == "0" && $2 == "" { print $3; exit }' /proc/self/cgroup)
+ if [ -n "$path" ]; then
+ mount=$(awk '{ for (i = 7; i <= NF; i++) if ($i == "-") {
+ if ($(i + 1) == "cgroup2") print $4, $5; break } }' /proc/self/mountinfo | head -1)
+ if [ -n "$mount" ]; then
+ dir=$(cgroup_path_under_mount "$path" "${mount%% *}" "${mount#* }")
+ if [ -n "$dir" ] && [ -f "$dir/memory.current" ]; then
+ CG_DIR=$dir
+ CG_VERSION=2
+ return 0
+ fi
+ fi
+ fi
+ # cgroup v1, or a hybrid host whose v2 hierarchy has no memory controller.
+ path=$(awk -F: '$2 ~ /(^|,)memory(,|$)/ { print $3; exit }' /proc/self/cgroup)
+ [ -n "$path" ] || return 0
+ mount=$(awk '{ for (i = 7; i <= NF; i++) if ($i == "-") {
+ if ($(i + 1) == "cgroup" && $(i + 3) ~ /(^|,)memory(,|$)/) print $4, $5; break } }' \
+ /proc/self/mountinfo | head -1)
+ [ -n "$mount" ] || return 0
+ dir=$(cgroup_path_under_mount "$path" "${mount%% *}" "${mount#* }")
+ if [ -n "$dir" ] && [ -f "$dir/memory.usage_in_bytes" ]; then
+ CG_DIR=$dir
+ CG_VERSION=1
+ fi
+}
+
+read_mb() {
+ awk '{ if ($1 == "max" || $1 >= 9e18) print "max"; else print int($1 / 1048576) "M" }' \
+ "$1" 2>/dev/null
+}
+
+cg_usage() {
+ case "$CG_VERSION" in
+ 2) read_mb "$CG_DIR/memory.current" ;;
+ 1) read_mb "$CG_DIR/memory.usage_in_bytes" ;;
+ *) echo "-" ;;
+ esac
+}
+
+cg_limit() {
+ case "$CG_VERSION" in
+ 2) read_mb "$CG_DIR/memory.max" ;;
+ 1) read_mb "$CG_DIR/memory.limit_in_bytes" ;;
+ *) echo "-" ;;
+ esac
+}
+
+cg_oom_kill() {
+ local v=""
+ case "$CG_VERSION" in
+ 2) v=$(awk '$1 == "oom_kill" { print $2 }' "$CG_DIR/memory.events" 2>/dev/null) ;;
+ 1) v=$(awk '$1 == "oom_kill" { print $2 }' "$CG_DIR/memory.oom_control" 2>/dev/null) ;;
+ esac
+ echo "${v:--}"
+}
+
+# Kernel-wide counter: on a hosted runner it covers the whole VM, so it may
+# include processes outside this job.
+host_oom_kill() {
+ local v
+ v=$(awk '$1 == "oom_kill" { print $2 }' /proc/vmstat 2>/dev/null)
+ echo "${v:--}"
+}
+
+snapshot_memory() {
+ local f
+ {
+ echo "== $1 $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+ if [ "$OS" = Linux ]; then
+ grep -E '^(MemTotal|MemAvailable|SwapTotal|SwapFree):' /proc/meminfo
+ echo "host oom_kill: $(host_oom_kill)"
+ if [ -n "$CG_DIR" ]; then
+ echo "cgroup v$CG_VERSION: $CG_DIR"
+ for f in memory.max memory.current memory.peak memory.events memory.swap.max \
+ memory.swap.current memory.limit_in_bytes memory.usage_in_bytes \
+ memory.max_usage_in_bytes memory.failcnt memory.oom_control; do
+ [ -r "$CG_DIR/$f" ] && echo "$f: $(tr '\n' ' ' < "$CG_DIR/$f")"
+ done
+ else
+ echo "cgroup: unavailable (memory cgroup of this process could not be resolved)"
+ fi
+ else
+ sysctl -n hw.memsize 2>/dev/null | awk '{ printf "hw.memsize: %d MB\n", $1 / 1048576 }'
+ sysctl vm.swapusage 2>/dev/null
+ memory_pressure 2>/dev/null | tail -1
+ fi
+ } >> "$MEM_LOG" 2>&1
+}
+
+# ----------------------------------------------------------------- JVM list
+
+java_role() {
+ case "$1" in
+ *"Gradle Test Executor"*)
+ echo "$1" | grep -o 'Gradle Test Executor [0-9]*' | head -1 | tr ' ' '_' ;;
+ *GradleDaemon*) echo GradleDaemon ;;
+ *GradleWrapperMain*) echo GradleWrapper ;;
+ *GradleWorkerMain*) echo GradleWorker ;;
+ *KotlinCompileDaemon*) echo KotlinDaemon ;;
+ *) echo java ;;
+ esac
+}
+
+# Prints " " for every running JVM.
+java_processes() {
+ if [ "$OS" = Linux ]; then
+ local d rss cmd
+ for d in /proc/[0-9]*; do
+ [ "$(cat "$d/comm" 2>/dev/null)" = java ] || continue
+ rss=$(awk '/^VmRSS:/ { print $2 }' "$d/status" 2>/dev/null)
+ cmd=$(tr '\0' ' ' < "$d/cmdline" 2>/dev/null)
+ echo "${d#/proc/} ${rss:-0} $(java_role "$cmd")"
+ done
+ else
+ ps -axo pid=,rss=,command= 2>/dev/null | while read -r pid rss cmd; do
+ case "$cmd" in
+ */bin/java\ * | java\ *) echo "$pid $rss $(java_role "$cmd")" ;;
+ esac
+ done
+ fi
+}
+
+memory_sampler() {
+ local prev=$DIAG_DIR/.jvms.prev cur=$DIAG_DIR/.jvms.cur i=0 ts line every
+ every=$((60 / SAMPLE_SECONDS))
+ [ "$every" -gt 0 ] || every=1
+ : > "$prev"
+ while :; do
+ ts=$(now)
+ java_processes | LC_ALL=C sort > "$cur"
+ # Full PID/role/RSS map on every sample, so a JVM that vanishes can be
+ # identified even if it died right after the previous sample.
+ awk -v ts="$ts" '{ printf "%s %s %s %dM\n", ts, $1, $3, $2 / 1024 }' "$cur" >> "$JVM_LOG"
+ { echo "# $ts"; cat "$cur"; } > "$JVM_LAST"
+ LC_ALL=C join -v1 "$prev" "$cur" 2>/dev/null | while read -r pid rss role; do
+ echo "$ts EXIT pid=$pid role=$role last_rss=$((rss / 1024))M" >> "$MEM_LOG"
+ done
+ line="$ts"
+ if [ "$OS" = Linux ]; then
+ line="$line avail=$(awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo)M"
+ line="$line cgroup=$(cg_usage)/$(cg_limit) cgroup_oom_kill=$(cg_oom_kill)"
+ line="$line host_oom_kill=$(host_oom_kill)"
+ fi
+ line="$line jvm_total=$(awk '{ s += $2 } END { print int(s / 1024) }' "$cur")M"
+ line="$line top=$(sort -k2 -nr "$cur" | head -4 \
+ | awk '{ printf "%s%s:%dM", sep, $3, $2 / 1024; sep = "," }')"
+ echo "$line" >> "$MEM_LOG"
+ # Stream a trend line to the job log once a minute; it survives even when
+ # the artifact upload never runs.
+ [ $((i % every)) -eq 0 ] && log "mem $line"
+ i=$((i + 1))
+ mv "$cur" "$prev"
+ sleep "$SAMPLE_SECONDS"
+ done
+}
+
+# --------------------------------------------------------------- diagnosis
+
+collect_files() {
+ local f gradle_home=${GRADLE_USER_HOME:-$HOME/.gradle}
+ # Fallback locations HotSpot uses when ErrorFile cannot be written.
+ for f in hs_err_pid*.log */hs_err_pid*.log /tmp/hs_err_pid*.log; do
+ [ -f "$f" ] && [ "$f" -nt "$MARKER" ] && cp -p "$f" "$DIAG_DIR/" 2>/dev/null
+ done
+ # Daemon logs of this stage only; tail them because they repeat build output.
+ for f in "$gradle_home"/daemon/*/daemon-*.out.log; do
+ [ -f "$f" ] && [ "$f" -nt "$MARKER" ] \
+ && tail -n 20000 "$f" > "$DIAG_DIR/gradle-$(basename "$f")"
+ done
+}
+
+signal_name() {
+ case "$1" in
+ 6) echo SIGABRT ;;
+ 9) echo SIGKILL ;;
+ 11) echo SIGSEGV ;;
+ 15) echo SIGTERM ;;
+ *) echo "signal $1" ;;
+ esac
+}
+
+exit_text() {
+ if [ "$1" -gt 128 ] && [ "$1" -lt 160 ]; then
+ echo "$1 (128+$(($1 - 128)), $(signal_name $(($1 - 128))))"
+ else
+ echo "$1"
+ fi
+}
+
+# One line per fatal-error log: owning JVM role, fatal reason, problematic frame.
+hs_err_summary() {
+ local f role
+ for f in "$DIAG_DIR"/hs_err_pid*.log; do
+ [ -f "$f" ] || continue
+ role=$(java_role "$(grep -m1 '^Command Line:' "$f" 2>/dev/null)")
+ awk -v file="${f##*/}" -v role="$role" '
+ !/^#/ { exit }
+ frame_next { frame = substr($0, 3); frame_next = 0; next }
+ /^# Problematic frame:/ { frame_next = 1; next }
+ /^# (Possible| )/ { next }
+ /^# [^ ]/ || /^# (There is insufficient memory|Native memory allocation|Out of Memory Error)/ {
+ if (n < 2) { sub(/^# +/, ""); reason = reason (n++ ? "; " : "") $0 }
+ }
+ END {
+ printf "%s [%s]: %s%s\n", file, role, (reason == "" ? "reason not found" : reason),
+ (frame == "" ? "" : "; frame: " frame)
+ }' "$f"
+ done
+}
+
+diagnose() {
+ local rc=$1 reason=$2 observed="" hints="" f n
+ local cg_after host_after hs_errs workers exits roles oom_lines wrong unfinished samples
+ local killed=0 cg_oom=0 host_oom=0 cg_known=0 daemon_gone=0 evidence=0
+
+ collect_files
+ snapshot_memory "after ($reason)"
+
+ obs() { observed="${observed}- $*"$'\n'; }
+ hint() { hints="${hints}- $*"$'\n'; }
+
+ obs "Command exit status: $(exit_text "$rc")."
+ [ "$rc" -eq 137 ] && killed=1
+ if [ "$reason" = interrupted ]; then
+ obs "The wrapper received $SIGNAL_SEEN before the command finished."
+ evidence=1
+ fi
+
+ if [ "$OS" = Linux ]; then
+ cg_after=$(cg_oom_kill)
+ host_after=$(host_oom_kill)
+ if [ -z "$CG_DIR" ]; then
+ obs "cgroup OOM kills: unavailable (memory cgroup not resolvable)."
+ elif [ "$CG_OOM_BEFORE" = - ] || [ "$cg_after" = - ]; then
+ obs "cgroup OOM kills: counter not exposed by cgroup v$CG_VERSION."
+ else
+ cg_known=1
+ obs "cgroup OOM kills during stage: $((cg_after - CG_OOM_BEFORE)) ($CG_DIR)."
+ [ "$cg_after" -gt "$CG_OOM_BEFORE" ] && cg_oom=1
+ fi
+ if [ "$HOST_OOM_BEFORE" != - ] && [ "$host_after" != - ]; then
+ obs "Kernel-wide OOM kills during stage: $((host_after - HOST_OOM_BEFORE)) (may include processes outside this job)."
+ [ "$host_after" -gt "$HOST_OOM_BEFORE" ] && host_oom=1
+ fi
+ fi
+
+ hs_errs=$(hs_err_summary)
+ if [ -n "$hs_errs" ]; then
+ obs "JVM fatal error log(s): $(echo "$hs_errs" | paste -sd ';' -)."
+ evidence=1
+ fi
+
+ workers=$(grep -oE "Process 'Gradle Test Executor [0-9]+' finished with non-zero exit value [0-9]+" \
+ "$CONSOLE_LOG" 2>/dev/null | LC_ALL=C sort -u | head -5)
+ if [ -n "$workers" ]; then
+ obs "Gradle test worker(s) exited abnormally: $(echo "$workers" | sed "s/^Process '//; s/' finished with non-zero exit value / -> exit /" | paste -sd ';' -)."
+ echo "$workers" | grep -q 'exit value 137$' && killed=1
+ evidence=1
+ fi
+ if grep -q 'daemon disappeared unexpectedly' "$CONSOLE_LOG" 2>/dev/null; then
+ obs "Gradle reported that its daemon disappeared unexpectedly."
+ daemon_gone=1
+ evidence=1
+ fi
+
+ # Test workers are recycled routinely (forkEvery), so prefer exits of the
+ # abnormal workers and of the Gradle daemon over the normal rotation.
+ roles=$( { echo "$workers" | grep -oE 'Executor [0-9]+' | sed 's/^Executor /Gradle_Test_Executor_/'; echo GradleDaemon; } \
+ | sed 's/.*/role=&( |$)/' | paste -sd '|' -)
+ exits=$(grep ' EXIT ' "$MEM_LOG" 2>/dev/null | grep -E "$roles" | tail -5)
+ if [ -n "$exits" ]; then
+ obs "Sampler saw these JVMs disappear: $(echo "$exits" | sed 's/ EXIT / /' | paste -sd ';' -)."
+ else
+ exits=$(grep ' EXIT ' "$MEM_LOG" 2>/dev/null | tail -5)
+ [ -n "$exits" ] && obs "Last JVM exits seen by the sampler (routine worker recycling also appears here): $(echo "$exits" | sed 's/ EXIT / /' | paste -sd ';' -)."
+ fi
+
+ oom_lines=$(grep -c 'OutOfMemoryError' "$CONSOLE_LOG" 2>/dev/null)
+ if [ "${oom_lines:-0}" -gt 0 ]; then
+ obs "Console lines mentioning OutOfMemoryError: $oom_lines, e.g. \`$(grep -m1 'OutOfMemoryError' "$CONSOLE_LOG" | cut -c1-160)\`."
+ evidence=1
+ fi
+
+ if [ "$IS_GRADLE" -eq 1 ]; then
+ if grep -qE '^BUILD (SUCCESSFUL|FAILED)' "$CONSOLE_LOG" 2>/dev/null; then
+ obs "Gradle completion marker: $(grep -E '^BUILD (SUCCESSFUL|FAILED)' "$CONSOLE_LOG" | tail -1)."
+ else
+ obs "Gradle completion marker (BUILD SUCCESSFUL/FAILED) not observed; read the exit status and console log together."
+ fi
+ fi
+
+ wrong=$(awk '/^\* What went wrong:/ { on = 1; next } /^\* Try:/ { on = 0 }
+ on && NF { print }' "$CONSOLE_LOG" 2>/dev/null | head -15)
+
+ unfinished=$(unfinished_suites | head -10)
+ [ -n "$unfinished" ] && obs "Test classes with a start but no end record: $(echo "$unfinished" | paste -sd ';' -)."
+
+ [ "$killed" -eq 1 ] || [ "$cg_oom" -eq 1 ] || [ "$host_oom" -eq 1 ] && evidence=1
+
+ # Interpretation, kept separate from the facts above. Only this job's cgroup
+ # counter ties an OOM kill to this stage; the kernel-wide one may not.
+ if [ "$cg_oom" -eq 1 ] && [ "$killed" -eq 1 ]; then
+ hint "A process was SIGKILLed and the OOM killer fired in this job's cgroup: memory exhaustion is the likely cause. Check memory.log and jvms.log around the EXIT time."
+ elif [ "$cg_oom" -eq 1 ]; then
+ hint "The OOM killer killed a process in this job's cgroup during the stage; which process is not confirmed. A killed Gradle daemon or worker would explain a plain failure exit; check the EXIT records in memory.log and jvms.log."
+ elif [ "$host_oom" -eq 1 ] && [ "$cg_known" -eq 1 ]; then
+ hint "The kernel OOM killer fired outside this job's cgroup; it may be unrelated to this failure."
+ elif [ "$host_oom" -eq 1 ]; then
+ hint "The kernel OOM killer fired, but this job's cgroup could not be checked, so its link to this failure is unconfirmed."
+ fi
+ if [ "$killed" -eq 1 ] && [ "$cg_oom" -eq 0 ] && [ "$host_oom" -eq 0 ]; then
+ if [ "$cg_known" -eq 1 ]; then
+ hint "SIGKILL without OOM-killer evidence in this cgroup: the sender is unknown (runner, timeout handling or another process)."
+ else
+ hint "SIGKILL observed, but OOM-killer counters were not available to confirm or rule out OOM."
+ fi
+ fi
+ [ -n "$hs_errs" ] && hint "A JVM hit a fatal error; the reason and problematic frame above come from its hs_err header."
+ [ "$daemon_gone" -eq 1 ] && [ -z "$hs_errs" ] && [ "$cg_oom" -eq 0 ] \
+ && hint "The Gradle daemon vanished without a fatal-error log or cgroup OOM kill: an external kill or a crash that could not write hs_err."
+ [ "$reason" = interrupted ] && hint "Signals usually come from a step timeout or run cancellation."
+ [ -n "$workers" ] && [ -z "$hs_errs" ] && [ "$killed" -eq 0 ] \
+ && hint "A test worker exited with a non-zero status without a fatal-error log: possible System.exit/halt in a test, or a crash that could not write hs_err."
+ [ "${oom_lines:-0}" -gt 0 ] \
+ && hint "OutOfMemoryError text is a clue only; confirm which JVM threw it and whether it was caught by a test."
+ if [ "$evidence" -eq 0 ] && [ "$IS_GRADLE" -eq 1 ]; then
+ hint "No crash, OOM-kill, signal or worker-exit evidence was found; the failing Gradle task in console.log is the next place to look."
+ elif [ "$evidence" -eq 0 ]; then
+ hint "No crash, OOM-kill, signal or worker-exit evidence was found; console.log shows the failing command's output."
+ elif [ -z "$hints" ]; then
+ hint "No single cause can be inferred; read the observed facts together."
+ fi
+
+ {
+ echo "### CI diagnosis: \`$STAGE\`"
+ echo
+ echo "**Observed**"
+ echo
+ printf '%s' "$observed"
+ if [ -n "$wrong" ]; then
+ echo
+ echo "Gradle \"What went wrong\":"
+ echo '```text'
+ echo "$wrong"
+ echo '```'
+ fi
+ if [ -n "$workers" ]; then
+ echo
+ echo "Last test records of the abnormal worker(s); a worker crash can still"
+ echo "produce end events, so read these together with the exit status:"
+ echo '```text'
+ echo "$workers" | grep -oE 'Executor [0-9]+' | awk '{ print $2 }' | while read -r n; do
+ recent_records "Gradle_Test_Executor_$n"
+ done
+ echo '```'
+ fi
+ samples=$(grep -E '^[0-9]{2}:[0-9]{2}:[0-9]{2} ' "$MEM_LOG" | tail -6)
+ if [ -n "$samples" ]; then
+ echo
+ echo "Last memory samples:"
+ echo '```text'
+ echo "$samples"
+ echo '```'
+ fi
+ echo
+ echo "**Possible causes (inferred, not confirmed)**"
+ echo
+ printf '%s' "$hints"
+ echo
+ echo "Evidence: artifact \`tron-test-logs-*\` → \`ci-logs/${DIAG_DIR##*/}/\`."
+ } > "$REPORT"
+
+ [ -n "${GITHUB_STEP_SUMMARY:-}" ] && cat "$REPORT" >> "$GITHUB_STEP_SUMMARY"
+ # One annotation, shown on the run page and in the PR checks list.
+ if [ -n "${GITHUB_ACTIONS:-}" ]; then
+ echo "::${CI_DIAG_ANNOTATION:-error} title=CI diagnosis ($STAGE)::$(printf '%sPossible causes:\n%s' \
+ "$observed" "$hints" | sed 's/%/%25/g' | awk '{ printf "%s%%0A", $0 }')"
+ fi
+ {
+ echo "== diagnosis"
+ printf 'Observed:\n%sPossible causes:\n%s' "$observed" "$hints"
+ } >> "$STATUS_FILE"
+ log "diagnosis written to $REPORT"
+ printf 'Observed:\n%sPossible causes:\n%s' "$observed" "$hints"
+}
+
+# Test progress records: "