From deaf0ba565a9b1031875ec4ba620814205457ebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Dinis=20Ferreira?= Date: Wed, 6 May 2026 23:09:30 +0200 Subject: [PATCH 1/2] ci: consolidate pmd, checkstyle, spotbugs into one static-analysis job with inline PR annotations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the separate pmd and checkstyle jobs with a single static-analysis job. Step 1 compiles and generates PMD/Checkstyle reports (no *:check goals → no Maven cascade-skip; every module's XML is produced). Step 2 is a Python pass that emits GitHub workflow-command annotations from each violation and exits 1 if any are found — fail-fast that bypasses SpotBugs (slow) when there are PMD/Checkstyle issues. Step 3 re-runs Maven `pmd:check pmd:cpd-check checkstyle:check` as a safety-net for official-tool validation. Steps 4-6 mirror the same pattern for SpotBugs (only run if PMD/Checkstyle are clean). Drops the now-redundant pmd/checkstyle/spotbugs goals from maven-verify (now just `mvn clean verify`). line-endings stays separate. Uses `-T 2C` (8 worker threads on a 4 vCPU runner; benchmark on this branch showed ~25% wall-clock saving over sequential). Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/scripts/annotate-violations.py | 100 +++++++++++++++++++++++++ .github/workflows/verify.yml | 66 +++++++++++----- 2 files changed, 148 insertions(+), 18 deletions(-) create mode 100755 .github/scripts/annotate-violations.py diff --git a/.github/scripts/annotate-violations.py b/.github/scripts/annotate-violations.py new file mode 100755 index 000000000..82a39b643 --- /dev/null +++ b/.github/scripts/annotate-violations.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Annotate PMD, Checkstyle, and SpotBugs violations as GitHub workflow commands. + +Walks `target/` reports under GITHUB_WORKSPACE, emits one `::warning` or `::error` +per violation (rendered inline on the PR's Files-changed view), and exits 1 if any +violation was found so the workflow step fails fast. +""" +import argparse +import os +import sys +from pathlib import Path +from xml.etree import ElementTree as ET + + +def make_emit(root: Path): + count = 0 + + def emit(level: str, file, line, title: str, msg: str | None) -> None: + nonlocal count + count += 1 + text = (msg or '').strip().replace('\n', ' ')[:1000] + try: + rel = Path(file).relative_to(root) + except ValueError: + rel = file + print(f"::{level} file={rel},line={line},title={title}::{text}") + + return emit, lambda: count + + +def parse_pmd(root: Path, emit) -> None: + for xml in root.rglob('target/pmd.xml'): + for f in ET.parse(xml).getroot().findall('file'): + for v in f.findall('violation'): + level = 'error' if v.attrib.get('priority') == '1' else 'warning' + emit(level, f.attrib['name'], v.attrib.get('beginline', '1'), + f"PMD/{v.attrib.get('rule', '')}", v.text) + + +def parse_checkstyle(root: Path, emit) -> None: + for xml in root.rglob('target/checkstyle-result.xml'): + for f in ET.parse(xml).getroot().findall('file'): + for e in f.findall('error'): + level = 'error' if e.attrib.get('severity') == 'error' else 'warning' + emit(level, f.attrib['name'], e.attrib.get('line', '1'), + f"Checkstyle/{e.attrib.get('source', '').split('.')[-1]}", + e.attrib.get('message', '')) + + +def parse_spotbugs(root: Path, emit) -> None: + for xml in root.rglob('target/spotbugsXml.xml'): + # SpotBugs sourcepath is package-relative (e.g. "com/avaloq/tools/ddk/Foo.java"), + # not repo-relative — combine with the module's source root so GitHub renders the + # annotation inline on the file in the PR's Files-changed view. + module_dir = xml.parent.parent + for b in ET.parse(xml).getroot().findall('BugInstance'): + sl = b.find('SourceLine') + lm = b.find('LongMessage') + if sl is None or lm is None: + continue + sourcepath = sl.attrib.get('sourcepath', '?') + file_path = None + for src_root in ('src', 'src/main/java', 'src-gen'): + candidate = module_dir / src_root / sourcepath + if candidate.exists(): + file_path = candidate + break + if file_path is None: + file_path = module_dir / sourcepath + emit('warning', file_path, sl.attrib.get('start', '1'), + f"SpotBugs/{b.attrib.get('type', '')}", lm.text) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--pmd', action='store_true', help='annotate PMD violations') + parser.add_argument('--checkstyle', action='store_true', help='annotate Checkstyle violations') + parser.add_argument('--spotbugs', action='store_true', help='annotate SpotBugs violations') + args = parser.parse_args() + + if not (args.pmd or args.checkstyle or args.spotbugs): + parser.error('pick at least one of --pmd, --checkstyle, --spotbugs') + + root = Path(os.environ.get('GITHUB_WORKSPACE', '.')) + emit, total = make_emit(root) + + if args.pmd: + parse_pmd(root, emit) + if args.checkstyle: + parse_checkstyle(root, emit) + if args.spotbugs: + parse_spotbugs(root, emit) + + kinds = ' + '.join(k for k, on in (('PMD', args.pmd), ('Checkstyle', args.checkstyle), ('SpotBugs', args.spotbugs)) if on) + print(f"{kinds} violations: {total()}") + return 1 if total() > 0 else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index c88269363..c6d904ba0 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -4,7 +4,7 @@ on: branches: [master] pull_request: jobs: - pmd: + static-analysis: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -14,20 +14,52 @@ jobs: java-version: '21' - name: Set up Workspace Environment Variable run: echo "WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV - - name: PMD Check - run: mvn pmd:pmd pmd:cpd pmd:check pmd:cpd-check -f ./ddk-parent/pom.xml --batch-mode --fail-at-end - checkstyle: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + - name: Cache Maven dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: - distribution: 'temurin' - java-version: '21' - - name: Set up Workspace Environment Variable - run: echo "WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV - - name: Checkstyle Check - run: mvn checkstyle:checkstyle checkstyle:check -f ./ddk-parent/pom.xml --batch-mode --fail-at-end + path: /home/runner/.m2/repository + key: ${{ runner.os }}-maven-0-${{ hashFiles('**/pom.xml') }} + - name: Compile + generate PMD/Checkstyle reports + # Report goals (pmd:pmd, pmd:cpd, checkstyle:checkstyle) never fail the build, so + # every module's XML is produced — no Maven cascade-skip. `compile` runs in the + # same invocation so PMD's type-resolving rules (e.g. InvalidLogMessageFormat) get + # the aux-classpath they need; without it, those rules misfire on idioms like + # SLF4J's trailing-Throwable pattern. + run: | + mvn -T 2C -f ./ddk-parent/pom.xml --batch-mode --fail-at-end \ + compile \ + pmd:pmd pmd:cpd \ + checkstyle:checkstyle + - name: Annotate + count PMD/Checkstyle violations + # Emits one GitHub workflow-command annotation per violation (visible inline in + # PR Files-changed) and exits 1 if any are found — fail-fast that bypasses + # SpotBugs (slow) when PMD/Checkstyle already has issues. Runs even if a previous + # step failed. + if: always() + run: python3 .github/scripts/annotate-violations.py --pmd --checkstyle + - name: Enforce PMD/Checkstyle thresholds (Maven safety-net) + # Redundant in the success case (Python check above already passed). Provides + # official-tool validation in case the Python parser misreads schema. `compile` + # is required in this invocation too — pmd:check re-runs analysis from scratch + # rather than reading the prior pmd.xml, so it needs its own aux-classpath. + run: | + mvn -T 2C -f ./ddk-parent/pom.xml --batch-mode --fail-at-end \ + compile \ + pmd:check pmd:cpd-check \ + checkstyle:check + - name: Generate SpotBugs report + run: | + mvn -T 2C -f ./ddk-parent/pom.xml --batch-mode --fail-at-end \ + compile \ + spotbugs:spotbugs + - name: Annotate + count SpotBugs violations + if: always() + run: python3 .github/scripts/annotate-violations.py --spotbugs + - name: Enforce SpotBugs threshold (Maven safety-net) + run: | + mvn -T 2C -f ./ddk-parent/pom.xml --batch-mode --fail-at-end \ + compile \ + spotbugs:check line-endings: runs-on: ubuntu-24.04 steps: @@ -64,10 +96,8 @@ jobs: with: path: /home/runner/.m2/repository key: ${{ runner.os }}-maven-0-${{ hashFiles('**/pom.xml') }} - - name: Build with Maven within a virtual X Server Environment - # Run pmd:pmd and pmd:cpd first to generate reports for all modules, then run pmd:check and pmd:cpd-check - # This ensures all violations are collected and reported before the build fails - run: xvfb-run mvn clean verify checkstyle:check pmd:pmd pmd:cpd pmd:check pmd:cpd-check spotbugs:check -f ./ddk-parent/pom.xml --batch-mode --fail-at-end + - name: Build with Maven within a virtual X Server Environment + run: xvfb-run mvn clean verify -f ./ddk-parent/pom.xml --batch-mode --fail-at-end - name: Fail on missing surefire reports # Tycho-Surefire writes no TEST-*.xml when discovery is empty — fail the job in that case. if: always() From 78c331745db6d6ff748506b1f6737a89a78c674c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Dinis=20Ferreira?= Date: Thu, 7 May 2026 11:10:00 +0200 Subject: [PATCH 2/2] ci: split static-analysis into 4 parallel jobs (pmd-cs, spotbugs, gate, verify) Replaces the single static-analysis job with four sibling jobs running in parallel: pmd-checkstyle-analyze (compile + pmd:pmd + cs:checkstyle + python annotate), spotbugs-analyze (compile + spotbugs + python annotate, MAVEN_OPTS=-Xmx4g), maven-gate (compile + all :check goals authoritatively, MAVEN_OPTS=-Xmx4g), and maven-verify (clean verify with tests, no static-analysis). The two analyze jobs each use -T 2C; tests stay single-threaded since the Tycho test reactor isn't safe to parallelize. Goal: cut wall-clock from ~32m (sequential) to ~max-of-the-jobs by running independent work on independent runners. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/verify.yml | 109 ++++++++++++++++++++++------------- 1 file changed, 70 insertions(+), 39 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index c6d904ba0..f9ca9ea66 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -4,7 +4,25 @@ on: branches: [master] pull_request: jobs: - static-analysis: + line-endings: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Check LF line endings in index + # .gitattributes declares `* text=auto eol=lf` with .bat/.cmd/.ps1 + # exempted. Git's clean filter normalizes on commit, but verify it + # explicitly in case a file is miscategorized as binary or a filter + # is bypassed. + run: | + violations=$(git ls-files --eol \ + | grep -E "^i/(crlf|mixed)" \ + | grep -vE "\.(bat|cmd|ps1)$" || true) + if [ -n "$violations" ]; then + echo "Files with CRLF/mixed line endings stored in the index:" + echo "$violations" + exit 1 + fi + pmd-checkstyle-analyze: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -20,11 +38,10 @@ jobs: path: /home/runner/.m2/repository key: ${{ runner.os }}-maven-0-${{ hashFiles('**/pom.xml') }} - name: Compile + generate PMD/Checkstyle reports - # Report goals (pmd:pmd, pmd:cpd, checkstyle:checkstyle) never fail the build, so - # every module's XML is produced — no Maven cascade-skip. `compile` runs in the - # same invocation so PMD's type-resolving rules (e.g. InvalidLogMessageFormat) get - # the aux-classpath they need; without it, those rules misfire on idioms like - # SLF4J's trailing-Throwable pattern. + # Report goals never fail the build, so every module's XML is produced — no + # Maven cascade-skip. `compile` runs in the same invocation so PMD's + # type-resolving rules (e.g. InvalidLogMessageFormat) get the aux-classpath they + # need; without it, those rules misfire on idioms like SLF4J's trailing-Throwable. run: | mvn -T 2C -f ./ddk-parent/pom.xml --batch-mode --fail-at-end \ compile \ @@ -32,22 +49,28 @@ jobs: checkstyle:checkstyle - name: Annotate + count PMD/Checkstyle violations # Emits one GitHub workflow-command annotation per violation (visible inline in - # PR Files-changed) and exits 1 if any are found — fail-fast that bypasses - # SpotBugs (slow) when PMD/Checkstyle already has issues. Runs even if a previous - # step failed. + # PR Files-changed) and exits 1 if any are found. Runs even if a previous step + # failed so partial reports still surface as annotations. if: always() run: python3 .github/scripts/annotate-violations.py --pmd --checkstyle - - name: Enforce PMD/Checkstyle thresholds (Maven safety-net) - # Redundant in the success case (Python check above already passed). Provides - # official-tool validation in case the Python parser misreads schema. `compile` - # is required in this invocation too — pmd:check re-runs analysis from scratch - # rather than reading the prior pmd.xml, so it needs its own aux-classpath. - run: | - mvn -T 2C -f ./ddk-parent/pom.xml --batch-mode --fail-at-end \ - compile \ - pmd:check pmd:cpd-check \ - checkstyle:check - - name: Generate SpotBugs report + spotbugs-analyze: + runs-on: ubuntu-24.04 + env: + MAVEN_OPTS: -Xmx4g + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: + distribution: 'temurin' + java-version: '21' + - name: Set up Workspace Environment Variable + run: echo "WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV + - name: Cache Maven dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: /home/runner/.m2/repository + key: ${{ runner.os }}-maven-0-${{ hashFiles('**/pom.xml') }} + - name: Compile + generate SpotBugs report run: | mvn -T 2C -f ./ddk-parent/pom.xml --batch-mode --fail-at-end \ compile \ @@ -55,29 +78,34 @@ jobs: - name: Annotate + count SpotBugs violations if: always() run: python3 .github/scripts/annotate-violations.py --spotbugs - - name: Enforce SpotBugs threshold (Maven safety-net) - run: | - mvn -T 2C -f ./ddk-parent/pom.xml --batch-mode --fail-at-end \ - compile \ - spotbugs:check - line-endings: + maven-gate: + # Authoritative Maven-side validation. Runs every analyzer's :check goal in one + # invocation so they share aux-classpath and JVM state. Defense against the Python + # parser misreading XML schema; if Maven and Python ever disagree on violations, + # this job surfaces the divergence. runs-on: ubuntu-24.04 + env: + MAVEN_OPTS: -Xmx4g steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: Check LF line endings in index - # .gitattributes declares `* text=auto eol=lf` with .bat/.cmd/.ps1 - # exempted. Git's clean filter normalizes on commit, but verify it - # explicitly in case a file is miscategorized as binary or a filter - # is bypassed. + - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: + distribution: 'temurin' + java-version: '21' + - name: Set up Workspace Environment Variable + run: echo "WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV + - name: Cache Maven dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: /home/runner/.m2/repository + key: ${{ runner.os }}-maven-0-${{ hashFiles('**/pom.xml') }} + - name: Compile + run all :check goals run: | - violations=$(git ls-files --eol \ - | grep -E "^i/(crlf|mixed)" \ - | grep -vE "\.(bat|cmd|ps1)$" || true) - if [ -n "$violations" ]; then - echo "Files with CRLF/mixed line endings stored in the index:" - echo "$violations" - exit 1 - fi + mvn -T 2C -f ./ddk-parent/pom.xml --batch-mode --fail-at-end \ + compile \ + pmd:check pmd:cpd-check \ + checkstyle:check \ + spotbugs:check maven-verify: runs-on: ubuntu-24.04 steps: @@ -97,6 +125,9 @@ jobs: path: /home/runner/.m2/repository key: ${{ runner.os }}-maven-0-${{ hashFiles('**/pom.xml') }} - name: Build with Maven within a virtual X Server Environment + # Tests aren't safely parallelizable across modules in this codebase (shared + # workspace state), so no -T flag here. Static-analysis :check goals are + # handled in the maven-gate job, not duplicated here. run: xvfb-run mvn clean verify -f ./ddk-parent/pom.xml --batch-mode --fail-at-end - name: Fail on missing surefire reports # Tycho-Surefire writes no TEST-*.xml when discovery is empty — fail the job in that case.