From 1dde4538b6f717b64d76cc69adbd3f5febca37cf Mon Sep 17 00:00:00 2001 From: Paul Adelsbach Date: Fri, 24 Jul 2026 15:18:25 -0700 Subject: [PATCH] CI: automate coverage comparison for test-refactor --- .github/workflows/code-coverage-refactor.yml | 100 ------------ .github/workflows/code-coverage.yml | 105 ++++++++---- tools/coverage-compare/compare_tracefiles.py | 161 +++++++++++++++++++ 3 files changed, 236 insertions(+), 130 deletions(-) delete mode 100644 .github/workflows/code-coverage-refactor.yml create mode 100644 tools/coverage-compare/compare_tracefiles.py diff --git a/.github/workflows/code-coverage-refactor.yml b/.github/workflows/code-coverage-refactor.yml deleted file mode 100644 index f3ef65cfe..000000000 --- a/.github/workflows/code-coverage-refactor.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: Code Coverage Refactor - -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] - -permissions: - contents: read - -jobs: - coverage: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - name: base - flags: "" - - name: dma - flags: "DMA=1" - - name: threadsafe - flags: "THREADSAFE=1" - - name: she - flags: "SHE=1" - - name: auth - flags: "AUTH=1" - - name: nocrypto - flags: "NOCRYPTO=1" - steps: - - uses: actions/checkout@v4 - - - name: Install gcovr - # Pin a modern gcovr via pipx; the apt-shipped 7.0 on ubuntu-noble - # cannot parse gcc 13's gcov output with block IDs >= 10000 (fixed - # upstream in gcovr 7.1, see gcovr PR #883). - run: pipx install 'gcovr==8.6' - - - name: Checkout wolfssl - uses: actions/checkout@v4 - with: - repository: wolfssl/wolfssl - path: wolfssl - - - name: Build, run, and emit tracefile (${{ matrix.name }}) - run: | - cd test-refactor/posix && make coverage-json \ - OUT=$GITHUB_WORKSPACE/cov-json/refactor-${{ matrix.name }}.json \ - WOLFSSL_DIR=../../wolfssl ${{ matrix.flags }} - - - name: Upload tracefile - uses: actions/upload-artifact@v4 - with: - name: refactor-tracefile-${{ matrix.name }} - path: cov-json/refactor-${{ matrix.name }}.json - retention-days: 7 - - merge: - needs: coverage - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install gcovr - run: pipx install 'gcovr==8.6' - - - name: Download refactor tracefiles - uses: actions/download-artifact@v4 - with: - pattern: refactor-tracefile-* - path: cov-json - merge-multiple: true - - - name: Merge tracefiles into HTML report - run: | - # nullglob so an empty match returns 0 files instead of the - # literal pattern, which would slip past the guard below. - shopt -s nullglob - files=(cov-json/refactor-*.json) - if [ ${#files[@]} -eq 0 ]; then - echo "No tracefiles found to merge" >&2 - exit 1 - fi - # Build "--add-tracefile " pairs for each tracefile. - args=() - for f in "${files[@]}"; do - args+=(--add-tracefile "$f") - done - mkdir -p coverage-refactor-merged - gcovr "${args[@]}" \ - --html-details coverage-refactor-merged/index.html \ - --print-summary - - - name: Upload merged coverage report - uses: actions/upload-artifact@v4 - with: - name: coverage-refactor-report - path: coverage-refactor-merged/ - retention-days: 30 diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 23bf8b457..bbbd9a27c 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -1,10 +1,12 @@ name: Code Coverage on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] + # Coverage is an on-demand, information-only report, not a PR gate. + # Run it manually from the Actions tab against any branch (including a + # PR branch); the weekly run keeps a fresh artifact and catches rot. + workflow_dispatch: + schedule: + - cron: '0 6 * * 1' permissions: contents: read @@ -15,7 +17,8 @@ jobs: strategy: fail-fast: false matrix: - include: + tree: [legacy, refactor] + config: - name: base flags: "" - name: dma @@ -43,20 +46,29 @@ jobs: repository: wolfssl/wolfssl path: wolfssl - - name: Build, run, and emit tracefile (${{ matrix.name }}) + - name: Build, run, and emit tracefile (${{ matrix.tree }} ${{ matrix.config.name }}) + # legacy and refactor trees differ only in the test dir and the + # relative path back to the wolfssl checkout; both emit paths + # rooted at the repo, so their tracefiles compare apples-to-apples. run: | - cd test && make coverage-json \ - OUT=$GITHUB_WORKSPACE/cov-json/legacy-${{ matrix.name }}.json \ - WOLFSSL_DIR=../wolfssl ${{ matrix.flags }} + if [ "${{ matrix.tree }}" = "legacy" ]; then + dir=test; wolfssl=../wolfssl + else + dir=test-refactor/posix; wolfssl=../../wolfssl + fi + cd "$dir" + make coverage-json \ + OUT=$GITHUB_WORKSPACE/cov-json/${{ matrix.tree }}-${{ matrix.config.name }}.json \ + WOLFSSL_DIR=$wolfssl ${{ matrix.config.flags }} - name: Upload tracefile uses: actions/upload-artifact@v4 with: - name: legacy-tracefile-${{ matrix.name }} - path: cov-json/legacy-${{ matrix.name }}.json + name: tracefile-${{ matrix.tree }}-${{ matrix.config.name }} + path: cov-json/${{ matrix.tree }}-${{ matrix.config.name }}.json retention-days: 7 - merge: + report: needs: coverage runs-on: ubuntu-latest steps: @@ -65,36 +77,69 @@ jobs: - name: Install gcovr run: pipx install 'gcovr==8.6' - - name: Download legacy tracefiles + - name: Download tracefiles uses: actions/download-artifact@v4 with: - pattern: legacy-tracefile-* + pattern: tracefile-* path: cov-json merge-multiple: true - - name: Merge tracefiles into HTML report + - name: Build HTML reports run: | # nullglob so an empty match returns 0 files instead of the # literal pattern, which would slip past the guard below. shopt -s nullglob - files=(cov-json/legacy-*.json) - if [ ${#files[@]} -eq 0 ]; then - echo "No tracefiles found to merge" >&2 - exit 1 - fi - # Build "--add-tracefile " pairs for each tracefile. - args=() - for f in "${files[@]}"; do - args+=(--add-tracefile "$f") + for tree in legacy refactor; do + files=(cov-json/$tree-*.json) + if [ ${#files[@]} -eq 0 ]; then + echo "No $tree tracefiles found to merge" >&2 + exit 1 + fi + args=() + for f in "${files[@]}"; do + args+=(--add-tracefile "$f") + done + mkdir -p coverage-$tree + gcovr "${args[@]}" \ + --html-details coverage-$tree/index.html \ + --print-summary done - mkdir -p coverage-merged - gcovr "${args[@]}" \ - --html-details coverage-merged/index.html \ - --print-summary - - name: Upload merged coverage report + - name: Upload legacy coverage report uses: actions/upload-artifact@v4 with: name: coverage-report - path: coverage-merged/ + path: coverage-legacy/ + retention-days: 30 + + - name: Upload refactor coverage report + uses: actions/upload-artifact@v4 + with: + name: coverage-refactor-report + path: coverage-refactor/ + retention-days: 30 + + compare: + needs: coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Download tracefiles + uses: actions/download-artifact@v4 + with: + pattern: tracefile-* + path: cov-json + merge-multiple: true + + - name: Compare legacy vs refactor coverage + run: | + python3 tools/coverage-compare/compare_tracefiles.py cov-json compare + cat compare/summary.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload coverage comparison + uses: actions/upload-artifact@v4 + with: + name: coverage-comparison + path: compare/ retention-days: 30 diff --git a/tools/coverage-compare/compare_tracefiles.py b/tools/coverage-compare/compare_tracefiles.py new file mode 100644 index 000000000..5d26d176e --- /dev/null +++ b/tools/coverage-compare/compare_tracefiles.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 + +# Compare legacy vs refactor gcovr tracefiles per config and report the +# src/ coverage that only the legacy suite provides. A line or branch +# covered by legacy but not by refactor (in the same build config) is +# coverage that would be lost if the legacy suite were trimmed, so it +# blocks trimming until the refactor tree catches up. Output groups by +# source file; mapping a file back to the legacy suite that exercises +# it is the reviewer's step. +# +# Usage: compare_tracefiles.py +# Pairs /legacy-.json with /refactor-.json. + +import glob +import json +import os +import sys +from collections import defaultdict + + +def load_tracefile(path): + # Return {file: {"lines": {n: covered}, "branches": {n: {key: cov}}}}. + # A line/branch is covered if any of its entries has count > 0. + with open(path) as fh: + data = json.load(fh) + files = {} + for entry in data.get("files", []): + name = entry["file"].lstrip("./") + lines = defaultdict(bool) + branches = defaultdict(lambda: defaultdict(bool)) + for ln in entry.get("lines", []): + n = ln["line_number"] + lines[n] = lines[n] or ln.get("count", 0) > 0 + for br in ln.get("branches", []): + key = (br.get("source_block_id"), + br.get("destination_block_id")) + covered = br.get("count", 0) > 0 + branches[n][key] = branches[n][key] or covered + files[name] = {"lines": dict(lines), "branches": branches} + return files + + +def legacy_only(legacy, refactor): + # Per file, find lines/branches covered in legacy but not refactor. + result = {} + for name, lf in legacy.items(): + rf = refactor.get(name) + only_lines = [] + for n, covered in lf["lines"].items(): + if not covered: + continue + if rf is None or not rf["lines"].get(n, False): + only_lines.append(n) + only_branches = [] + for n, keys in lf["branches"].items(): + for key, covered in keys.items(): + if not covered: + continue + rcov = rf and rf["branches"].get(n, {}).get(key, False) + if not rcov: + only_branches.append(n) + if only_lines or only_branches: + result[name] = { + "file_absent_in_refactor": rf is None, + "lines": sorted(only_lines), + "branch_count": len(only_branches), + } + return result + + +def discover_configs(cov_dir): + configs = [] + for p in sorted(glob.glob(os.path.join(cov_dir, "legacy-*.json"))): + name = os.path.basename(p)[len("legacy-"):-len(".json")] + if os.path.exists(os.path.join(cov_dir, "refactor-%s.json" % name)): + configs.append(name) + return configs + + +def main(argv): + if len(argv) != 3: + sys.stderr.write("usage: %s \n" % argv[0]) + return 2 + cov_dir, out_dir = argv[1], argv[2] + os.makedirs(out_dir, exist_ok=True) + + configs = discover_configs(cov_dir) + if not configs: + sys.stderr.write("no legacy/refactor tracefile pairs found\n") + return 1 + + per_config = {} + # union[file] -> set of configs where the file has legacy-only lines + union = defaultdict(set) + for cfg in configs: + legacy = load_tracefile(os.path.join(cov_dir, "legacy-%s.json" % cfg)) + refactor = load_tracefile( + os.path.join(cov_dir, "refactor-%s.json" % cfg)) + diff = legacy_only(legacy, refactor) + per_config[cfg] = diff + for name, info in diff.items(): + if info["lines"]: + union[name].add(cfg) + + with open(os.path.join(out_dir, "legacy-only-coverage.json"), "w") as fh: + json.dump({"configs": per_config}, fh, indent=2, sort_keys=True) + fh.write("\n") + + write_summary(os.path.join(out_dir, "summary.md"), configs, per_config, + union) + return 0 + + +def write_summary(path, configs, per_config, union): + lines = ["## Legacy-only coverage (trim blockers)", ""] + lines.append("Lines and branches covered by the legacy suite but not " + "the refactor suite, per build config. A file listed here " + "still has coverage the refactor tree does not reproduce.") + lines.append("") + for cfg in configs: + diff = per_config[cfg] + total_lines = sum(len(i["lines"]) for i in diff.values()) + total_branches = sum(i["branch_count"] for i in diff.values()) + header = "### %s: %d legacy-only lines, %d branches" % ( + cfg, total_lines, total_branches) + lines.append(header) + if not diff: + lines.append("") + lines.append("Refactor covers everything legacy does.") + lines.append("") + continue + lines.append("") + lines.append("| file | lines | branches | file absent |") + lines.append("|------|------:|---------:|:-----------:|") + for name in sorted(diff): + info = diff[name] + absent = "yes" if info["file_absent_in_refactor"] else "" + lines.append("| %s | %d | %d | %s |" % ( + name, len(info["lines"]), info["branch_count"], absent)) + lines.append("") + + lines.append("### Trim-blocker files (legacy-only lines in any config)") + lines.append("") + if union: + lines.append("| file | configs |") + lines.append("|------|---------|") + for name in sorted(union): + lines.append("| %s | %s |" % (name, + ", ".join(sorted(union[name])))) + else: + lines.append("None. Every file's legacy line coverage is reproduced " + "by the refactor suite.") + lines.append("") + + with open(path, "w") as fh: + fh.write("\n".join(lines)) + fh.write("\n") + + +if __name__ == "__main__": + sys.exit(main(sys.argv))