From 0628befbe282bfa7f9a1d5da3c3f3c2c3ead7fc6 Mon Sep 17 00:00:00 2001 From: Abul Date: Wed, 3 Jun 2026 23:14:00 +0530 Subject: [PATCH] Add performance measurement harness for DMD --- .github/scripts/perf_comment.py | 89 +++++++++++++++++++++ .github/workflows/perf.yml | 93 ++++++++++++++++++++++ tools/perfrunner/.gitignore | 7 ++ tools/perfrunner/README.md | 54 +++++++++++++ tools/perfrunner/dub.json | 8 ++ tools/perfrunner/source/app.d | 61 +++++++++++++++ tools/perfrunner/source/cachegrind.d | 38 +++++++++ tools/perfrunner/source/metrics.d | 94 +++++++++++++++++++++++ tools/perfrunner/source/report.d | 75 ++++++++++++++++++ tools/perfrunner/source/runner.d | 17 ++++ tools/perfrunner/source/stats.d | 17 ++++ tools/perfrunner/source/workloads/hello.d | 6 ++ 12 files changed, 559 insertions(+) create mode 100644 .github/scripts/perf_comment.py create mode 100644 .github/workflows/perf.yml create mode 100644 tools/perfrunner/.gitignore create mode 100644 tools/perfrunner/README.md create mode 100644 tools/perfrunner/dub.json create mode 100644 tools/perfrunner/source/app.d create mode 100644 tools/perfrunner/source/cachegrind.d create mode 100644 tools/perfrunner/source/metrics.d create mode 100644 tools/perfrunner/source/report.d create mode 100644 tools/perfrunner/source/runner.d create mode 100644 tools/perfrunner/source/stats.d create mode 100644 tools/perfrunner/source/workloads/hello.d diff --git a/.github/scripts/perf_comment.py b/.github/scripts/perf_comment.py new file mode 100644 index 000000000000..af121175499c --- /dev/null +++ b/.github/scripts/perf_comment.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Render the perfrunner results.json into a sticky PR comment. + +Reads a schema v1 results.json, prints the markdown table, and — when run in +CI with a token — upserts a single comment identified by a hidden marker so +force-pushes update one comment instead of spamming. +""" + +import json +import os +import sys +import urllib.request + +MARKER = "" + + +def fmt_value(value, unit): + if unit == "count": + return f"{value / 1e6:,.1f} M" + if unit == "bytes": + return f"{value / (1024 * 1024):.2f} MB" + if unit == "kb": + return f"{value / 1024:.0f} MB" + return str(value) + + +def fmt_delta(pct): + return f"+{pct:.2f}%" if pct > 0 else f"{pct:.2f}%" + + +def render(results): + lines = [ + MARKER, + "### DMD perf check", + "", + "| Metric | Base | PR | delta |", + "|--------|------|----|-------|", + ] + for m in results["metrics"]: + lines.append("| {} | {} | {} | {} |".format( + m["label"], + fmt_value(m["base"], m["unit"]), + fmt_value(m["head"], m["unit"]), + fmt_delta(m["delta_pct"]), + )) + return "\n".join(lines) + "\n" + + +def api(method, url, token, payload=None): + data = json.dumps(payload).encode() if payload is not None else None + req = urllib.request.Request(url, data=data, method=method) + req.add_header("Authorization", f"Bearer {token}") + req.add_header("Accept", "application/vnd.github+json") + if data: + req.add_header("Content-Type", "application/json") + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read() or "null") + + +def upsert(body, repo, pr, token): + base = f"https://api.github.com/repos/{repo}" + comments = api("GET", f"{base}/issues/{pr}/comments?per_page=100", token) + existing = next((c for c in comments if MARKER in (c.get("body") or "")), None) + if existing: + api("PATCH", f"{base}/issues/comments/{existing['id']}", token, {"body": body}) + else: + api("POST", f"{base}/issues/{pr}/comments", token, {"body": body}) + + +def main(): + if len(sys.argv) != 2: + sys.exit("usage: perf_comment.py results.json") + + with open(sys.argv[1]) as f: + results = json.load(f) + + body = render(results) + print(body) + + token = os.environ.get("GITHUB_TOKEN") + repo = os.environ.get("REPO") + pr = os.environ.get("PR_NUMBER") + if token and repo and pr: + upsert(body, repo, pr, token) + print(f"upserted comment on {repo}#{pr}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 000000000000..627399013900 --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,93 @@ +name: perf + +on: + pull_request: + paths-ignore: + - 'spec/**' + - 'changelog/**' + - '**/*.md' + push: + branches: [master] + +concurrency: + group: perf-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + +jobs: + perf: + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + OS_NAME: linux + MODEL: 64 + FULL_BUILD: false + HOST_DMD: dmd-2.112.0 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set parallelism + run: echo "N=$(nproc)" >> "$GITHUB_ENV" + + - name: Compute base/head SHAs + id: refs + run: | + set -uexo pipefail + HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}" + git fetch --no-tags origin master + BASE_SHA="$(git merge-base "$HEAD_SHA" origin/master)" + echo "head=$HEAD_SHA" >> "$GITHUB_OUTPUT" + echo "base=$BASE_SHA" >> "$GITHUB_OUTPUT" + echo "branch=${GITHUB_BASE_REF:-master}" >> "$GITHUB_OUTPUT" + + - name: Install prerequisites + run: | + sudo apt-get update + sudo apt-get install -y valgrind time + valgrind --version + + - name: Install host compiler + run: ci/run.sh install_host_compiler + + # Each ref is built in its own worktree so the resulting dmd keeps a + # working dmd.conf (druntime/phobos paths are relative to its own tree). + - name: Build dmd at base and head + run: | + set -uexo pipefail + build_ref() { + git worktree add --force "$2" "$1" + ( cd "$2" && ci/run.sh setup_repos "${{ steps.refs.outputs.branch }}" && ci/run.sh build 0 ) + } + build_ref "${{ steps.refs.outputs.base }}" "$RUNNER_TEMP/dmd-base" + build_ref "${{ steps.refs.outputs.head }}" "$RUNNER_TEMP/dmd-head" + + - name: Measure + run: | + set -uexo pipefail + source ~/dlang/*/activate + built=generated/$OS_NAME/release/$MODEL/dmd + cd tools/perfrunner + dub run -- \ + --base-dmd "$RUNNER_TEMP/dmd-base/$built" \ + --head-dmd "$RUNNER_TEMP/dmd-head/$built" \ + --base-sha "${{ steps.refs.outputs.base }}" \ + --head-sha "${{ steps.refs.outputs.head }}" \ + --pr "${{ github.event.pull_request.number || 0 }}" \ + --host-dmd "${HOST_DMD#dmd-}" \ + --out "$GITHUB_WORKSPACE/results.json" + + - name: Post sticky PR comment + if: github.event_name == 'pull_request' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: python3 .github/scripts/perf_comment.py results.json diff --git a/tools/perfrunner/.gitignore b/tools/perfrunner/.gitignore new file mode 100644 index 000000000000..f89d9b4adda1 --- /dev/null +++ b/tools/perfrunner/.gitignore @@ -0,0 +1,7 @@ +.dub/ +perfrunner +perfrunner.exe +*-test-* +*.o +*.obj +results.json diff --git a/tools/perfrunner/README.md b/tools/perfrunner/README.md new file mode 100644 index 000000000000..1a64063ee70b --- /dev/null +++ b/tools/perfrunner/README.md @@ -0,0 +1,54 @@ +# perfrunner + +Performance measurement harness for DMD. Given two already-built `dmd` +binaries (a base and a head), it runs the Phase A workload, measures it, and +writes a `results.json`. The GitHub workflow builds the two compilers; this +tool only measures — it never touches git. + +This is **Phase A (MVP)**: a single committed `hello.d` workload and the five +metrics below. The data repo and dashboard are later phases. + +## Usage + +``` +dub run perfrunner -- \ + --base-dmd --head-dmd \ + --base-sha --head-sha \ + [--pr ] --out results.json +``` + +- `--base-dmd` / `--head-dmd`: paths to the two already-built compilers. +- `--base-sha` / `--head-sha` / `--pr`: metadata, copied into the report. +- `--os` / `--host-dmd`: runner metadata for the report (default + `ubuntu-latest`, host dmd version blank). +- `--out`: where to write `results.json` (default `results.json`). + +Requires `valgrind`, `strip`, and GNU `/usr/bin/time -v` on the PATH (Linux). + +## Metrics (Phase A) + +| id | what | tool | +|----|------|------| +| `compile_hello_debug_instr` | instructions to compile `hello.d` | cachegrind | +| `compile_hello_release_instr` | instructions to compile `hello.d -O -release` | cachegrind | +| `dmd_binary_size` | stripped size of the `dmd` binary | stat | +| `hello_binary_size` | stripped size of the `hello` executable | stat | +| `hello_max_rss` | peak RSS compiling `hello.d` | time -v | + +## Layout + +- `source/app.d` — CLI entry: parse args, measure, write `results.json`. +- `source/runner.d` — shell-out helper: run a command, capture output. +- `source/cachegrind.d` — wrap valgrind, parse `I refs:`. +- `source/metrics.d` — the five Phase A metric definitions. +- `source/stats.d` — `% delta` helper. +- `source/report.d` — `results.json` schema v1. +- `source/workloads/hello.d` — the committed Phase A workload. + +## Tests + +``` +dub test +``` + +Covers stats, cachegrind/`time -v` parsing, and report serialisation. diff --git a/tools/perfrunner/dub.json b/tools/perfrunner/dub.json new file mode 100644 index 000000000000..c81b48d14e2d --- /dev/null +++ b/tools/perfrunner/dub.json @@ -0,0 +1,8 @@ +{ + "name": "perfrunner", + "description": "DMD performance measurement harness (Phase A).", + "license": "BSL-1.0", + "targetType": "executable", + "sourcePaths": ["source"], + "excludedSourceFiles": ["source/workloads/*"] +} diff --git a/tools/perfrunner/source/app.d b/tools/perfrunner/source/app.d new file mode 100644 index 000000000000..71c1be5e2f9f --- /dev/null +++ b/tools/perfrunner/source/app.d @@ -0,0 +1,61 @@ +module app; + +import std.file : mkdirRecurse, tempDir, write; +import std.getopt : getopt; +import std.path : buildPath, dirName; +import std.stdio : stderr, writeln; + +import metrics : measure, phaseA; +import report : MetricResult, render, Report; + +/// Phase A workload: the one source file we compile to measure DMD. +enum workload = buildPath(__FILE_FULL_PATH__.dirName, "workloads", "hello.d"); + +version (unittest) {} else +int main(string[] args) +{ + string baseDmd, headDmd, baseSha, headSha, hostDmd; + string os = "ubuntu-latest"; + string outPath = "results.json"; + long pr; + + auto help = getopt(args, + "base-dmd", "path to the base (merge-base) dmd binary", &baseDmd, + "head-dmd", "path to the head (PR) dmd binary", &headDmd, + "base-sha", "base commit sha (metadata)", &baseSha, + "head-sha", "head commit sha (metadata)", &headSha, + "pr", "pull request number (metadata)", &pr, + "os", "runner OS label (metadata)", &os, + "host-dmd", "bootstrap dmd version (metadata)", &hostDmd, + "out", "where to write results.json", &outPath, + ); + + if (help.helpWanted) + { + writeln("usage: perfrunner --base-dmd --head-dmd " + ~ "[--base-sha --head-sha --pr ] --out results.json"); + return 0; + } + + if (baseDmd.length == 0 || headDmd.length == 0) + { + stderr.writeln("error: --base-dmd and --head-dmd are required"); + return 2; + } + + auto tmp = buildPath(tempDir, "perfrunner"); + mkdirRecurse(tmp); + + auto base = measure(baseDmd, workload, tmp, "base"); + auto head = measure(headDmd, workload, tmp, "head"); + + MetricResult[] metrics; + foreach (def; phaseA) + metrics ~= MetricResult(def.id, def.label, def.unit, def.method, + base[def.id], head[def.id]); + + auto rep = Report(baseSha, "merge-base", headSha, pr, os, hostDmd, metrics); + write(outPath, render(rep)); + writeln("wrote ", outPath); + return 0; +} diff --git a/tools/perfrunner/source/cachegrind.d b/tools/perfrunner/source/cachegrind.d new file mode 100644 index 000000000000..fbbc0ba00e30 --- /dev/null +++ b/tools/perfrunner/source/cachegrind.d @@ -0,0 +1,38 @@ +module cachegrind; + +import std.array : replace; +import std.conv : to; +import std.path : buildPath; +import std.regex : ctRegex, matchFirst; + +import runner : run; + +private enum iRefsRe = ctRegex!(`I\s+refs:\s+([\d,]+)`); + +/// Parse the "I refs:" instruction count out of cachegrind's output. +long parseIRefs(string output) +{ + auto m = matchFirst(output, iRefsRe); + if (m.empty) + throw new Exception("could not parse cachegrind 'I refs:'"); + return m[1].replace(",", "").to!long; +} + +/// Compile the workload under cachegrind and return the instruction count. +long instructions(string dmd, string[] dflags, string workload, string tmp, string tag) +{ + auto obj = buildPath(tmp, tag ~ ".o"); + auto cgOut = buildPath(tmp, tag ~ ".cgout"); + auto cmd = ["valgrind", "--tool=cachegrind", "--cachegrind-out-file=" ~ cgOut, + dmd, "-c"] ~ dflags ~ [workload, "-of=" ~ obj]; + auto r = run(cmd); + if (r.status != 0) + throw new Exception("cachegrind failed:\n" ~ r.output); + return parseIRefs(r.output); +} + +unittest +{ + auto sample = "==42== I refs: 1,234,500,000\n"; + assert(parseIRefs(sample) == 1_234_500_000); +} diff --git a/tools/perfrunner/source/metrics.d b/tools/perfrunner/source/metrics.d new file mode 100644 index 000000000000..18c3668b32b0 --- /dev/null +++ b/tools/perfrunner/source/metrics.d @@ -0,0 +1,94 @@ +module metrics; + +import std.conv : to; +import std.file : copy, exists, getSize, remove; +import std.path : buildPath; +import std.regex : ctRegex, matchFirst; + +import cachegrind : instructions; +import runner : run; + +struct MetricDef +{ + string id; + string label; + string unit; + string method; +} + +/// The five Phase A metrics, in the order they appear in the report (plan §6). +immutable MetricDef[] phaseA = [ + MetricDef("compile_hello_debug_instr", "compile hello.d (instr)", "count", "cachegrind"), + MetricDef("compile_hello_release_instr", "compile hello.d -O (instr)", "count", "cachegrind"), + MetricDef("dmd_binary_size", "dmd binary size (stripped)", "bytes", "stat"), + MetricDef("hello_binary_size", "hello binary size", "bytes", "stat"), + MetricDef("hello_max_rss", "peak RSS (compile hello.d)", "kb", "time -v"), +]; + +/// Measure every Phase A metric for one dmd binary. `tag` ("base"/"head") +/// keeps the two runs' temp files apart. +long[string] measure(string dmd, string workload, string tmp, string tag) +{ + return [ + "compile_hello_debug_instr": instructions(dmd, [], workload, tmp, tag ~ "-dbg"), + "compile_hello_release_instr": instructions(dmd, ["-O", "-release"], workload, tmp, tag ~ "-rel"), + "dmd_binary_size": strippedSize(dmd, buildPath(tmp, tag ~ "-dmd")), + "hello_binary_size": helloSize(dmd, workload, tmp, tag), + "hello_max_rss": maxRss(dmd, workload, tmp, tag), + ]; +} + +/// Byte size of `binary` after copying and stripping it. +private long strippedSize(string binary, string copyPath) +{ + if (exists(copyPath)) + remove(copyPath); + copy(binary, copyPath); + strip(copyPath); + return getSize(copyPath); +} + +/// Compile the workload to an executable, strip it, and return its size. +private long helloSize(string dmd, string workload, string tmp, string tag) +{ + auto exe = buildPath(tmp, tag ~ "-hello"); + auto r = run([dmd, workload, "-of=" ~ exe]); + if (r.status != 0) + throw new Exception("compiling hello executable failed:\n" ~ r.output); + strip(exe); + return getSize(exe); +} + +private void strip(string path) +{ + auto r = run(["strip", path]); + if (r.status != 0) + throw new Exception("strip failed:\n" ~ r.output); +} + +/// Peak resident set size (KiB) of compiling the workload, via /usr/bin/time. +private long maxRss(string dmd, string workload, string tmp, string tag) +{ + auto obj = buildPath(tmp, tag ~ "-rss.o"); + auto r = run(["/usr/bin/time", "-v", dmd, "-c", workload, "-of=" ~ obj]); + if (r.status != 0) + throw new Exception("/usr/bin/time failed:\n" ~ r.output); + return parseMaxRss(r.output); +} + +private enum rssRe = ctRegex!(`Maximum resident set size \(kbytes\):\s+(\d+)`); + +/// Pull the max-RSS value (KiB) out of `/usr/bin/time -v` output. +long parseMaxRss(string output) +{ + auto m = matchFirst(output, rssRe); + if (m.empty) + throw new Exception("could not parse max RSS"); + return m[1].to!long; +} + +unittest +{ + auto sample = "\tMaximum resident set size (kbytes): 184320\n"; + assert(parseMaxRss(sample) == 184320); +} diff --git a/tools/perfrunner/source/report.d b/tools/perfrunner/source/report.d new file mode 100644 index 000000000000..bb09237d17d3 --- /dev/null +++ b/tools/perfrunner/source/report.d @@ -0,0 +1,75 @@ +module report; + +import std.json : JSONValue, parseJSON; +import std.math : round; + +import stats : deltaPct; + +struct MetricResult +{ + string id; + string label; + string unit; + string method; + long base; + long head; +} + +struct Report +{ + string baseSha; + string baseRef; + string headSha; + long pr; + string os; + string hostDmd; + MetricResult[] metrics; +} + +/// Serialise a report to the schema v1 JSON string (plan §7). +string render(Report rep) +{ + JSONValue[] metrics; + foreach (m; rep.metrics) + { + metrics ~= JSONValue([ + "id": JSONValue(m.id), + "label": JSONValue(m.label), + "unit": JSONValue(m.unit), + "method": JSONValue(m.method), + "base": JSONValue(m.base), + "head": JSONValue(m.head), + "delta_pct": JSONValue(round(deltaPct(m.base, m.head) * 100) / 100.0), + ]); + } + + JSONValue root = [ + "schema_version": JSONValue(1), + "base": JSONValue(["sha": JSONValue(rep.baseSha), "ref": JSONValue(rep.baseRef)]), + "head": JSONValue(["sha": JSONValue(rep.headSha), "pr": JSONValue(rep.pr)]), + "runner": JSONValue(["os": JSONValue(rep.os), "host_dmd": JSONValue(rep.hostDmd)]), + "metrics": JSONValue(metrics), + ]; + + return root.toPrettyString(); +} + +unittest +{ + auto rep = Report("base1", "merge-base", "head1", 7, "ubuntu-latest", "2.112.0", + [MetricResult("compile_hello_debug_instr", "compile hello.d (instr)", + "count", "cachegrind", 1000, 1010)]); + + auto j = parseJSON(render(rep)); + assert(j["schema_version"].integer == 1); + assert(j["base"]["sha"].str == "base1"); + assert(j["head"]["pr"].integer == 7); + assert(j["metrics"].array.length == 1); + + auto m = j["metrics"][0]; + assert(m["id"].str == "compile_hello_debug_instr"); + assert(m["base"].integer == 1000); + + import std.math : isClose; + assert(isClose(m["delta_pct"].floating, 1.0)); +} diff --git a/tools/perfrunner/source/runner.d b/tools/perfrunner/source/runner.d new file mode 100644 index 000000000000..d8f9356a725d --- /dev/null +++ b/tools/perfrunner/source/runner.d @@ -0,0 +1,17 @@ +module runner; + +import std.process : execute; + +/// Outcome of running a single command. +struct RunResult +{ + int status; /// process exit code + string output; /// combined stdout + stderr +} + +/// Run `cmd` and capture its exit code and output. +RunResult run(string[] cmd) +{ + auto r = execute(cmd); + return RunResult(r.status, r.output); +} diff --git a/tools/perfrunner/source/stats.d b/tools/perfrunner/source/stats.d new file mode 100644 index 000000000000..9e99281c19c1 --- /dev/null +++ b/tools/perfrunner/source/stats.d @@ -0,0 +1,17 @@ +module stats; + +/// Percent change from `base` to `head`. A zero base yields zero. +double deltaPct(double base, double head) +{ + if (base == 0) + return 0; + return (head - base) / base * 100.0; +} + +unittest +{ + import std.math : isClose; + assert(isClose(deltaPct(100, 101), 1.0)); + assert(isClose(deltaPct(200, 150), -25.0)); + assert(deltaPct(0, 5) == 0); +} diff --git a/tools/perfrunner/source/workloads/hello.d b/tools/perfrunner/source/workloads/hello.d new file mode 100644 index 000000000000..a708e640b550 --- /dev/null +++ b/tools/perfrunner/source/workloads/hello.d @@ -0,0 +1,6 @@ +import std.stdio; + +void main() +{ + writeln("hello"); +}