Add performance measurement harness for DMD

.github/scripts/perf_comment.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""Render the perfrunner results.json into a sticky PR comment.
+
+Reads a schema v1 results.json, prints the markdown table, and — when run in
+CI with a token — upserts a single comment identified by a hidden marker so
+force-pushes update one comment instead of spamming.
+"""
+
+import json
+import os
+import sys
+import urllib.request
+
+MARKER = "<!-- dmd-perf-bot -->"
+
+
+def fmt_value(value, unit):
+    if unit == "count":
+        return f"{value / 1e6:,.1f} M"
+    if unit == "bytes":
+        return f"{value / (1024 * 1024):.2f} MB"
+    if unit == "kb":
+        return f"{value / 1024:.0f} MB"
+    return str(value)
+
+
+def fmt_delta(pct):
+    return f"+{pct:.2f}%" if pct > 0 else f"{pct:.2f}%"
+
+
+def render(results):
+    lines = [
+        MARKER,
+        "### DMD perf check",
+        "",
+        "| Metric | Base | PR | delta |",
+        "|--------|------|----|-------|",
+    ]
+    for m in results["metrics"]:
+        lines.append("| {} | {} | {} | {} |".format(
+            m["label"],
+            fmt_value(m["base"], m["unit"]),
+            fmt_value(m["head"], m["unit"]),
+            fmt_delta(m["delta_pct"]),
+        ))
+    return "\n".join(lines) + "\n"
+
+
+def api(method, url, token, payload=None):
+    data = json.dumps(payload).encode() if payload is not None else None
+    req = urllib.request.Request(url, data=data, method=method)
+    req.add_header("Authorization", f"Bearer {token}")
+    req.add_header("Accept", "application/vnd.github+json")
+    if data:
+        req.add_header("Content-Type", "application/json")
+    with urllib.request.urlopen(req) as resp:
+        return json.loads(resp.read() or "null")
+
+
+def upsert(body, repo, pr, token):
+    base = f"https://api.github.com/repos/{repo}"
+    comments = api("GET", f"{base}/issues/{pr}/comments?per_page=100", token)
+    existing = next((c for c in comments if MARKER in (c.get("body") or "")), None)
+    if existing:
+        api("PATCH", f"{base}/issues/comments/{existing['id']}", token, {"body": body})
+    else:
+        api("POST", f"{base}/issues/{pr}/comments", token, {"body": body})
+
+
+def main():
+    if len(sys.argv) != 2:
+        sys.exit("usage: perf_comment.py results.json")
+
+    with open(sys.argv[1]) as f:
+        results = json.load(f)
+
+    body = render(results)
+    print(body)
+
+    token = os.environ.get("GITHUB_TOKEN")
+    repo = os.environ.get("REPO")
+    pr = os.environ.get("PR_NUMBER")
+    if token and repo and pr:
+        upsert(body, repo, pr, token)
+        print(f"upserted comment on {repo}#{pr}")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/perf.yml

@@ -0,0 +1,93 @@
+name: perf
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'spec/**'
+      - 'changelog/**'
+      - '**/*.md'
+  push:
+    branches: [master]
+
+concurrency:
+  group: perf-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  perf:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      OS_NAME: linux
+      MODEL: 64
+      FULL_BUILD: false
+      HOST_DMD: dmd-2.112.0
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set parallelism
+        run: echo "N=$(nproc)" >> "$GITHUB_ENV"
+
+      - name: Compute base/head SHAs
+        id: refs
+        run: |
+          set -uexo pipefail
+          HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
+          git fetch --no-tags origin master
+          BASE_SHA="$(git merge-base "$HEAD_SHA" origin/master)"
+          echo "head=$HEAD_SHA"     >> "$GITHUB_OUTPUT"
+          echo "base=$BASE_SHA"     >> "$GITHUB_OUTPUT"
+          echo "branch=${GITHUB_BASE_REF:-master}" >> "$GITHUB_OUTPUT"
+
+      - name: Install prerequisites
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y valgrind time
+          valgrind --version
+
+      - name: Install host compiler
+        run: ci/run.sh install_host_compiler
+
+      # Each ref is built in its own worktree so the resulting dmd keeps a
+      # working dmd.conf (druntime/phobos paths are relative to its own tree).
+      - name: Build dmd at base and head
+        run: |
+          set -uexo pipefail
+          build_ref() {
+            git worktree add --force "$2" "$1"
+            ( cd "$2" && ci/run.sh setup_repos "${{ steps.refs.outputs.branch }}" && ci/run.sh build 0 )
+          }
+          build_ref "${{ steps.refs.outputs.base }}" "$RUNNER_TEMP/dmd-base"
+          build_ref "${{ steps.refs.outputs.head }}" "$RUNNER_TEMP/dmd-head"
+
+      - name: Measure
+        run: |
+          set -uexo pipefail
+          source ~/dlang/*/activate
+          built=generated/$OS_NAME/release/$MODEL/dmd
+          cd tools/perfrunner
+          dub run -- \
+            --base-dmd "$RUNNER_TEMP/dmd-base/$built" \
+            --head-dmd "$RUNNER_TEMP/dmd-head/$built" \
+            --base-sha "${{ steps.refs.outputs.base }}" \
+            --head-sha "${{ steps.refs.outputs.head }}" \
+            --pr "${{ github.event.pull_request.number || 0 }}" \
+            --host-dmd "${HOST_DMD#dmd-}" \
+            --out "$GITHUB_WORKSPACE/results.json"
+
+      - name: Post sticky PR comment
+        if: github.event_name == 'pull_request'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: python3 .github/scripts/perf_comment.py results.json

tools/perfrunner/.gitignore

@@ -0,0 +1,7 @@
+.dub/
+perfrunner
+perfrunner.exe
+*-test-*
+*.o
+*.obj
+results.json

tools/perfrunner/README.md

@@ -0,0 +1,54 @@
+# perfrunner
+
+Performance measurement harness for DMD. Given two already-built `dmd`
+binaries (a base and a head), it runs the Phase A workload, measures it, and
+writes a `results.json`. The GitHub workflow builds the two compilers; this
+tool only measures — it never touches git.
+
+This is **Phase A (MVP)**: a single committed `hello.d` workload and the five
+metrics below. The data repo and dashboard are later phases.
+
+## Usage
+
+```
+dub run perfrunner -- \
+  --base-dmd <path> --head-dmd <path> \
+  --base-sha <sha>  --head-sha <sha> \
+  [--pr <number>]   --out results.json
+```
+
+- `--base-dmd` / `--head-dmd`: paths to the two already-built compilers.
+- `--base-sha` / `--head-sha` / `--pr`: metadata, copied into the report.
+- `--os` / `--host-dmd`: runner metadata for the report (default
+  `ubuntu-latest`, host dmd version blank).
+- `--out`: where to write `results.json` (default `results.json`).
+
+Requires `valgrind`, `strip`, and GNU `/usr/bin/time -v` on the PATH (Linux).
+
+## Metrics (Phase A)
+
+| id | what | tool |
+|----|------|------|
+| `compile_hello_debug_instr`   | instructions to compile `hello.d`           | cachegrind |
+| `compile_hello_release_instr` | instructions to compile `hello.d -O -release` | cachegrind |
+| `dmd_binary_size`             | stripped size of the `dmd` binary           | stat |
+| `hello_binary_size`           | stripped size of the `hello` executable     | stat |
+| `hello_max_rss`               | peak RSS compiling `hello.d`                | time -v |
+
+## Layout
+
+- `source/app.d` — CLI entry: parse args, measure, write `results.json`.
+- `source/runner.d` — shell-out helper: run a command, capture output.
+- `source/cachegrind.d` — wrap valgrind, parse `I refs:`.
+- `source/metrics.d` — the five Phase A metric definitions.
+- `source/stats.d` — `% delta` helper.
+- `source/report.d` — `results.json` schema v1.
+- `source/workloads/hello.d` — the committed Phase A workload.
+
+## Tests
+
+```
+dub test
+```
+
+Covers stats, cachegrind/`time -v` parsing, and report serialisation.

tools/perfrunner/dub.json

@@ -0,0 +1,8 @@
+{
+	"name": "perfrunner",
+	"description": "DMD performance measurement harness (Phase A).",
+	"license": "BSL-1.0",
+	"targetType": "executable",
+	"sourcePaths": ["source"],
+	"excludedSourceFiles": ["source/workloads/*"]
+}

tools/perfrunner/source/app.d

@@ -0,0 +1,61 @@
+module app;
+
+import std.file : mkdirRecurse, tempDir, write;
+import std.getopt : getopt;
+import std.path : buildPath, dirName;
+import std.stdio : stderr, writeln;
+
+import metrics : measure, phaseA;
+import report : MetricResult, render, Report;
+
+/// Phase A workload: the one source file we compile to measure DMD.
+enum workload = buildPath(__FILE_FULL_PATH__.dirName, "workloads", "hello.d");
+
+version (unittest) {} else
+int main(string[] args)
+{
+    string baseDmd, headDmd, baseSha, headSha, hostDmd;
+    string os = "ubuntu-latest";
+    string outPath = "results.json";
+    long pr;
+
+    auto help = getopt(args,
+        "base-dmd", "path to the base (merge-base) dmd binary", &baseDmd,
+        "head-dmd", "path to the head (PR) dmd binary", &headDmd,
+        "base-sha", "base commit sha (metadata)", &baseSha,
+        "head-sha", "head commit sha (metadata)", &headSha,
+        "pr",       "pull request number (metadata)", &pr,
+        "os",       "runner OS label (metadata)", &os,
+        "host-dmd", "bootstrap dmd version (metadata)", &hostDmd,
+        "out",      "where to write results.json", &outPath,
+    );
+
+    if (help.helpWanted)
+    {
+        writeln("usage: perfrunner --base-dmd <path> --head-dmd <path> "
+            ~ "[--base-sha <sha> --head-sha <sha> --pr <n>] --out results.json");
+        return 0;
+    }
+
+    if (baseDmd.length == 0 || headDmd.length == 0)
+    {
+        stderr.writeln("error: --base-dmd and --head-dmd are required");
+        return 2;
+    }
+
+    auto tmp = buildPath(tempDir, "perfrunner");
+    mkdirRecurse(tmp);
+
+    auto base = measure(baseDmd, workload, tmp, "base");
+    auto head = measure(headDmd, workload, tmp, "head");
+
+    MetricResult[] metrics;
+    foreach (def; phaseA)
+        metrics ~= MetricResult(def.id, def.label, def.unit, def.method,
+            base[def.id], head[def.id]);
+
+    auto rep = Report(baseSha, "merge-base", headSha, pr, os, hostDmd, metrics);
+    write(outPath, render(rep));
+    writeln("wrote ", outPath);
+    return 0;
+}

tools/perfrunner/source/cachegrind.d

@@ -0,0 +1,38 @@
+module cachegrind;
+
+import std.array : replace;
+import std.conv : to;
+import std.path : buildPath;
+import std.regex : ctRegex, matchFirst;
+
+import runner : run;
+
+private enum iRefsRe = ctRegex!(`I\s+refs:\s+([\d,]+)`);
+
+/// Parse the "I refs:" instruction count out of cachegrind's output.
+long parseIRefs(string output)
+{
+    auto m = matchFirst(output, iRefsRe);
+    if (m.empty)
+        throw new Exception("could not parse cachegrind 'I refs:'");
+    return m[1].replace(",", "").to!long;
+}
+
+/// Compile the workload under cachegrind and return the instruction count.
+long instructions(string dmd, string[] dflags, string workload, string tmp, string tag)
+{
+    auto obj = buildPath(tmp, tag ~ ".o");
+    auto cgOut = buildPath(tmp, tag ~ ".cgout");
+    auto cmd = ["valgrind", "--tool=cachegrind", "--cachegrind-out-file=" ~ cgOut,
+        dmd, "-c"] ~ dflags ~ [workload, "-of=" ~ obj];
+    auto r = run(cmd);
+    if (r.status != 0)
+        throw new Exception("cachegrind failed:\n" ~ r.output);
+    return parseIRefs(r.output);
+}
+
+unittest
+{
+    auto sample = "==42== I   refs:      1,234,500,000\n";
+    assert(parseIRefs(sample) == 1_234_500_000);
+}