diff --git a/.github/matrix.yaml b/.github/matrix.yaml new file mode 100644 index 0000000..8efa5bb --- /dev/null +++ b/.github/matrix.yaml @@ -0,0 +1,108 @@ +# SPDX-FileCopyrightText: 2026 Yaroslav Halchenko +# SPDX-License-Identifier: MIT +# +# Single source of truth for the eval-under CI matrix. +# +# The matrix is backends x targets. A "backend" answers "which +# filesystem?" (bin/eval-under-); a "target" answers "which test +# suite do we run under it?" (bin/ci/target-.sh). Every +# combination below becomes one cell of .github/workflows/test.yaml's +# job matrix. +# +# Kept external, and read rather than templated, so that: +# - the workflow YAML stays thin (see .claude/CLAUDE.md); +# - `bin/ci/matrix.sh` and the workflow agree by construction instead +# of by discipline -- both read this file; +# - adding a filesystem or a suite is a data edit, not a code edit. +# +# Consumed by: +# bin/ci/matrix.sh sourced by every other bin/ci script +# bin/ci/matrix-json.sh renders the GitHub `matrix:` value +# +# After editing, run `bin/ci/gen-readme-matrix.sh` to refresh the +# README's badge grid. + +repo-slug: con/eval-under + +# Where install-target.sh unpacks/builds source trees on the *runner* +# (deliberately not on the filesystem under test -- we want the build to +# be fast and the suite's I/O to be the only thing hitting the mount). +src-dir: /opt/eval-under-src + +# Pinned upstream refs. Bump deliberately, never "latest": a moving +# testsuite makes a red cell ambiguous (did the filesystem regress, or +# did upstream add a test?). +refs: + git: v2.55.0 + # pjd/pjdfstest carries exactly one tag upstream, "0.1" (2016), and it + # no longer builds: major()/minor()/makedev() moved to + # in glibc 2.28 and the tree compiles with -Werror, + # so the implicit declarations are hard errors on anything modern. + # Master builds clean, so pin a commit on master instead -- same + # determinism, minus the patch we would otherwise have to carry. + pjdfstest: 85a8aea9e685999ef0540392fd80535f873d7ff7 + +# Column order of the README CI matrix. +# +# timeout wall-clock budget for the wrapped suite, in seconds. +# Kept below the job-level timeout so a runaway suite +# produces a `timeout` exit code (and our logs) rather +# than a bare GitHub cancellation. +# loop-size-mb backing-image size for `eval-under loop --size`. +# needs-root the suite is meaningless without privilege, so the +# NFS backend must export no_root_squash for it. +# pjdfstest is half privileged-vs-unprivileged +# assertions and refuses to run otherwise; stress-ng's +# chown/mknod stressors need CAP_CHOWN / CAP_MKNOD. +targets: + - name: git-annex + label: git-annex test + timeout: 2400 + loop-size-mb: 100 + needs-root: false + needs-git-annex: true + + # 174 scripts, ~2 min on ext4; a sync-heavy backend is far slower. + - name: git + label: git testsuite + timeout: 2400 + loop-size-mb: 512 + needs-root: false + needs-git-annex: false + + - name: stress-ng + label: stress-ng + timeout: 900 + # copy-file's minimum --copy-file-bytes is 128M, and it needs source + # + destination, so 512 leaves uncomfortably little slack. + loop-size-mb: 768 + needs-root: true + needs-git-annex: false + + # ~8500 assertions, run serially; ext4 does it in minutes but a + # sync-heavy NFS or BeeGFS mount is an order of magnitude slower. + - name: pjdfstest + label: pjdfstest + timeout: 2400 + loop-size-mb: 256 + needs-root: true + needs-git-annex: false + +# Row order of the README CI matrix. `version` is the literal "n/a" for +# backends with nothing to pin (see bin/ci/install-backend.sh). +backends: + - backend: beegfs + version: 7.4.6 + label: BeeGFS 7.4.6 + - backend: beegfs + version: 8.1.0 + label: BeeGFS 8.1.0 + - backend: nfs + version: n/a + label: NFS (localhost) + - backend: loop + version: vfat + label: Loop vfat + - backend: loop + version: ext4 + label: Loop ext4 diff --git a/.github/workflows/_test-under.yaml b/.github/workflows/_test-under.yaml deleted file mode 100644 index d9e7d93..0000000 --- a/.github/workflows/_test-under.yaml +++ /dev/null @@ -1,72 +0,0 @@ -name: (reusable) git-annex test under - -# Reusable body for the eval-under framework. Takes a backend -# (beegfs / nfs / loop) plus a backend-version (a BeeGFS point release, -# a loop filesystem name, or `n/a` for NFS), installs the matching -# client-side packages, fetches the freshest con/git-annex daily, and -# runs the full `git annex test` under the requested filesystem. -# -# Invoked by per-flavour dispatcher workflows so each backend/version -# combination gets its own badge URL. (GitHub only exposes one badge -# per workflow file, not per matrix cell.) -# -# All non-trivial shell logic lives in bin/ci/*.sh so it can be shellcheck'd -# and exercised locally in the Vagrant VM. See .claude/CLAUDE.md. - -on: - workflow_call: - inputs: - backend: - description: "eval-under backend: beegfs | nfs | loop" - required: true - type: string - backend-version: - description: >- - For beegfs: point release (e.g. 7.4.6, 8.1.0). - For loop: filesystem type (e.g. vfat, ext4). - For nfs: literal "n/a" (no version to pin). - required: true - type: string - -jobs: - test: - # ubuntu-22.04: kernel 5.15/6.5, within BeeGFS 7.4.x and 8.x DKMS - # support. ubuntu-24.04 hosted runners ship 6.17-azure which BeeGFS - # kernel modules cannot build against. NFS + loop backends don't - # need this, but sharing the runner OS keeps the reusable simple. - runs-on: ubuntu-22.04 - timeout-minutes: 60 - permissions: - contents: read - actions: read # to download artifacts from con/git-annex - steps: - - uses: actions/checkout@v4 - - - name: Install backend client dependencies - run: bin/ci/install-backend.sh "${{ inputs.backend }}" "${{ inputs.backend-version }}" - - - name: Fetch latest git-annex daily build from con/git-annex - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: bin/ci/install-git-annex-daily.sh - - - name: Configure git identity - run: | - git config --global user.email test@github.land - git config --global user.name "GitHub Almighty" - - - name: Run git annex test under ${{ inputs.backend }} - run: sudo -E bin/ci/run-under.sh "${{ inputs.backend }}" "${{ inputs.backend-version }}" - - - name: Dump failure logs - if: failure() - run: bin/ci/dump-failure-logs.sh "${{ inputs.backend }}" "${{ inputs.backend-version }}" - - - name: Upload logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: logs-${{ inputs.backend }}-${{ inputs.backend-version }} - path: | - /var/log/beegfs-* - if-no-files-found: ignore diff --git a/.github/workflows/test-beegfs-7.4.6.yaml b/.github/workflows/test-beegfs-7.4.6.yaml deleted file mode 100644 index 82b83cf..0000000 --- a/.github/workflows/test-beegfs-7.4.6.yaml +++ /dev/null @@ -1,38 +0,0 @@ -name: BeeGFS 7.4.6 - -# Thin dispatcher so this flavour gets a distinct README badge. -# Delegates all work to _test-under.yaml (reusable). - -on: - push: - branches: [main, master] - paths: - - "bin/**" - - "fixtures/beegfs/**" - - "provision/**" - - ".github/workflows/_test-under.yaml" - - ".github/workflows/test-beegfs-7.4.6.yaml" - pull_request: - paths: - - "bin/**" - - "fixtures/beegfs/**" - - "provision/**" - - ".github/workflows/_test-under.yaml" - - ".github/workflows/test-beegfs-7.4.6.yaml" - workflow_dispatch: - schedule: - - cron: "17 5 * * 1" # Mon 05:17 UTC - -concurrency: - group: test-beegfs-7.4.6-${{ github.ref }} - cancel-in-progress: true - -jobs: - call: - uses: ./.github/workflows/_test-under.yaml - with: - backend: "beegfs" - backend-version: "7.4.6" - permissions: - contents: read - actions: read diff --git a/.github/workflows/test-beegfs-8.1.0.yaml b/.github/workflows/test-beegfs-8.1.0.yaml deleted file mode 100644 index 6705492..0000000 --- a/.github/workflows/test-beegfs-8.1.0.yaml +++ /dev/null @@ -1,38 +0,0 @@ -name: BeeGFS 8.1.0 - -# Thin dispatcher so this flavour gets a distinct README badge. -# Delegates all work to _test-under.yaml (reusable). - -on: - push: - branches: [main, master] - paths: - - "bin/**" - - "fixtures/beegfs/**" - - "provision/**" - - ".github/workflows/_test-under.yaml" - - ".github/workflows/test-beegfs-8.1.0.yaml" - pull_request: - paths: - - "bin/**" - - "fixtures/beegfs/**" - - "provision/**" - - ".github/workflows/_test-under.yaml" - - ".github/workflows/test-beegfs-8.1.0.yaml" - workflow_dispatch: - schedule: - - cron: "17 5 * * 1" # Mon 05:17 UTC - -concurrency: - group: test-beegfs-8.1.0-${{ github.ref }} - cancel-in-progress: true - -jobs: - call: - uses: ./.github/workflows/_test-under.yaml - with: - backend: "beegfs" - backend-version: "8.1.0" - permissions: - contents: read - actions: read diff --git a/.github/workflows/test-loop-ext4.yaml b/.github/workflows/test-loop-ext4.yaml deleted file mode 100644 index 3da3ad2..0000000 --- a/.github/workflows/test-loop-ext4.yaml +++ /dev/null @@ -1,40 +0,0 @@ -name: Loop ext4 - -# Thin dispatcher so this flavour gets a distinct README badge. -# Delegates all work to _test-under.yaml (reusable). -# -# Baseline flavour: ext4 loop mount, close to the default Linux root -# filesystem behaviour. Regressions here point at git-annex itself -# rather than filesystem-specific edge cases. - -on: - push: - branches: [main, master] - paths: - - "bin/**" - - "provision/**" - - ".github/workflows/_test-under.yaml" - - ".github/workflows/test-loop-ext4.yaml" - pull_request: - paths: - - "bin/**" - - "provision/**" - - ".github/workflows/_test-under.yaml" - - ".github/workflows/test-loop-ext4.yaml" - workflow_dispatch: - schedule: - - cron: "17 5 * * 1" # Mon 05:17 UTC - -concurrency: - group: test-loop-ext4-${{ github.ref }} - cancel-in-progress: true - -jobs: - call: - uses: ./.github/workflows/_test-under.yaml - with: - backend: "loop" - backend-version: "ext4" - permissions: - contents: read - actions: read diff --git a/.github/workflows/test-loop-vfat.yaml b/.github/workflows/test-loop-vfat.yaml deleted file mode 100644 index 71e84bf..0000000 --- a/.github/workflows/test-loop-vfat.yaml +++ /dev/null @@ -1,40 +0,0 @@ -name: Loop vfat - -# Thin dispatcher so this flavour gets a distinct README badge. -# Delegates all work to _test-under.yaml (reusable). -# -# `--fs vfat` is git-annex's canonical "crippled filesystem" flavour -# (no fifos, no symlinks, case-insensitive) and triggers adjusted- -# branch handling in git-annex init. - -on: - push: - branches: [main, master] - paths: - - "bin/**" - - "provision/**" - - ".github/workflows/_test-under.yaml" - - ".github/workflows/test-loop-vfat.yaml" - pull_request: - paths: - - "bin/**" - - "provision/**" - - ".github/workflows/_test-under.yaml" - - ".github/workflows/test-loop-vfat.yaml" - workflow_dispatch: - schedule: - - cron: "17 5 * * 1" # Mon 05:17 UTC - -concurrency: - group: test-loop-vfat-${{ github.ref }} - cancel-in-progress: true - -jobs: - call: - uses: ./.github/workflows/_test-under.yaml - with: - backend: "loop" - backend-version: "vfat" - permissions: - contents: read - actions: read diff --git a/.github/workflows/test-nfs.yaml b/.github/workflows/test-nfs.yaml deleted file mode 100644 index e62ad29..0000000 --- a/.github/workflows/test-nfs.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: NFS - -# Thin dispatcher so this flavour gets a distinct README badge. -# Delegates all work to _test-under.yaml (reusable). -# -# NFS backend uses `nfs-kernel-server` for a localhost loopback export; -# no version to pin (kernel + nfs-utils are ambient on the runner OS). - -on: - push: - branches: [main, master] - paths: - - "bin/**" - - "provision/**" - - ".github/workflows/_test-under.yaml" - - ".github/workflows/test-nfs.yaml" - pull_request: - paths: - - "bin/**" - - "provision/**" - - ".github/workflows/_test-under.yaml" - - ".github/workflows/test-nfs.yaml" - workflow_dispatch: - schedule: - - cron: "17 5 * * 1" # Mon 05:17 UTC - -concurrency: - group: test-nfs-${{ github.ref }} - cancel-in-progress: true - -jobs: - call: - uses: ./.github/workflows/_test-under.yaml - with: - backend: "nfs" - backend-version: "n/a" - permissions: - contents: read - actions: read diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..ad78515 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,141 @@ +name: test + +# The whole eval-under matrix, in one workflow. +# +# Cells come from .github/matrix.yaml via bin/ci/matrix-json.sh, rather +# than being spelled out here: adding a filesystem or a suite should be +# a data edit, and the shell scripts in bin/ci/ read the same file, so +# they cannot drift from what CI actually runs. +# +# All non-trivial shell logic lives in bin/ci/*.sh so it can be +# shellcheck'd and exercised locally in the Vagrant VM. See +# .claude/CLAUDE.md. + +on: + push: + branches: [main, master] + paths: + - "bin/**" + - "provision/**" + - ".github/matrix.yaml" + - ".github/workflows/test.yaml" + pull_request: + paths: + - "bin/**" + - "provision/**" + - ".github/matrix.yaml" + - ".github/workflows/test.yaml" + workflow_dispatch: + schedule: + - cron: "17 5 * * 1" # Mon 05:17 UTC + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + matrix: + runs-on: ubuntu-latest + outputs: + cells: ${{ steps.cells.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - id: cells + run: echo "matrix=$(bin/ci/matrix-json.sh)" >> "$GITHUB_OUTPUT" + + test: + needs: matrix + name: ${{ matrix.name }} + # ubuntu-22.04: kernel 5.15/6.5, within BeeGFS 7.4.x and 8.x DKMS + # support. ubuntu-24.04 hosted runners ship 6.17-azure which BeeGFS + # kernel modules cannot build against. NFS + loop backends don't + # need this, but sharing the runner OS keeps the matrix uniform. + runs-on: ubuntu-22.04 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.matrix.outputs.cells) }} + permissions: + contents: read + actions: read # to download artifacts from con/git-annex + steps: + - uses: actions/checkout@v4 + + - name: Install backend client dependencies + run: bin/ci/install-backend.sh "${{ matrix.backend }}" "${{ matrix.version }}" + + - name: Install target ${{ matrix.target }} + run: bin/ci/install-target.sh "${{ matrix.target }}" + + # Only the git-annex target needs the daily build (and the token + # that fetching it requires). + - name: Fetch latest git-annex daily build from con/git-annex + if: matrix.target == 'git-annex' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bin/ci/install-git-annex-daily.sh + + - name: Configure git identity + run: | + git config --global user.email test@github.land + git config --global user.name "GitHub Almighty" + + - name: Run ${{ matrix.target }} under ${{ matrix.backend }} + run: >- + sudo -E bin/ci/run-under.sh + "${{ matrix.backend }}" "${{ matrix.version }}" "${{ matrix.target }}" + + - name: Dump failure logs + if: failure() + run: >- + bin/ci/dump-failure-logs.sh + "${{ matrix.backend }}" "${{ matrix.version }}" "${{ matrix.target }}" + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: logs-${{ matrix.slug }} + path: | + /var/log/beegfs-* + /opt/eval-under-src/git/t/test-results/** + if-no-files-found: ignore + + # One tiny artifact per cell, read back by the `badges` job. A + # matrix job cannot contribute to a `needs.*.outputs` map, so this + # is how a fan-out reports 20 individual verdicts to a fan-in. + - name: Record cell result + if: always() + run: | + mkdir -p result + echo "${{ job.status }}" > result/conclusion + + - name: Upload cell result + if: always() + uses: actions/upload-artifact@v4 + with: + name: result-${{ matrix.slug }} + path: result/conclusion + + publish: + needs: test + # always(): the point is to publish the red cells too. + # Not on pull_request: a PR must not rewrite the status the README + # and the report page show for master. + if: always() && github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: write + actions: read # to resolve each cell's job URL for the report + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + pattern: result-* + path: results + + - name: Merge results and publish the status site + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bin/ci/publish-status.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7bb7933 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.py[cod] +__pycache__/ diff --git a/GOTCHAS.md b/GOTCHAS.md new file mode 100644 index 0000000..ff0883f --- /dev/null +++ b/GOTCHAS.md @@ -0,0 +1,233 @@ +# Gotchas + +A result from this matrix only means something relative to *how* the +filesystem was made and mounted. `vfat` mounted with `fmask=0177` +behaves differently from `vfat` mounted with kernel defaults; NFS +exported `root_squash` behaves differently from `no_root_squash`. So +this file records two things: + +1. **[The settings](#backend-settings)** each backend actually uses, and + what each one implies for the suites running on top. +2. **[The known-red cells](#known-red-cells)** and their root causes, so + nobody re-investigates a failure that is already understood. + +If you add a backend or change a mount option, update this file in the +same commit. A knob that is not written down here is a knob that will be +rediscovered the hard way. + +## Backend settings + +### Loop (`bin/eval-under-loop`) + +A sparse backing image is `dd`'d, `mkfs.`'d, and loop-mounted. + +| Knob | Value | Why | +| --- | --- | --- | +| `mkfs` options | none -- distro defaults | Whatever a user gets from `mkfs.ext4 /dev/sdX`, deliberately. | +| Image size | per target, `target_loop_size_mb()` in `bin/ci/matrix.sh` | `git annex test` needs room for many small objects; the other three do not. | +| Mount (vfat, msdos, exfat, ntfs) | `-o uid=,gid=` | These filesystems store no ownership. Without `uid=`, everything belongs to root and an unprivileged wrapped command cannot write. | +| Mount (everything else) | plain `mount`, then `chown ` on the mountpoint | ext4/xfs/btrfs carry real ownership; setting it once on the root is enough. | + +**The vfat consequence worth knowing.** `fmask`, `dmask` and `umask` are +left at kernel defaults, so every file on the mount reads as mode `0755` +and every directory as `0755`. `chmod` cannot change that -- vfat has one +`read-only` bit and nothing else. Two things follow: + +- Every file on a vfat mount is *executable* as far as `test -x` is + concerned. This is not a detail; it is the direct cause of most of the + `Loop vfat / git testsuite` failures (see below). +- Mounting instead with `fmask=0177,dmask=0077`, or with `showexec` (which + restricts the execute bit to `.com`/`.exe`/`.bat`), would make several of + those failures disappear. We do **not** do that: the point of the vfat + row is to show what a default-mounted vfat does to a POSIX-assuming + tool. Changing the mask would be measuring a different filesystem. + +### NFS (`bin/eval-under-nfs`) + +A local directory is exported to `localhost` and re-mounted over NFS. + +| Knob | Value | Why | +| --- | --- | --- | +| Export | `exportfs -o rw,async localhost:` | `--sync` switches to `rw,sync`. | +| Mount | `mount -t nfs -o rw,async localhost:` | | +| Protocol version | whatever the kernel negotiates (4.2 on ubuntu-22.04 runners) | **Not pinned** -- see [Not yet covered](#not-yet-covered). | +| Squashing | `root_squash` (the kernel default), *except* for targets that opt out | | + +**Export options and mount options are different namespaces.** `sync` and +`async` are valid in both; `no_root_squash` is export-only and `mount(8)` +rejects it outright. The script builds the two option strings separately +for exactly this reason -- if you add an option, work out which side it +belongs on first. + +**`root_squash` is the default on purpose.** A normal user's NFS home is +squashed, and reproducing that is half the reason the NFS backend exists. +But two targets cannot measure anything under it: + +- **pjdfstest** is largely a set of privileged-vs-unprivileged assertions + and refuses to run as non-root at all. +- **stress-ng**'s `chown` and `mknod` stressors need `CAP_CHOWN` / + `CAP_MKNOD`. + +So `--no-root-squash` (env: `EVAL_UNDER_NFS_NO_ROOT_SQUASH`) exports with +`no_root_squash` *and* keeps the wrapped command running as root. +`target_needs_root()` in `bin/ci/matrix.sh` decides which targets get it; +`git` and `git-annex` deliberately do not, and run squashed like a real +user would. The loop and BeeGFS backends already run the wrapped command +as root, so the flag is a no-op there. + +### BeeGFS (`bin/eval-under-beegfs`) + +A containerised cluster (`fixtures/beegfs/docker-compose-v{7,8}.yml`) plus +a kernel module built against the runner's kernel. + +| Knob | Value | Why | +| --- | --- | --- | +| Mount | `mount -t beegfs beegfs_nodev -o cfgFile=` | | +| Client conf | `fixtures/beegfs/beegfs-client.conf.template` | Auth disabled, all daemons on `127.0.0.1`, non-default ports (8004-8008) so nothing collides with the runner. | +| `sysMountSanityCheckMS` | `0` **on v8 only** | BeeGFS v8 dropped the standalone `beegfs-helperd` binary; with no helperd the sanity check cannot complete and the mount would hang. v7 keeps the check. | + +## Known-red cells + +A red cell here is a finding, not a bug report against this repo. These +are the ones already run down. + +### `Loop vfat / *` -- all four targets + +vfat is not a POSIX filesystem. No symlinks, no ownership, no +permissions, no hardlinks, 2-second timestamp granularity, +case-insensitive names, and a restricted filename charset (`:` `?` `*` +`"` `<` `>` `|` are all illegal, and git's own test suite creates +filenames using several of them). Every target trips over some subset. +The row exists to show *which* subset, per layer. + +### `Loop vfat / git testsuite` -- the interesting one + +Worth spelling out, because the obvious reaction is "surely git's own +suite handles this?" + +Git *does* probe the filesystem for some of its prerequisites -- +`SYMLINKS` is `ln -s x y && test -h y`, `CASE_INSENSITIVE_FS` writes +`CamelCase` and reads back `camelcase`, `FILEMODE` consults +`core.filemode`, which git auto-detects. Those all come out correct on +vfat, and the tests gated on them skip cleanly. + +`POSIXPERM` is not one of them. In `t/test-lib.sh` it is set from +`uname -s`: + +```sh +case $uname_s in +Darwin) test_set_prereq POSIXPERM ;; +*MINGW*) # no POSIX permissions + ;; +*CYGWIN*) test_set_prereq POSIXPERM ;; +*) test_set_prereq POSIXPERM ;; # <-- Linux lands here, always +esac +``` + +On Linux `POSIXPERM` is unconditionally true, whatever the work tree is +sitting on. So on vfat git runs every permission-dependent assertion +against a filesystem that reports mode `0755` for everything. + +The clearest instance is `t0001-init.sh`, whose `check_config()` helper +contains: + +```sh +if test_have_prereq POSIXPERM && test -x "$1/config" +then + echo "$1/config is executable?" + return 1 +fi +``` + +On vfat `.git/config` *is* executable, so `check_config` fails. It has 13 +call sites in `t0001-init.sh`, and the vfat cell reports exactly 13 +failures in that script. + +**This is not a git bug and not a regression.** Nobody upstream runs +git's test suite on a vfat work tree, so an OS-derived `POSIXPERM` has +never cost them anything. It is a fair finding about the *test suite's* +portability assumptions rather than about git the program -- which is +precisely the kind of thing a backends × targets grid is for. + +### `NFS (localhost) / pjdfstest` + +Runs to completion (238 files, ~8800 assertions) and reports real +divergence, not a setup failure: + +- `chown/00.t`: ~106 of 1280 failing, plus a batch of "TODO passed" + assertions -- upstream expects those to fail and NFS passes them. + Mostly setuid/setgid clearing behaviour on `chown`. +- `chmod/00.t`: 1 failure. +- `unlink/14.t`: 1 failure. + +Note this cell runs `--no-root-squash` (see above); under the default +`root_squash` the suite refuses to start at all. + +### `BeeGFS 7.4.6 / pjdfstest`, `BeeGFS 8.1.0 / pjdfstest` + +Real failures on both versions, not yet broken down per assertion. + +### `BeeGFS * / git-annex test` + +The original motivating bug. Note the useful negative result now +available beside it: `BeeGFS * / git testsuite` **passes** on both +versions. Whatever BeeGFS does differently, it is not breaking git's +index, refs, or object plumbing -- so the cause sits in what git-annex +layers on top, or in the syscalls the pjdfstest column is flagging. + +### `Loop ext4 / git-annex test` + +Pre-dates the matrix; ext4 is the control row, so this one *is* a real +bug worth chasing rather than a filesystem property. + +## Red that is not a finding + +Distinct from the cells above: these are harness races, and the fix is in +this repo rather than in anything under test. + +### BeeGFS: `chown: cannot access '/mnt/beegfs': Communication error on send` + +`mount -t beegfs` returns as soon as the client module has registered +with mgmtd and downloaded the node groups. That is not the same as its +connections to the meta and storage nodes being usable: the kernel logs +`BeeGFS mount ready` and the very next metadata operation can still fail +with `ECOMM`. `start_cluster()` waiting for the three daemons to bind +does not help -- a listening port only proves the *server* side is up. + +It presents as a cell that was green last run and red this one, with a +single-line error and no suite output at all, which reads like a finding +about the filesystem and is not. + +`wait_for_mount_usable()` in `bin/eval-under-beegfs` now probes a fresh +mount with a real create + write + read-back (meta node for the create, a +storage target for the write) and only proceeds once that succeeds, up to +`--mount-ready-wait` / `EVAL_UNDER_BEEGFS_MOUNT_READY_WAIT` seconds. If +it ever does time out, the error carries the last probe failure and a +`dmesg | grep beegfs` tail, so the next occurrence is diagnosable from +the job log alone. + +## Reading a red cell + +1. Look at the summary block at the end of the job log. Every target + ends with a machine-countable one: `prove`'s `Test Summary Report` + for `git` and `pjdfstest`, a pass/skip/fail tally for `stress-ng`. +2. Then the `=== ... failures ===` dump printed by + `bin/ci/dump-failure-logs.sh`, which names the failing assertions and + their output. +3. Then the uploaded `logs---` artifact, which + carries the per-script `.out` files in full. + +If a cell produces no summary at all, the suite died before finishing -- +check the mount actually came up. + +## Not yet covered + +- **NFS variants.** One localhost export with kernel-negotiated defaults + is a thin proxy for "NFS". The settings that actually bite on HPC are + protocol version (v3 vs v4.x), attribute caching (`ac` vs `noac` / + `actimeo=0`), locking (`lock` vs `nolock`, and whether `rpc.statd` is + even up), `sync` vs `async` on the export, and squashing. Each is a + plausible row of its own. +- **Per-assertion breakdown** of the BeeGFS pjdfstest failures. +- **A `Loop vfat` mask variant**, if we ever want to separate "vfat is + not POSIX" from "vfat mounted with defaults is not POSIX". diff --git a/README.md b/README.md index a3fb166..5101593 100644 --- a/README.md +++ b/README.md @@ -7,23 +7,109 @@ timestamp granularity, locking, permissions, adjusted-branch fallbacks, `root_squash` interactions) in tools like git-annex, DataLad, and rsync -- classes of bugs that don't show up in plain-ext4 CI. -git-annex is the immediate demo target: this repo runs the full -`git annex test` suite against every backend on every push. The harness -itself is backend-agnostic -- new backends are dropped in as -`bin/eval-under-` scripts (see below). +git-annex is the immediate demo target: this repo runs `git annex test` +-- plus git's own testsuite, pjdfstest, and a curated set of stress-ng +filesystem stressors -- against every backend on every push. The harness +itself is both backend- and suite-agnostic: new filesystems drop in as +`bin/eval-under-` scripts, new suites as `bin/ci/target-.sh` +(see below). + +> **Read [GOTCHAS.md](GOTCHAS.md) before drawing conclusions from a red +> cell.** It records the exact mkfs / mount / export settings each +> backend uses -- results only mean something relative to those -- and +> the root cause of every failure already run down, so nobody +> re-investigates a known one. ## CI status -| Backend | Status | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| BeeGFS 7.4.6 | [![BeeGFS 7.4.6](https://github.com/yarikoptic/beegfs-test/actions/workflows/test-beegfs-7.4.6.yaml/badge.svg)](https://github.com/yarikoptic/beegfs-test/actions/workflows/test-beegfs-7.4.6.yaml) | -| BeeGFS 8.1.0 | [![BeeGFS 8.1.0](https://github.com/yarikoptic/beegfs-test/actions/workflows/test-beegfs-8.1.0.yaml/badge.svg)](https://github.com/yarikoptic/beegfs-test/actions/workflows/test-beegfs-8.1.0.yaml) | -| NFS (localhost) | [![NFS](https://github.com/yarikoptic/beegfs-test/actions/workflows/test-nfs.yaml/badge.svg)](https://github.com/yarikoptic/beegfs-test/actions/workflows/test-nfs.yaml) | -| Loop vfat | [![Loop vfat](https://github.com/yarikoptic/beegfs-test/actions/workflows/test-loop-vfat.yaml/badge.svg)](https://github.com/yarikoptic/beegfs-test/actions/workflows/test-loop-vfat.yaml) | -| Loop ext4 | [![Loop ext4](https://github.com/yarikoptic/beegfs-test/actions/workflows/test-loop-ext4.yaml/badge.svg)](https://github.com/yarikoptic/beegfs-test/actions/workflows/test-loop-ext4.yaml) | - -Each row runs the **full `git annex test`** against a fresh daily build -from [con/git-annex](https://github.com/con/git-annex). + +| Backend | git-annex test | git testsuite | stress-ng | pjdfstest | +| --- | --- | --- | --- | --- | +| BeeGFS 7.4.6 | [![BeeGFS 7.4.6 / git-annex test](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-7.4.6-git-annex.svg)](https://con.github.io/eval-under/#beegfs-7.4.6-git-annex) | [![BeeGFS 7.4.6 / git testsuite](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-7.4.6-git.svg)](https://con.github.io/eval-under/#beegfs-7.4.6-git) | [![BeeGFS 7.4.6 / stress-ng](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-7.4.6-stress-ng.svg)](https://con.github.io/eval-under/#beegfs-7.4.6-stress-ng) | [![BeeGFS 7.4.6 / pjdfstest](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-7.4.6-pjdfstest.svg)](https://con.github.io/eval-under/#beegfs-7.4.6-pjdfstest) | +| BeeGFS 8.1.0 | [![BeeGFS 8.1.0 / git-annex test](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-8.1.0-git-annex.svg)](https://con.github.io/eval-under/#beegfs-8.1.0-git-annex) | [![BeeGFS 8.1.0 / git testsuite](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-8.1.0-git.svg)](https://con.github.io/eval-under/#beegfs-8.1.0-git) | [![BeeGFS 8.1.0 / stress-ng](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-8.1.0-stress-ng.svg)](https://con.github.io/eval-under/#beegfs-8.1.0-stress-ng) | [![BeeGFS 8.1.0 / pjdfstest](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-8.1.0-pjdfstest.svg)](https://con.github.io/eval-under/#beegfs-8.1.0-pjdfstest) | +| NFS (localhost) | [![NFS (localhost) / git-annex test](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/nfs-git-annex.svg)](https://con.github.io/eval-under/#nfs-git-annex) | [![NFS (localhost) / git testsuite](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/nfs-git.svg)](https://con.github.io/eval-under/#nfs-git) | [![NFS (localhost) / stress-ng](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/nfs-stress-ng.svg)](https://con.github.io/eval-under/#nfs-stress-ng) | [![NFS (localhost) / pjdfstest](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/nfs-pjdfstest.svg)](https://con.github.io/eval-under/#nfs-pjdfstest) | +| Loop vfat | [![Loop vfat / git-annex test](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-vfat-git-annex.svg)](https://con.github.io/eval-under/#loop-vfat-git-annex) | [![Loop vfat / git testsuite](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-vfat-git.svg)](https://con.github.io/eval-under/#loop-vfat-git) | [![Loop vfat / stress-ng](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-vfat-stress-ng.svg)](https://con.github.io/eval-under/#loop-vfat-stress-ng) | [![Loop vfat / pjdfstest](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-vfat-pjdfstest.svg)](https://con.github.io/eval-under/#loop-vfat-pjdfstest) | +| Loop ext4 | [![Loop ext4 / git-annex test](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-ext4-git-annex.svg)](https://con.github.io/eval-under/#loop-ext4-git-annex) | [![Loop ext4 / git testsuite](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-ext4-git.svg)](https://con.github.io/eval-under/#loop-ext4-git) | [![Loop ext4 / stress-ng](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-ext4-stress-ng.svg)](https://con.github.io/eval-under/#loop-ext4-stress-ng) | [![Loop ext4 / pjdfstest](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-ext4-pjdfstest.svg)](https://con.github.io/eval-under/#loop-ext4-pjdfstest) | + + +Rows are **backends** (which filesystem the work happens on), columns +are **targets** (which suite is run on it). All 20 cells are one job +matrix in [`.github/workflows/test.yaml`](.github/workflows/test.yaml), +fanned out from [`.github/matrix.yaml`](.github/matrix.yaml) -- adding a +filesystem or a suite is a data edit, not a code edit. + +The badges are ours, not GitHub's: GitHub publishes one badge per +workflow *file*, so a per-cell grid is not something it can serve for a +single matrix workflow. Each run renders an SVG per cell onto the +[`gh-pages`](../../tree/gh-pages) branch, and **each badge links to that +cell's own job log** via the [status page](https://con.github.io/eval-under/) -- +GitHub has no stable URL for "the latest job of this matrix cell", so +the page's `#` anchor supplies the indirection. The page also +carries what a badge cannot: which run produced the result, how long +ago, and why a cell is red on purpose. + +State lives in +[`status.json`](../../blob/gh-pages/status.json) on that branch and is +*merged* across runs, so a partial run -- "Re-run failed jobs" being the +common one -- updates only its own cells instead of blanking the rest. + +Run `bin/ci/gen-readme-matrix.sh` after editing the matrix to refresh +the table. + +A red cell is not automatically a bug: vfat has no symlinks, ownership, +or xattrs, and NFS has its own locking and close-to-open rules. The +matrix exists to make *which* filesystem breaks *which* layer visible at +a glance. [GOTCHAS.md](GOTCHAS.md) lists the cells that are red for a +known reason, with the reason. + +## Test targets + +Budgets below are what the suite itself costs on the loop-ext4 cell; a +sync-heavy backend (NFS, BeeGFS) runs the same work considerably slower, +which is why each target carries its own timeout in `bin/ci/matrix.sh`. + +| Target | What it runs | Prep | Budget | +| ----------- | ----------------------------------------------------- | ----------------------------------- | ------ | +| `git-annex` | The full `git annex test` suite | Daily build from [con/git-annex][ga] | tens of minutes | +| `git` | 174 scripts of git's own testsuite (`t0*.sh t1*.sh`, ~10k assertions) | Source build, pinned tag | ~4 min build + ~2 min on ext4 | +| `stress-ng` | 20 curated filesystem stressors, one at a time, `--verify` | `apt install stress-ng` | ~2 min | +| `pjdfstest` | POSIX conformance: 238 scripts, ~8800 assertions | Source build, pinned commit | ~3.5 min on ext4 | + +[ga]: https://github.com/con/git-annex + +Why these four, in order of how deep they sit: + +- **stress-ng** is the cheapest and the bluntest: it hammers `rename`, + `link`/`symlink`, `locka`/`lockf`, `xattr`, `chmod`/`chown`, `utime` + with verification on, so a filesystem that returns success while doing + the wrong thing is caught before anything is built on top of it. Each + stressor runs separately, so "vfat has no xattrs" reports as a skip + rather than poisoning the run. +- **pjdfstest** is the specification check: it tells you *which syscall* + returned *which errno* where POSIX says otherwise. Where `git annex + test` says "something is wrong", this says `rename` returns `EEXIST` + instead of `ENOTEMPTY`. +- **git** is the integration check for the layer git-annex actually + stands on -- index, refs, object store, attributes. Git's suite is the + canonical filesystem-picky testbed and is maintained far more + rigorously than anything hand-written here. Only the trash directories + (where the tests' file operations happen) live on the mount; the build + itself stays on the runner's disk. It is driven through `prove`, git's + TAP harness, rather than the default `make test` target -- under `make` + the first failing script aborts the run before the totals are ever + printed, whereas `prove` finishes every script and ends with a summary + naming each failure. +- **git-annex** is the original motivation and the top of the stack. + +Pinned upstream refs live in `bin/ci/matrix.sh` and are bumped +deliberately: with a moving testsuite, a newly-red cell is ambiguous -- +did the filesystem regress, or did upstream add a test? git is pinned to +a release tag. pjdfstest is pinned to a commit rather than its one +upstream tag (`0.1`, 2016), which no longer compiles: `major()`, +`minor()` and `makedev()` moved to `` in glibc 2.28 and +the tree builds with `-Werror`, so those implicit declarations are hard +errors on any current toolchain. Master builds clean, so we pin a commit +on it instead of carrying a patch. ## Motivation @@ -37,10 +123,11 @@ doesn't ([`eval_under_nfs`](https://github.com/datalad/datalad/blob/maint/tools/ [`eval_under_testloopfs`](https://github.com/datalad/datalad/blob/maint/tools/eval_under_testloopfs)). This repo consolidates and generalises that pattern: one dispatcher -(`bin/eval-under`), one reusable CI workflow (`_test-under.yaml`), and -a small backend script per filesystem. New filesystems slot in -uniformly; the git-annex-under-BeeGFS coverage that motivated the repo -is now just one of several dispatcher workflows. +(`bin/eval-under`), one CI workflow driven by an external matrix +definition, a small backend script per filesystem, and a small target +script per test suite. Both axes slot in uniformly; the +git-annex-under-BeeGFS coverage that motivated the repo is now one cell +of that matrix. ## CLI usage @@ -72,19 +159,28 @@ the full flag / env-var / default table per backend. ## File layout -| Path | Purpose | -| ---------------------------------------- | ----------------------------------------------------------------------------- | -| `Vagrantfile` + `provision/` | Ubuntu 24.04 libvirt VM with docker + BeeGFS + NFS + loop deps + git-annex | -| `bin/eval-under` | Dispatcher: routes to `bin/eval-under-` | -| `bin/eval-under-beegfs` | BeeGFS backend (containerised cluster + kernel client mount) | -| `bin/eval-under-nfs` | NFS backend (localhost loopback export) | -| `bin/eval-under-loop` | Loop-device backend (dd + losetup + mkfs. + mount) | -| `fixtures/beegfs/docker-compose-v7.yml` | BeeGFS v7 test cluster (mgmtd + meta + storage), `network_mode: host` | -| `fixtures/beegfs/docker-compose-v8.yml` | Same, for BeeGFS v8.x (different mgmtd command style / gRPC control plane) | -| `fixtures/beegfs/beegfs-*.conf.template` | Minimal client + helperd confs for the throwaway cluster | -| `.github/workflows/_test-under.yaml` | Reusable workflow parameterised on `backend` + `backend-version` | -| `.github/workflows/test-*.yaml` | Per-flavour dispatchers (one badge each) | -| `drafts/git-annex-test-beegfs.yaml` | Copy-target workflow for `con/git-annex` (external PR target) | +| Path | Purpose | +| ---------------------------------------- | ---------------------------------------------------------------------------------- | +| `Vagrantfile` + `provision/` | Ubuntu 24.04 libvirt VM with docker + BeeGFS + NFS + loop deps + git-annex | +| `bin/eval-under` | Dispatcher: routes to `bin/eval-under-` | +| `bin/eval-under-beegfs` | BeeGFS backend (containerised cluster + kernel client mount) | +| `bin/eval-under-nfs` | NFS backend (localhost loopback export) | +| `bin/eval-under-loop` | Loop-device backend (dd + losetup + mkfs. + mount) | +| `fixtures/beegfs/docker-compose-v7.yml` | BeeGFS v7 test cluster (mgmtd + meta + storage), `network_mode: host` | +| `fixtures/beegfs/docker-compose-v8.yml` | Same, for BeeGFS v8.x (different mgmtd command style / gRPC control plane) | +| `fixtures/beegfs/beegfs-*.conf.template` | Minimal client + helperd confs for the throwaway cluster | +| `.github/matrix.yaml` | Single source of truth: backends x targets, pinned upstream refs, per-target knobs | +| `bin/ci/matrix.sh` | Shell accessors over `.github/matrix.yaml`, sourced by every other `bin/ci` script | +| `bin/ci/matrix-json.sh` | Renders that file as the workflow's `matrix:` value (via `fromJson`) | +| `bin/ci/install-target.sh` | Runner-side prep for a target (apt package, or source build at a pinned tag) | +| `bin/ci/target-.sh` | The suite itself, run inside the mount by `bin/ci/run-under.sh` | +| `bin/ci/gen-readme-matrix.sh` | Regenerates the README badge grid from `.github/matrix.yaml` | +| `bin/ci/render-badge.sh` | Renders one status badge as a self-contained SVG | +| `bin/ci/update-status.py` | Merges a run's per-cell results into the persistent `status.json` | +| `bin/ci/render-report.py` | Renders `status.json` into the badge set + the report page | +| `bin/ci/publish-status.sh` | Ties those together and pushes the site to `gh-pages` | +| `.github/workflows/test.yaml` | The whole matrix: one `matrix` job, 20 `test` cells, one `publish` job | +| `drafts/git-annex-test-beegfs.yaml` | Copy-target workflow for `con/git-annex` (external PR target) | ## Local iteration (VM) @@ -102,6 +198,13 @@ sudo bin/eval-under beegfs --set-home -- bash -c ' ' sudo bin/eval-under nfs --set-home -- git annex test sudo bin/eval-under loop --fs vfat --set-home -- git annex test + +# Or run a whole CI cell exactly as the runner would. install-target.sh +# is the one-off prep (source builds land in $EVAL_UNDER_SRC_DIR, not on +# the mount); run-under.sh then wraps the suite in the backend. +bin/ci/install-target.sh pjdfstest +sudo -E bin/ci/run-under.sh loop ext4 pjdfstest +sudo -E bin/ci/run-under.sh nfs n/a stress-ng ``` Optional: install `act` in the VM to replay the GitHub workflow locally. @@ -127,9 +230,26 @@ filesystem testing. 3. At the end, run the wrapped command with `TMPDIR`, `DATALAD_TESTS_TEMP_DIR`, and (if `--set-home`) `HOME` pointing at the mount. -4. Add a `.github/workflows/test-.yaml` dispatcher and a - row to the CI badge table above. -5. Update `provision/setup.sh` if the backend needs new host packages. +4. Add a row to `backends:` in `.github/matrix.yaml`, then run + `bin/ci/gen-readme-matrix.sh` to refresh the README grid above. + The workflow picks the new cells up on its own. Commit the result. +5. Teach `bin/ci/install-backend.sh` how to install its client packages. +6. Update `provision/setup.sh` if the backend needs new host packages. + +## Adding a new test target + +1. Write `bin/ci/target-.sh`. It runs *inside* the mount, with + `TMPDIR` (and `HOME`, with `--set-home`) already pointing at the + filesystem under test. Exit non-zero on failure; skip -- loudly -- + rather than fail on operations the filesystem genuinely cannot do. +2. Teach `bin/ci/install-target.sh` how to prepare it on the runner. + Build source trees into `$EVAL_UNDER_SRC_DIR` (the runner's own disk), + never onto the mount: only the suite's I/O should exercise the + filesystem under test. Pin any upstream checkout to a tag. +3. Add an entry to `targets:` in `.github/matrix.yaml` with its `label`, + `timeout`, `loop-size-mb`, `needs-root`, and `needs-git-annex`. +4. Run `bin/ci/gen-readme-matrix.sh` and commit the new README column. +5. `shellcheck bin/ci/*.sh bin/eval-under*` before committing. ## Upstream targets diff --git a/REUSE.toml b/REUSE.toml index 2095483..3400ada 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -12,6 +12,8 @@ path = [ "*.md", "Vagrantfile", ".noannex", + ".gitignore", + ".codespellrc", ".claude/**", ".datalad/**", ".github/**", diff --git a/bin/ci/dump-failure-logs.sh b/bin/ci/dump-failure-logs.sh index 2114638..6253b18 100755 --- a/bin/ci/dump-failure-logs.sh +++ b/bin/ci/dump-failure-logs.sh @@ -9,15 +9,20 @@ # without masking the underlying failure. # # usage: -# bin/ci/dump-failure-logs.sh +# bin/ci/dump-failure-logs.sh [target] # -# beegfs: `docker compose logs` + dmesg-filtered-for-beegfs -# nfs/loop: full dmesg tail +# backend side: +# beegfs: `docker compose logs` + dmesg-filtered-for-beegfs +# nfs/loop: full dmesg tail +# target side: +# git: the failing assertions + output from t/test-results/*.out set -uo pipefail BACKEND="${1:?backend required}" VERSION="${2:-}" +TARGET="${3:-}" +SRC_DIR="${EVAL_UNDER_SRC_DIR:-/opt/eval-under-src}" case "$BACKEND" in beegfs) @@ -33,12 +38,85 @@ case "$BACKEND" in sudo dmesg | grep -i beegfs | tail -50 || true ;; nfs|loop) - echo "=== dmesg (last 100) ===" - sudo dmesg | tail -100 || true + # Filtered rather than `dmesg | tail -100`: on a hosted runner the + # last 100 kernel lines are almost entirely boot spam (hyperv, pci, + # apparmor), which buries the failure in the job log. Warnings and + # errors plus anything naming the filesystem under test is what + # actually matters here. + echo "=== dmesg (warnings and errors, last 40) ===" + sudo dmesg --level=emerg,alert,crit,err,warn 2>/dev/null | tail -40 || true + echo "=== dmesg (mentioning $BACKEND/$VERSION, last 30) ===" + sudo dmesg 2>/dev/null \ + | grep -iE "loop|nfs|${VERSION:-nomatch}" \ + | tail -30 || true ;; *) echo "unknown backend: $BACKEND" >&2 ;; esac +# git's testsuite runs under `prove` here (see bin/ci/target-git.sh), so +# there are no .counts files -- test-lib.sh only writes those when no TAP +# harness is active. What --verbose-log leaves behind is one .out per +# script, holding that script's full TAP stream. +# +# prove's own Test Summary Report is in the job log above this dump, but +# on a filesystem that fails broadly (vfat) this dump is thousands of +# lines, which scrolls that summary out of reach. So: detail for the +# first few failing scripts, and a compact roll-up printed LAST, so the +# end of the job log always answers "how many, and where" without +# scrolling. +GIT_DUMP_MAX_SCRIPTS="${EVAL_UNDER_GIT_DUMP_MAX_SCRIPTS:-8}" +GIT_DUMP_TAIL_LINES="${EVAL_UNDER_GIT_DUMP_TAIL_LINES:-30}" + +if [ "$TARGET" = "git" ]; then + results="$SRC_DIR/git/t/test-results" + echo "=== git testsuite failures ($results) ===" + if [ -d "$results" ]; then + # Pass 1: which scripts failed, and how badly. + names=() counts=() + for out in "$results"/*.out; do + [ -e "$out" ] || continue + n="$(grep -c '^not ok ' "$out" 2>/dev/null || true)" + [ "${n:-0}" -gt 0 ] || continue + names+=("$(basename "${out%.out}")") + counts+=("$n") + done + + if [ "${#names[@]}" -eq 0 ]; then + echo "no .out file recorded a failure (crash or setup failure?)" + else + # Pass 2: detail, for the first few only. The rest are in the + # uploaded artifact -- dumping 100 scripts inline helps nobody. + shown=0 + for i in "${!names[@]}"; do + [ "$shown" -lt "$GIT_DUMP_MAX_SCRIPTS" ] || break + shown=$((shown + 1)) + out="$results/${names[$i]}.out" + echo "--- ${names[$i]}: ${counts[$i]} failed ---" + grep '^not ok ' "$out" | head -40 || true + echo " ... last $GIT_DUMP_TAIL_LINES lines of ${names[$i]}.out:" + tail -"$GIT_DUMP_TAIL_LINES" "$out" | sed 's/^/ | /' || true + echo + done + if [ "${#names[@]}" -gt "$shown" ]; then + echo "($(( ${#names[@]} - shown )) further failing script(s) not detailed here" + echo " -- full .out files are in the uploaded logs-* artifact)" + echo + fi + + # Pass 3: the roll-up, deliberately last. + total=0 + echo "=== git testsuite summary: ${#names[@]} script(s) with failures ===" + for i in "${!names[@]}"; do + printf ' %-40s %s failed\n' "${names[$i]}" "${counts[$i]}" + total=$((total + counts[i])) + done + echo " $(printf '%-40s %s' 'TOTAL' "$total") failed assertion(s)" + fi + else + echo "no test-results directory (the suite never started?)" + fi +fi + exit 0 diff --git a/bin/ci/gen-readme-matrix.sh b/bin/ci/gen-readme-matrix.sh new file mode 100755 index 0000000..ed3d5b8 --- /dev/null +++ b/bin/ci/gen-readme-matrix.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2026 Yaroslav Halchenko +# SPDX-License-Identifier: MIT +# +# Generated with Claude Code +# +# Regenerate the README's CI matrix table from .github/matrix.yaml. +# +# Replaces the old gen-dispatchers.sh, which also had to emit 20 +# near-identical workflow files -- one per cell, purely so that each cell +# had a workflow-level badge URL. .github/workflows/test.yaml is now a +# single matrix over the same data, and the per-cell badges are SVGs we +# render ourselves onto the site branch (bin/ci/publish-status.sh). +# +# Each badge links to that cell's anchor on the report page, which is +# regenerated every run with a deep link to the cell's actual job log. +# GitHub has no stable URL for "the latest job of this matrix cell", so +# that indirection is what makes a badge clickable at all. +# +# The badge images are served from raw.githubusercontent rather than from +# Pages, so the grid renders even before Pages has been enabled on the +# repo; only the links need Pages. +# +# usage: +# bin/ci/gen-readme-matrix.sh # rewrite the README table +# bin/ci/gen-readme-matrix.sh --check # exit 1 if it is stale +# +# env: +# EVAL_UNDER_SITE_BRANCH branch holding the site (gh-pages) + +set -euo pipefail + +here="$(cd "$(dirname "$0")" && pwd)" +# matrix.sh is a sourced library, resolved at runtime relative to $here. +# shellcheck source=bin/ci/matrix.sh disable=SC1091 +. "$here/matrix.sh" + +root="$EVAL_UNDER_ROOT" +README="$root/README.md" +BRANCH="${EVAL_UNDER_SITE_BRANCH:-gh-pages}" +BEGIN_MARK='' +END_MARK='' + +CHECK=0 +[ "${1:-}" = "--check" ] && CHECK=1 + +grep -qF "$BEGIN_MARK" "$README" || { + echo "ERROR: $README has no '$BEGIN_MARK' marker" >&2 + exit 1 +} + +owner="${EVAL_UNDER_REPO_SLUG%%/*}" +name="${EVAL_UNDER_REPO_SLUG#*/}" + +badge_url() { + echo "https://raw.githubusercontent.com/$EVAL_UNDER_REPO_SLUG/$BRANCH/badges/$1.svg" +} + +# Anchor on the report page, which carries the deep link to this cell's +# most recent job log. +cell_url() { + echo "https://$owner.github.io/$name/#$1" +} + +table_md() { + printf '| Backend |' + for target in "${EVAL_UNDER_TARGETS[@]}"; do + printf ' %s |' "$(target_label "$target")" + done + printf '\n| --- |' + for _ in "${EVAL_UNDER_TARGETS[@]}"; do printf ' --- |'; done + printf '\n' + + for cell in "${EVAL_UNDER_BACKENDS[@]}"; do + IFS='|' read -r backend version label <<< "$cell" + printf '| %s |' "$label" + for target in "${EVAL_UNDER_TARGETS[@]}"; do + slug="$(cell_slug "$backend" "$version" "$target")" + printf ' [![%s / %s](%s)](%s) |' \ + "$label" "$(target_label "$target")" \ + "$(badge_url "$slug")" "$(cell_url "$slug")" + done + printf '\n' + done +} + +new_readme="$(mktemp)" +trap 'rm -f "$new_readme"' EXIT + +awk -v begin="$BEGIN_MARK" -v end="$END_MARK" -v table="$(table_md)" ' + $0 == begin { print; print table; skip = 1; next } + $0 == end { skip = 0 } + !skip { print } +' "$README" > "$new_readme" + +if [ "$CHECK" = 1 ]; then + if diff -q "$README" "$new_readme" >/dev/null 2>&1; then + echo "up to date: README CI matrix" + exit 0 + fi + echo "stale: README.md CI matrix (run bin/ci/gen-readme-matrix.sh)" >&2 + exit 1 +fi + +cp "$new_readme" "$README" +echo "refreshed the README CI matrix ($((${#EVAL_UNDER_BACKENDS[@]} * ${#EVAL_UNDER_TARGETS[@]})) cells)" diff --git a/bin/ci/install-target.sh b/bin/ci/install-target.sh new file mode 100755 index 0000000..508eebc --- /dev/null +++ b/bin/ci/install-target.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2026 Yaroslav Halchenko +# SPDX-License-Identifier: MIT +# +# Generated with Claude Code +# +# Runner-side install/build of an eval-under *test target* (the suite we +# run under the backend filesystem). Counterpart of install-backend.sh, +# which installs the backend (the filesystem) instead. +# +# Everything built here lands on the runner's own disk +# ($EVAL_UNDER_SRC_DIR), never on the filesystem under test -- only the +# suite's own I/O should exercise the mount. +# +# usage: +# bin/ci/install-target.sh +# +# target = git-annex | git | stress-ng | pjdfstest +# +# env overrides: +# EVAL_UNDER_SRC_DIR where to clone/build (/opt/eval-under-src) +# EVAL_UNDER_GIT_REF git tag to build (see bin/ci/matrix.sh) +# EVAL_UNDER_PJDFSTEST_REF pjdfstest tag to build (see bin/ci/matrix.sh) +# +# Idempotent enough for CI re-runs: an already-built tree is left alone. + +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive + +here="$(cd "$(dirname "$0")" && pwd)" +# matrix.sh is a sourced library, resolved at runtime relative to $here. +# SC1091 is only silenceable by running shellcheck with -x; disable it +# here so a plain `shellcheck bin/ci/*.sh` stays clean. +# shellcheck source=bin/ci/matrix.sh disable=SC1091 +. "$here/matrix.sh" + +TARGET="${1:?target required (git-annex|git|stress-ng|pjdfstest)}" +target_known "$TARGET" || { + echo "unknown target: $TARGET (expected: ${EVAL_UNDER_TARGETS[*]})" >&2 + exit 1 +} + +# Give unattended-upgrades a moment on ubuntu-22.04 runners rather than +# hard-failing on a dpkg lock (same rationale as install-backend.sh). +APT_LOCK_TIMEOUT=(-o "DPkg::Lock::Timeout=60") + +apt_install() { + sudo apt-get "${APT_LOCK_TIMEOUT[@]}" install -y --no-install-recommends "$@" +} + +apt_update() { + sudo apt-get "${APT_LOCK_TIMEOUT[@]}" update -qq +} + +# Shallow-fetch a pinned ref into $EVAL_UNDER_SRC_DIR/, or report +# that it is already there. Sets $checkout. +# +# init + fetch + checkout rather than `clone --branch`, because --branch +# only accepts a branch or tag name: this way a ref can equally be a +# commit SHA (which is what pjdfstest needs -- see EVAL_UNDER_PJDFSTEST_REF +# in bin/ci/matrix.sh). +checkout="" +fetch_pinned() { + local name="$1" url="$2" ref="$3" + checkout="$EVAL_UNDER_SRC_DIR/$name" + if [ -d "$checkout/.git" ]; then + echo "I: $name already checked out at $checkout" + return 0 + fi + sudo mkdir -p "$EVAL_UNDER_SRC_DIR" + sudo chown "$(id -u):$(id -g)" "$EVAL_UNDER_SRC_DIR" + echo "I: fetching $url @ $ref -> $checkout" + mkdir -p "$checkout" + git -C "$checkout" init -q + git -C "$checkout" remote add origin "$url" + git -C "$checkout" fetch -q --depth 1 origin "$ref" + git -C "$checkout" checkout -q FETCH_HEAD + git -C "$checkout" --no-pager log -1 --format='I: %H %s' +} + +install_git_annex() { + # Nothing to build: the daily git-annex build is installed by + # bin/ci/install-git-annex-daily.sh, which needs a GH token and so + # stays a separate workflow step. + echo "I: target git-annex needs no extra build step here" + echo "I: (git-annex itself comes from bin/ci/install-git-annex-daily.sh)" +} + +install_git() { + apt_update + # Build deps for a functional git (curl/expat/gettext matter for the + # t0xxx/t1xxx range: without them the corresponding tests silently skip). + apt_install build-essential gettext perl \ + zlib1g-dev libssl-dev libcurl4-openssl-dev libexpat1-dev + + # bin/ci/target-git.sh drives the suite through `make prove`, so the + # TAP harness itself is a hard dependency. It ships in perl-modules + # (pulled in by perl above), but assert it here: a missing prove + # would otherwise surface as an opaque make failure minutes into the + # run, on the mount, after the whole build. + command -v prove >/dev/null || { + echo "ERROR: prove not found -- install perl-modules (TAP harness)" >&2 + exit 3 + } + + fetch_pinned git https://github.com/git/git "$EVAL_UNDER_GIT_REF" + + if [ -x "$checkout/git" ]; then + echo "I: git already built ($("$checkout/git" --version))" + return 0 + fi + # NO_TCLTK: gitk/git-gui are irrelevant here and pull in a tcl + # toolchain. Everything else stays default so the testsuite's + # prerequisites resolve the way upstream CI sees them. + make -C "$checkout" -j"$(nproc)" NO_TCLTK=1 + "$checkout/git" --version +} + +install_stress_ng() { + apt_update + apt_install stress-ng + stress-ng --version +} + +install_pjdfstest() { + apt_update + apt_install build-essential autoconf automake libtool perl + + fetch_pinned pjdfstest https://github.com/pjd/pjdfstest "$EVAL_UNDER_PJDFSTEST_REF" + + if [ -x "$checkout/pjdfstest" ]; then + echo "I: pjdfstest already built" + return 0 + fi + ( cd "$checkout" && autoreconf -ifs && ./configure && make pjdfstest ) + test -x "$checkout/pjdfstest" +} + +case "$TARGET" in + git-annex) install_git_annex ;; + git) install_git ;; + stress-ng) install_stress_ng ;; + pjdfstest) install_pjdfstest ;; +esac diff --git a/bin/ci/matrix-json.sh b/bin/ci/matrix-json.sh new file mode 100755 index 0000000..6c1ca13 --- /dev/null +++ b/bin/ci/matrix-json.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2026 Yaroslav Halchenko +# SPDX-License-Identifier: MIT +# +# Generated with Claude Code +# +# Render .github/matrix.yaml as the value of a GitHub Actions `matrix:` +# key -- i.e. {"include": [ {...}, ... ]} -- for consumption via +# fromJson() in .github/workflows/test.yaml. +# +# Each entry carries everything the job body needs, so the workflow +# never has to recompute anything about a cell: +# +# name job name, e.g. "BeeGFS 7.4.6 / git testsuite". Set as the +# job's `name:` so the checks list reads like the README grid +# instead of GitHub's default "test (beegfs, 7.4.6, git)". +# backend eval-under backend (beegfs | nfs | loop) +# version backend version, or "n/a" +# target suite to run under it +# slug filename-safe cell id, for artifact names +# +# usage: +# bin/ci/matrix-json.sh # all cells +# +# The output is a single line: GitHub's `fromJson` wants one value, and +# a multi-line $GITHUB_OUTPUT needs heredoc quoting for no benefit. + +set -euo pipefail + +here="$(cd "$(dirname "$0")" && pwd)" +# matrix.sh is a sourced library, resolved at runtime relative to $here. +# shellcheck source=bin/ci/matrix.sh disable=SC1091 +. "$here/matrix.sh" + +entries=() +for cell in "${EVAL_UNDER_BACKENDS[@]}"; do + IFS='|' read -r backend version label <<< "$cell" + for target in "${EVAL_UNDER_TARGETS[@]}"; do + entries+=("$backend|$version|$label|$target|$(target_label "$target")|$(cell_slug "$backend" "$version" "$target")") + done +done + +printf '%s\n' "${entries[@]}" | python3 -c ' +import json, sys + +include = [] +for line in sys.stdin: + line = line.rstrip("\n") + if not line: + continue + backend, version, blabel, target, tlabel, slug = line.split("|") + include.append({ + "name": "%s / %s" % (blabel, tlabel), + "backend": backend, + "version": version, + "target": target, + "slug": slug, + }) + +if not include: + sys.exit("matrix-json: no cells produced") +print(json.dumps({"include": include}, separators=(",", ":"))) +' diff --git a/bin/ci/matrix.sh b/bin/ci/matrix.sh new file mode 100755 index 0000000..9a2dd70 --- /dev/null +++ b/bin/ci/matrix.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2026 Yaroslav Halchenko +# SPDX-License-Identifier: MIT +# +# Generated with Claude Code +# +# Shared CI matrix accessors for eval-under. Sourced (not executed) by +# bin/ci/{install-target,run-under,target-*,matrix-json,...}.sh. +# +# This file used to *hold* the matrix. It now *reads* it, from +# .github/matrix.yaml -- so the workflow and these scripts cannot drift +# apart, because both parse the same file. Everything here is accessors +# over that data plus the naming rules. +# +# shellcheck shell=bash + +# Repo root, resolved from this file's location so callers can be run +# from anywhere (CI checks out to a different path than the Vagrant VM). +EVAL_UNDER_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +EVAL_UNDER_MATRIX_FILE="${EVAL_UNDER_MATRIX_FILE:-$EVAL_UNDER_ROOT/.github/matrix.yaml}" + +[ -r "$EVAL_UNDER_MATRIX_FILE" ] || { + echo "matrix.sh: cannot read $EVAL_UNDER_MATRIX_FILE" >&2 + # Sourced normally, so `return` is the live path; the `exit` is the + # fallback for anyone who runs this file directly. shellcheck only + # sees the static text and calls the second half unreachable. + # shellcheck disable=SC2317 + return 1 2>/dev/null || exit 1 +} + +# Parse once, into shell. python3 + PyYAML rather than `yq`: both are +# present on GitHub's ubuntu images, but python3 is also what every +# developer box and the Vagrant VM already have, and it lets us fail +# loudly on a malformed file instead of silently yielding "null". +# +# Emits assignments only; anything unexpected is a parse error there, +# not a surprise `eval` here. +eval "$(python3 - "$EVAL_UNDER_MATRIX_FILE" <<'PYEOF' +import sys, shlex, yaml + +with open(sys.argv[1]) as fh: + d = yaml.safe_load(fh) + +targets = d["targets"] +backends = d["backends"] +if not targets or not backends: + sys.exit("matrix.yaml: empty targets or backends") + +def q(v): + return shlex.quote(str(v)) + +out = [] +out.append("EVAL_UNDER_TARGETS=(%s)" % " ".join(q(t["name"]) for t in targets)) +out.append("EVAL_UNDER_BACKENDS=(%s)" % " ".join( + q("%s|%s|%s" % (b["backend"], b["version"], b["label"])) for b in backends)) + +out.append("declare -A _EU_LABEL=(%s)" % " ".join( + "[%s]=%s" % (q(t["name"]), q(t["label"])) for t in targets)) +out.append("declare -A _EU_TIMEOUT=(%s)" % " ".join( + "[%s]=%s" % (q(t["name"]), q(t["timeout"])) for t in targets)) +out.append("declare -A _EU_LOOP_MB=(%s)" % " ".join( + "[%s]=%s" % (q(t["name"]), q(t["loop-size-mb"])) for t in targets)) +out.append("declare -A _EU_NEEDS_ROOT=(%s)" % " ".join( + "[%s]=%s" % (q(t["name"]), q(int(bool(t["needs-root"])))) for t in targets)) +out.append("declare -A _EU_NEEDS_GA=(%s)" % " ".join( + "[%s]=%s" % (q(t["name"]), q(int(bool(t["needs-git-annex"])))) for t in targets)) + +# Env overrides win, so these are defaults only. +out.append(": \"${EVAL_UNDER_REPO_SLUG:=%s}\"" % q(d["repo-slug"])) +out.append(": \"${EVAL_UNDER_SRC_DIR:=%s}\"" % q(d["src-dir"])) +out.append(": \"${EVAL_UNDER_GIT_REF:=%s}\"" % q(d["refs"]["git"])) +out.append(": \"${EVAL_UNDER_PJDFSTEST_REF:=%s}\"" % q(d["refs"]["pjdfstest"])) + +print("\n".join(out)) +PYEOF +)" || { + echo "matrix.sh: failed to parse $EVAL_UNDER_MATRIX_FILE" >&2 + # Sourced normally, so `return` is the live path; the `exit` is the + # fallback for anyone who runs this file directly. shellcheck only + # sees the static text and calls the second half unreachable. + # shellcheck disable=SC2317 + return 1 2>/dev/null || exit 1 +} + +export EVAL_UNDER_REPO_SLUG EVAL_UNDER_SRC_DIR +export EVAL_UNDER_GIT_REF EVAL_UNDER_PJDFSTEST_REF + +# Filename-safe identifier for a backend cell: "beegfs-7.4.6", "nfs", +# "loop-vfat". +backend_slug() { + local backend="$1" version="$2" + if [ "$version" = "n/a" ]; then + echo "$backend" + else + echo "$backend-$version" + fi +} + +# Filename-safe identifier for a whole matrix cell: "beegfs-7.4.6-git", +# "nfs-pjdfstest", "loop-vfat-stress-ng". +# +# Exists because the NFS backend's version is the literal "n/a", and a +# naive "--" therefore contains a slash. +# actions/upload-artifact rejects slashes in artifact names outright, so +# `logs-nfs-n/a-git` is a hard error -- one that stayed invisible until +# the git target started producing files to upload at all (before that, +# `if-no-files-found: ignore` short-circuited before the name was ever +# validated). +cell_slug() { + local backend="$1" version="$2" target="$3" + echo "$(backend_slug "$backend" "$version")-$target" +} + +target_known() { + local t + for t in "${EVAL_UNDER_TARGETS[@]}"; do + [ "$t" = "$1" ] && return 0 + done + return 1 +} + +# Human-readable column header / job-name fragment. +target_label() { echo "${_EU_LABEL[$1]:-$1}"; } +target_timeout() { echo "${_EU_TIMEOUT[$1]:-1800}"; } +target_loop_size_mb() { echo "${_EU_LOOP_MB[$1]:-100}"; } + +# Does this target need to run as root to mean anything? The loop and +# beegfs backends already run the wrapped command as root, but the NFS +# backend deliberately drops back to the invoking user and exports with +# root_squash -- exactly right for git-annex/git, useless for the other +# two. run-under.sh passes --no-root-squash for these. +target_needs_root() { [ "${_EU_NEEDS_ROOT[$1]:-0}" = 1 ]; } +target_needs_git_annex() { [ "${_EU_NEEDS_GA[$1]:-0}" = 1 ]; } diff --git a/bin/ci/publish-status.sh b/bin/ci/publish-status.sh new file mode 100755 index 0000000..94497bb --- /dev/null +++ b/bin/ci/publish-status.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2026 Yaroslav Halchenko +# SPDX-License-Identifier: MIT +# +# Generated with Claude Code +# +# Publish the CI status site: merge this run's per-cell results into the +# persistent status file, re-render the badges and the report page, and +# push the result to the site branch (default: gh-pages). +# +# Replaces the earlier publish-badges.sh, which derived the whole grid +# from a single run's artifacts. That was wrong for any run that does not +# cover every cell -- "Re-run failed jobs" being the obvious one -- and +# would have rewritten the untouched cells' badges to "unknown". State +# now lives in status.json on the site branch and is merged into, the +# same shape con/git-annex uses for its client badges. +# +# History on the site branch is kept rather than force-pushed: status.json +# then doubles as a log of when each cell changed state. +# +# usage: +# bin/ci/publish-status.sh [--dry-run] +# +# env: +# GH_TOKEN token with contents:write (required unless --dry-run) +# GITHUB_REPOSITORY owner/repo (default: repo-slug from matrix.yaml) +# GITHUB_RUN_ID used to fetch job URLs, and recorded per cell +# GITHUB_RUN_NUMBER } the per-cell monotonic guard +# GITHUB_RUN_ATTEMPT } +# EVAL_UNDER_SITE_BRANCH branch to publish to (default: gh-pages) +# EVAL_UNDER_RESULTS_DIR downloaded result-* artifacts (default: results) + +set -euo pipefail + +here="$(cd "$(dirname "$0")" && pwd)" +# matrix.sh is a sourced library, resolved at runtime relative to $here. +# shellcheck source=bin/ci/matrix.sh disable=SC1091 +. "$here/matrix.sh" + +DRY_RUN=0 +[ "${1:-}" = "--dry-run" ] && DRY_RUN=1 + +BRANCH="${EVAL_UNDER_SITE_BRANCH:-gh-pages}" +REPO="${GITHUB_REPOSITORY:-$EVAL_UNDER_REPO_SLUG}" +RESULTS="${EVAL_UNDER_RESULTS_DIR:-results}" + +work="$(mktemp -d)" +jobs_json="$(mktemp)" +trap 'rm -rf "$work" "$jobs_json"' EXIT + +# Existing site, so status.json can be merged into rather than replaced. +# A first run (no branch yet) starts from an empty tree. +remote="https://github.com/${REPO}" +[ -n "${GH_TOKEN:-}" ] && remote="https://x-access-token:${GH_TOKEN}@github.com/${REPO}" +if git clone -q --depth 1 --branch "$BRANCH" "$remote" "$work" 2>/dev/null; then + echo "I: cloned existing $BRANCH" +else + echo "I: $BRANCH does not exist yet; starting a fresh site" + rm -rf "$work"; mkdir -p "$work" + git init -q "$work" +fi + +# Per-cell log URLs. Job names come straight from the matrix entry's +# `name`, so they match exactly -- same resolution trick as +# con/git-annex's .github/workflows/tools/set-pr-status. +if [ -n "${GITHUB_RUN_ID:-}" ] && command -v gh >/dev/null 2>&1; then + if gh api --paginate "repos/${REPO}/actions/runs/${GITHUB_RUN_ID}/jobs" \ + > "$jobs_json" 2>/dev/null; then + echo "I: fetched job list for run ${GITHUB_RUN_ID}" + else + echo "W: could not fetch job list; report links will be empty" >&2 + : > "$jobs_json" + fi +else + echo "I: no run id or no gh; skipping job-URL resolution" + : > "$jobs_json" +fi + +"$here/update-status.py" "$work/status.json" "$RESULTS" --jobs "$jobs_json" +"$here/render-report.py" "$work/status.json" "$work" + +if [ "$DRY_RUN" = 1 ]; then + echo "I: --dry-run; site left in $work" + trap - EXIT + rm -f "$jobs_json" + exit 0 +fi + +: "${GH_TOKEN:?GH_TOKEN required to push (or pass --dry-run)}" + +cd "$work" +git config user.email "actions@github.com" +git config user.name "eval-under CI" +git add -A +if git diff --cached --quiet; then + echo "I: nothing changed; not pushing" + exit 0 +fi +git commit -q -m "Status for run ${GITHUB_RUN_NUMBER:-?} (${GITHUB_RUN_ID:-local})" +echo "I: pushing to $BRANCH" +git push -q "$remote" "HEAD:refs/heads/$BRANCH" +echo "I: published https://${REPO%%/*}.github.io/${REPO#*/}/" diff --git a/bin/ci/render-badge.sh b/bin/ci/render-badge.sh new file mode 100755 index 0000000..4bf8ef2 --- /dev/null +++ b/bin/ci/render-badge.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2026 Yaroslav Halchenko +# SPDX-License-Identifier: MIT +# +# Generated with Claude Code +# +# Render one status badge as a self-contained SVG on stdout. +# +# We draw these ourselves rather than linking shields.io endpoint +# badges: the grid is 20 cells, so a README render would otherwise be +# 20 third-party requests, and the badges would go blank whenever that +# service is unreachable. An SVG committed to the `badges` branch has +# neither problem and is diffable. +# +# Single-segment on purpose: in the README grid the row and column +# headers already name the cell, so a "BeeGFS 7.4.6 / git" prefix on +# every badge would trible the table width and say nothing. The full +# cell name goes in , which is what a screen reader announces +# and what a browser shows on hover. +# +# usage: +# bin/ci/render-badge.sh <status> [title] +# +# status = success | failure | cancelled | skipped | <anything else> +# title = tooltip / accessible name (default: the status text) +# +# e.g. +# bin/ci/render-badge.sh success "BeeGFS 7.4.6 / git testsuite" + +set -euo pipefail + +STATUS="${1:?status required}" +TITLE="${2:-}" + +# Colours match the shields.io "flat" palette so these sit comfortably +# next to any conventional badge elsewhere in the README. +case "$STATUS" in + success) text="passing"; color="#4c1" ;; + failure) text="failing"; color="#e05d44" ;; + cancelled) text="cancelled"; color="#9f9f9f" ;; + skipped) text="skipped"; color="#9f9f9f" ;; + *) text="unknown"; color="#9f9f9f" ;; +esac + +[ -n "$TITLE" ] || TITLE="$text" + +# Width: DejaVu Sans at 11px averages just under 7px/char for lowercase +# ASCII, plus 5px padding either side. Approximate is fine -- the text +# is centred, so a few px of slack shows up as symmetric padding rather +# than as clipping. +width=$(( ${#text} * 7 + 10 )) +mid=$(( width * 5 )) # centre, in the 10x-scaled text coordinate space + +# XML-escape the title: cell labels are plain ASCII today, but a future +# backend label with an "&" in it should not emit invalid SVG. +esc_title=$(printf '%s' "$TITLE" \ + | sed -e 's/&/\&/g' -e 's/</\</g' -e 's/>/\>/g' -e 's/"/\"/g') + +cat <<EOF +<svg xmlns="http://www.w3.org/2000/svg" width="$width" height="20" role="img" aria-label="$esc_title: $text"> + <title>$esc_title: $text + + + + + + + + + + + $text + $text + + +EOF diff --git a/bin/ci/render-report.py b/bin/ci/render-report.py new file mode 100755 index 0000000..06da586 --- /dev/null +++ b/bin/ci/render-report.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Yaroslav Halchenko +# SPDX-License-Identifier: MIT +# +# Generated with Claude Code +# +# Render the status file into the published site: one badge SVG per cell +# plus an index.html carrying the grid. +# +# The page is what makes the badges clickable in a useful way. GitHub +# publishes one badge per workflow *file*, and there is no stable URL for +# "the latest job of this matrix cell" -- so a per-cell link has to come +# from somewhere we control. Each cell here anchors as # and links +# to the exact job log of the run that produced its current state, the +# same shape as con/git-annex's con.github.io/git-annex-ci-reports. +# +# It also carries what a badge cannot: which run, how long ago, and the +# standing explanation for a cell that is red on purpose. +# +# usage: +# bin/ci/render-report.py + +from __future__ import annotations + +import argparse +import html +import json +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] + +# Cells that are red for a known, documented reason. Keeps the page +# honest: a red cell here is a finding, not a regression to chase. +# Keyed by slug; kept in sync with GOTCHAS.md by hand (there are ten). +KNOWN_RED = { + "loop-vfat-git-annex": "vfat has no symlinks or ownership", + "loop-vfat-git": "git's POSIXPERM prereq is set from uname, never probed", + "loop-vfat-stress-ng": "vfat lacks chown/xattr/hardlink semantics", + "loop-vfat-pjdfstest": "vfat is not a POSIX filesystem", + "beegfs-7.4.6-pjdfstest": "BeeGFS POSIX conformance gaps", + "beegfs-8.1.0-pjdfstest": "BeeGFS POSIX conformance gaps", + "beegfs-7.4.6-git-annex": "the bug this repo exists to characterise", + "beegfs-8.1.0-git-annex": "the bug this repo exists to characterise", + "nfs-pjdfstest": "NFS chown/setuid divergence (106 of 1280 assertions)", + "loop-ext4-git-annex": "pre-existing, predates this harness", +} + +STATE = { + "success": ("passing", "#4c1"), + "failure": ("failing", "#e05d44"), + "cancelled": ("cancelled", "#9f9f9f"), + "skipped": ("skipped", "#9f9f9f"), +} + +CSS = """ +:root { color-scheme: light dark; + --fg:#1f2328; --bg:#fff; --muted:#59636e; --line:#d1d9e0; --accent:#0969da; --hl:#fff8c5; } +@media (prefers-color-scheme: dark) { :root { + --fg:#f0f6fc; --bg:#0d1117; --muted:#9198a1; --line:#3d444d; --accent:#4493f8; --hl:#2d2a1f; } } +* { box-sizing:border-box } +body { margin:0; padding:2rem 1rem; background:var(--bg); color:var(--fg); + font:15px/1.5 -apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif; } +main { max-width:60rem; margin:0 auto } +h1 { font-size:1.5rem; margin:0 0 .25rem } +.sub { color:var(--muted); margin:0 0 1.5rem } +.wrap { overflow-x:auto; border:1px solid var(--line); border-radius:6px } +table { border-collapse:collapse; width:100%; min-width:44rem } +th,td { padding:.6rem .75rem; text-align:left; border-bottom:1px solid var(--line); vertical-align:top } +th { font-size:.8rem; text-transform:uppercase; letter-spacing:.04em; color:var(--muted); font-weight:600 } +tr:last-child td { border-bottom:0 } +td.cell:target { background:var(--hl); } +a { color:var(--accent); text-decoration:none } +a:hover { text-decoration:underline } +.meta { display:block; font-size:.75rem; color:var(--muted); margin-top:.3rem } +.why { display:block; font-size:.75rem; color:var(--muted); font-style:italic; margin-top:.15rem } +footer { margin-top:2rem; padding-top:1rem; border-top:1px solid var(--line); + color:var(--muted); font-size:.85rem } +""" + + +def render_badge(conclusion: str, title: str, out: Path) -> None: + out.parent.mkdir(parents=True, exist_ok=True) + r = subprocess.run( + [str(HERE / "render-badge.sh"), conclusion, title], + capture_output=True, text=True, check=True, + ) + out.write_text(r.stdout) + + +def ago(iso: str) -> str: + if not iso: + return "never" + try: + then = datetime.fromisoformat(iso) + except ValueError: + return iso + secs = (datetime.now(timezone.utc) - then).total_seconds() + for div, unit in ((86400, "d"), (3600, "h"), (60, "m")): + if secs >= div: + return f"{int(secs // div)}{unit} ago" + return "just now" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("status_file", type=Path) + ap.add_argument("outdir", type=Path) + args = ap.parse_args() + + status = json.loads(args.status_file.read_text()) + cells = status["cells"] + + # Preserve matrix order rather than sorting: the page should read like + # the README grid. + with (ROOT / ".github/matrix.yaml").open() as fh: + import yaml + m = yaml.safe_load(fh) + backends = [(b["backend"] if b["version"] == "n/a" else f"{b['backend']}-{b['version']}", + b["label"]) for b in m["backends"]] + targets = [(t["name"], t["label"]) for t in m["targets"]] + + npass = sum(1 for c in cells.values() if c.get("conclusion") == "success") + total = len(cells) + overall = "success" if npass == total else "failure" + render_badge(overall, f"eval-under: {npass}/{total} cells passing", + args.outdir / "badges" / "overall.svg") + + rows = [] + for bslug, blabel in backends: + tds = [f"{html.escape(blabel)}"] + for tname, tlabel in targets: + slug = f"{bslug}-{tname}" + c = cells.get(slug, {"conclusion": "unknown"}) + concl = c.get("conclusion", "unknown") + label = c.get("label", slug) + render_badge(concl, label, args.outdir / "badges" / f"{slug}.svg") + + text, _ = STATE.get(concl, ("unknown", "#9f9f9f")) + img = (f'{html.escape(label)}: {text}') + url = c.get("job_url", "") + body = f'{img}' if url else img + + meta = "" + if c.get("run_number"): + attempt = c.get("run_attempt", 1) + run = f"#{c['run_number']}" + (f".{attempt}" if attempt > 1 else "") + meta = (f'{run} · {ago(c.get("updated", ""))}') + why = "" + if concl == "failure" and slug in KNOWN_RED: + why = f'expected: {html.escape(KNOWN_RED[slug])}' + tds.append(f'{body}{meta}{why}') + rows.append("" + "".join(tds) + "") + + head = "".join(f"{html.escape(l)}" for _, l in targets) + repo = "con/eval-under" + doc = f""" + + + +eval-under CI status + +
+

eval-under CI status

+

{npass}/{total} cells passing · +run #{status.get('run_number', '?')} · +updated {ago(status.get('updated', ''))}

+
+ +{head} + +{chr(10).join(rows)} + +
Backend
+
+
+Rows are backends (which filesystem), columns are targets (which suite). +Each badge links to that cell's job log from the run that produced its +current state. A red cell is not automatically a bug — see +GOTCHAS.md. +Generated by bin/ci/render-report.py from +status.json; +source at {repo}. +
+
+""" + args.outdir.mkdir(parents=True, exist_ok=True) + (args.outdir / "index.html").write_text(doc) + print(f"I: wrote {args.outdir}/index.html and {total + 1} badge(s) " + f"({npass}/{total} passing)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bin/ci/run-under.sh b/bin/ci/run-under.sh index bd321c5..8580891 100755 --- a/bin/ci/run-under.sh +++ b/bin/ci/run-under.sh @@ -4,35 +4,71 @@ # # Generated with Claude Code 2.1.233 / Claude Opus 4.7 # -# Invoke `bin/eval-under BACKEND [OPTS] --set-home` around a full -# `git annex test` run. Handles the per-backend option translation -# (--version for beegfs, --fs for loop, nothing for nfs) so the -# workflow YAML doesn't need a case. +# Invoke `bin/eval-under BACKEND [OPTS] --set-home` around one of the +# test targets in bin/ci/target-.sh. Handles the per-backend +# option translation (--version for beegfs, --fs/--size for loop, +# nothing for nfs) and the per-target timeout, so the workflow YAML +# doesn't need a case. # # usage: -# bin/ci/run-under.sh +# bin/ci/run-under.sh [target] +# +# backend = beegfs | nfs | loop +# version = for beegfs: point release (e.g. 7.4.6, 8.1.0) +# for loop: filesystem type (e.g. vfat, ext4) +# for nfs: literal "n/a" +# target = git-annex (default) | git | stress-ng | pjdfstest +# +# env overrides: +# EVAL_UNDER_TIMEOUT seconds for the wrapped suite +# EVAL_UNDER_LOOP_SIZE_MB loop backing image size +# EVAL_UNDER_SRC_DIR where install-target.sh built the suites # # Runs as the current user; expects to be launched under sudo when the # backend requires root (beegfs/loop mount, NFS server bring-up). set -euo pipefail +here="$(cd "$(dirname "$0")" && pwd)" +# matrix.sh is a sourced library, resolved at runtime relative to $here. +# shellcheck source=bin/ci/matrix.sh disable=SC1091 +. "$here/matrix.sh" + BACKEND="${1:?backend required}" VERSION="${2:?version required}" +TARGET="${3:-git-annex}" + +target_known "$TARGET" || { + echo "unknown target: $TARGET (expected: ${EVAL_UNDER_TARGETS[*]})" >&2 + exit 1 +} + +# The target scripts re-derive their own defaults from matrix.sh, but an +# override handed to us must survive into the wrapped child. +export EVAL_UNDER_SRC_DIR + +TIMEOUT="${EVAL_UNDER_TIMEOUT:-$(target_timeout "$TARGET")}" opts=() case "$BACKEND" in beegfs) opts=(--version "$VERSION") ;; - loop) opts=(--fs "$VERSION") ;; - nfs) opts=() ;; + loop) opts=(--fs "$VERSION" + --size "${EVAL_UNDER_LOOP_SIZE_MB:-$(target_loop_size_mb "$TARGET")}") ;; + nfs) opts=() + # See target_needs_root() in matrix.sh: root-requiring suites + # need an export that does not squash root, and need to keep + # their privileges rather than being dropped to the invoker. + target_needs_root "$TARGET" && opts=(--no-root-squash) ;; *) echo "unknown backend: $BACKEND" >&2; exit 1 ;; esac +runner="$here/target-$TARGET.sh" +[ -x "$runner" ] || { echo "no target runner at $runner" >&2; exit 1; } + +echo "I: $(target_label "$TARGET") under $BACKEND/$VERSION (timeout ${TIMEOUT}s)" + # Sudo is expected to be in place already (workflow uses `sudo -E`); the -# script itself just forwards. Timeout keeps runaway `git annex test` -# invocations from hitting the workflow-level timeout with no signal. -# The single-quoted bash -c body is deliberate: $HOME must expand in the -# eval-under-launched child shell (where HOME=/home), not here. -# shellcheck disable=SC2016 -exec ./bin/eval-under "$BACKEND" "${opts[@]}" --set-home -- \ - timeout 2400 bash -c 'cd "$HOME" && git annex version | head -1 && git annex test' +# script itself just forwards. The timeout keeps a runaway suite from +# hitting the workflow-level timeout with no signal of its own. +exec "$here/../eval-under" "$BACKEND" "${opts[@]}" --set-home -- \ + timeout "$TIMEOUT" "$runner" diff --git a/bin/ci/target-git-annex.sh b/bin/ci/target-git-annex.sh new file mode 100755 index 0000000..c981a5b --- /dev/null +++ b/bin/ci/target-git-annex.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2026 Yaroslav Halchenko +# SPDX-License-Identifier: MIT +# +# Generated with Claude Code +# +# eval-under *target*: the full `git annex test` suite. +# +# Runs INSIDE the eval-under wrapper, i.e. with TMPDIR / HOME already +# pointing at the filesystem under test. Do not invoke directly for CI +# purposes -- go through bin/ci/run-under.sh git-annex. +# +# usage: +# bin/ci/target-git-annex.sh +# +# env (set by eval-under, honoured here): +# HOME /home -- the suite runs here, so every repo it +# creates lives on the filesystem under test +# TMPDIR + +set -euo pipefail + +cd "$HOME" + +# `git annex test` builds real repos and needs a committer identity. The +# workflow configures one for the runner user, but eval-under --set-home +# repoints HOME at the mount, so the runner's ~/.gitconfig is out of +# scope. Seed one here rather than depend on the caller's HOME. +git config --global --get user.email >/dev/null 2>&1 \ + || git config --global user.email test@github.land +git config --global --get user.name >/dev/null 2>&1 \ + || git config --global user.name "GitHub Almighty" + +git annex version | head -1 + +exec git annex test diff --git a/bin/ci/target-git.sh b/bin/ci/target-git.sh new file mode 100755 index 0000000..cf59a0b --- /dev/null +++ b/bin/ci/target-git.sh @@ -0,0 +1,126 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2026 Yaroslav Halchenko +# SPDX-License-Identifier: MIT +# +# Generated with Claude Code +# +# eval-under *target*: a subset of git's own testsuite. +# +# Rationale: git-annex sits on top of git plumbing, so a filesystem that +# breaks git's index/refs/object handling breaks git-annex in ways that +# `git annex test` reports only indirectly. Git's suite is the canonical +# filesystem-picky testbed and is far more rigorously maintained than +# anything we could hand-write. +# +# The git build lives on the runner's disk (bin/ci/install-target.sh); +# only the trash directories -- where every test's actual file operation +# happens -- are placed on the filesystem under test, via the suite's +# own `--root` option. +# +# Runs INSIDE the eval-under wrapper. Go through +# bin/ci/run-under.sh git. +# +# usage: +# bin/ci/target-git.sh +# +# env: +# EVAL_UNDER_SRC_DIR where install-target.sh built git +# EVAL_UNDER_GIT_TESTS glob(s) selecting the subset, evaluated inside +# git's t/ directory. Default: 't0*.sh t1*.sh' +# -- 174 scripts / ~10k assertions covering +# plumbing (t0xxx: init, index, attributes, +# object store) and the porcelain layer built +# straight on it (t1xxx: refs, config, fsck, +# worktrees, sparse-checkout). That is exactly +# the layer git-annex stands on, and it runs in +# under two minutes on ext4. Space-separated +# globs, e.g. 't00*.sh t13*.sh', to narrow it. +# EVAL_UNDER_GIT_JOBS parallel test jobs (default: nproc) +# EVAL_UNDER_GIT_TEST_OPTS +# extra options appended to GIT_TEST_OPTS, e.g. +# '-x' for shell tracing in the per-script logs +# (verbose, and it roughly triples artifact size). +# TMPDIR -- trash directories go under here + +set -euo pipefail + +here="$(cd "$(dirname "$0")" && pwd)" +# matrix.sh is a sourced library, resolved at runtime relative to $here. +# shellcheck source=bin/ci/matrix.sh disable=SC1091 +. "$here/matrix.sh" + +SRC="$EVAL_UNDER_SRC_DIR/git" +TESTS_GLOB="${EVAL_UNDER_GIT_TESTS:-t0*.sh t1*.sh}" +JOBS="${EVAL_UNDER_GIT_JOBS:-$(nproc)}" # prove --jobs, not make -j + +[ -x "$SRC/git" ] || { + echo "ERROR: no git build at $SRC -- run bin/ci/install-target.sh git first" >&2 + exit 3 +} + +# Trash directories (one per test script) on the filesystem under test. +root="${TMPDIR:-/tmp}/git-testsuite" +mkdir -p "$root" + +# Expand the glob ourselves inside t/ and hand `make` an explicit list. +# Passing the raw glob as T= happens to work via GNU make's wildcard +# expansion of prerequisites, but an explicit list fails loudly (empty +# selection) instead of silently running the whole suite. +cd "$SRC/t" +shopt -s nullglob +# shellcheck disable=SC2206 # deliberate word-split + glob of the pattern +selected=( $TESTS_GLOB ) +shopt -u nullglob + +if [ "${#selected[@]}" -eq 0 ]; then + echo "ERROR: EVAL_UNDER_GIT_TESTS='$TESTS_GLOB' matched no tests in $SRC/t" >&2 + exit 3 +fi + +echo "I: git $("$SRC/git" --version | awk '{print $3}') @ ${EVAL_UNDER_GIT_REF}" +echo "I: ${#selected[@]} test scripts selected by '$TESTS_GLOB'" +echo "I: trash directories under $root" + +# Run under `prove` (TAP), not the default `make test` target. +# +# This matters for reporting, and the difference is not cosmetic: +# +# `make test` hangs the per-script recipes off a single +# `aggregate-results-and-cleanup` target. Under `make -j` without +# `-k`, the *first* failing script stops make from scheduling any +# more, so the run aborts partway through and `aggregate-results` +# -- the thing that prints the totals -- never runs at all. The job +# log then ends on whichever scripts happened to still be in flight, +# which is why a failing cell used to end on a screenful of passes +# and no summary anywhere. +# +# `prove` is a TAP harness: it runs every selected script regardless +# of failures and ends with a "Test Summary Report" naming each +# failing script, its failing assertion numbers, and the totals +# ("Files=N, Tests=N ... Result: FAIL"). Same shape as the pjdfstest +# target's output, which is also prove-driven. +# +# UNIT_TESTS= empties git's C unit-test list, which the prove target +# otherwise appends to $(T): those are built binaries testing in-process +# data structures, they never touch the mount, and they would need a +# separate build step here. +# +# --verbose-log tees each script's full output to +# test-results/