diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a856e0..2246002 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: name: Lint (yamllint) runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # zizmor: artipacked # pipx is preinstalled on ubuntu-24.04 (same path as diff-cover/zizmor); pin the version so the @@ -49,7 +49,7 @@ jobs: name: Lint (markdownlint) runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # zizmor: artipacked # node is preinstalled on ubuntu-24.04; `make lint-md` runs the version-pinned markdownlint-cli2 @@ -66,7 +66,7 @@ jobs: SHFMT_VERSION: "3.13.1" SHFMT_SHA256: "fb096c5d1ac6beabbdbaa2874d025badb03ee07929f0c9ff67563ce8c75398b1" steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # no pushes from CI; don't leave the token in .git/config (zizmor: artipacked) # Install PINNED, checksum-verified shellcheck + shfmt instead of the runner's preinstalled @@ -98,7 +98,7 @@ jobs: name: Test suite runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # zizmor: artipacked # Ubuntu is RigForge's supported target. The suite still exercises the macOS code path here via @@ -111,7 +111,7 @@ jobs: name: Test suite (macOS) runs-on: macos-14 # Apple silicon; pinned (not macos-latest) to avoid image drift steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # zizmor: artipacked # The Linux job only SIMULATES macOS (STUB_UNAME_S=Darwin + stubbed sed/launchctl). Here the suite @@ -137,11 +137,14 @@ jobs: name: End-to-end (Docker) runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # zizmor: artipacked # Runs the real script end-to-end inside a disposable Ubuntu container (RigForge's documented # Linux target), exercising the genuine Linux deploy path and /etc idempotency with real tools. + # linux.sh runs TWO passes (a fresh container each): the standard deploy, then + # RIGFORGE_APPLIANCE=1 (#348) proving the appliance contracts against a real /etc — units in + # /run/systemd/system, no package installs, no fstab/limits/logrotate writes, --runtime enables. - name: Run end-to-end suite run: make test-e2e @@ -156,7 +159,7 @@ jobs: # so fall back to the pushed branch itself — that's an empty diff that trivially passes. BASE_REF: ${{ github.base_ref || github.ref_name }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 # diff-cover needs history to diff the PR against its base branch persist-credentials: false # the fetch below is read-only on a public repo (zizmor: artipacked) diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index 235222d..9d0be34 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -24,7 +24,7 @@ jobs: # lychee uses GITHUB_TOKEN to make authenticated GitHub requests and dodge the anon rate limit. GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # zizmor: artipacked - name: Install pinned lychee diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6bfac7b..a76a847 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG: ${{ github.ref_name }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Releasing goes through `gh` with GH_TOKEN below, not `git push`, so the checkout never # needs the token left in .git/config (zizmor: artipacked). diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index b2d27f1..d942b03 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -36,7 +36,7 @@ jobs: GITLEAKS_VERSION: "8.30.1" GITLEAKS_SHA256: "551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb" steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 # scan EVERY commit, not just the tip — a secret is still a leak once pushed persist-credentials: false # zizmor: artipacked @@ -60,7 +60,7 @@ jobs: env: ZIZMOR_VERSION: "1.25.2" steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # zizmor: artipacked # pipx is preinstalled on ubuntu-24.04; same install path as diff-cover in ci.yml's coverage job. diff --git a/.github/workflows/tool-bump.yml b/.github/workflows/tool-bump.yml index f8849f5..60cb09d 100644 --- a/.github/workflows/tool-bump.yml +++ b/.github/workflows/tool-bump.yml @@ -24,7 +24,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # zizmor: artipacked; the push authenticates explicitly below - name: Check every pinned tool, open one PR per stale pin diff --git a/.github/workflows/xmrig-bump.yml b/.github/workflows/xmrig-bump.yml index aa9426f..371b7bf 100644 --- a/.github/workflows/xmrig-bump.yml +++ b/.github/workflows/xmrig-bump.yml @@ -23,7 +23,7 @@ jobs: pull-requests: write actions: write steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # zizmor: artipacked — never persist the token in .git/config; the push step below # authenticates explicitly with the job token instead. diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a3f385..707e43c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,125 @@ All notable changes to RigForge are documented here. The format is based on ## [Unreleased] +## [1.15.0] - 2026-08-15 + +### Added + +- **The enriched feed mirrors the last control outcome (#346).** Pithead's host runner caps its + synchronous `/status` poll after a worker-apply, so a slow auto-rollback outruns it and the change + history row froze at `accepted`. The sister API's `rigforge` block now carries `control` — + `{change_id, status, reason}` read from the control status file — so the next routine poll of the + open read feed catches the terminal outcome without a new authenticated dial to the control port. + A rig that never took a control change, or an unreadable/malformed status file, serves + `control: null`; the mirror never breaks the feed. + +- **`doctor` and `apply` now notice when the rig isn't mining at all (#343).** With `pools[0].url` + pointed at an unresolvable host, XMRig loops on DNS errors forever — and `doctor` still said "all + critical checks passed" while `apply` said "Applied config and restarted". Both now ask the miner + itself: its local API's `connection.uptime` is positive exactly while a stratum connection is + live. `doctor` gains a pool-connection check — connected reports the pool, connection age and + accepted shares; running-but-disconnected counts as an issue (the rig is not mining); a silent + API is advisory only; and with the service stopped, one guarded TCP dial of `pools[0]` stands in. + `apply` polls briefly after the restart and warns when no live connection appears — warn, never + refuse: the pool may be legitimately down at apply time, so the exit code stays 0. + +### Fixed + +- **An interrupted live sweep no longer persists the mid-sweep candidate (#347).** `tune --live` + (and `tune --now --long`) applies every candidate straight into `tune-overrides.json` and restarts + the miner on it, but the cleanup trap only restored the temp dir and the service — a Ctrl+C, a + dropped SSH session, or an error mid-sweep left an arbitrary half-measured candidate saved and + running, silently costing hashrate until the next re-tune. The trap now restores the pre-sweep + overrides and restarts the miner on them, through the same restore step the `--confirm` revert leg + uses. `autotune` (and `tune --now`) had the same gap with its per-trial prefetch merges; its trap + restores the pre-sweep mode via the same merge the sweep uses, so knobs pinned by an offline + `tune` survive the abort too. + +- **`status` no longer aborts on a healthy rig (#341).** XMRig's `/2/summary` reports + `"hugepages"` as an array (`[pages, total]`); the status renderer fed it to jq's `@tsv`, + which rejects arrays — so the operator's first command printed an abort line instead of + the stats block, on every rig running v1.14.0. The renderer now joins the pair as + `HugePages: N/N`, scalars pass through, and the suite's shared fixture carries the real + array shape so the class can't return. + +## [1.14.0] - 2026-08-02 + +### Added + +- **Missing AES-NI / AVX2 is surfaced instead of mining silently slow (#338).** RandomX without + AES-NI falls back to XMRig's soft-AES path, roughly 4x slower, and nothing anywhere said why — the + last undelivered acceptance criterion from #1. `setup`/`apply` now warn at configure time when the + CPU flags lack `aes` (and, advisory, `avx2` — that one only slows dataset init), and `doctor` + counts a missing AES-NI as an issue. Judged only when an x86-style `flags` line exists in + `/proc/cpuinfo`; macOS, ARM and stubbed sandboxes read as unknown, and unknown never manufactures + an issue (the #333 lockdown stance). Never aborts: a knowingly-old rig is a valid choice. + +### Fixed + +- **Debian: dependency install no longer fails on `linux-tools-common` (#327).** The apt dependency + list named `linux-tools-common` unconditionally, but that package is Ubuntu-only — Debian ships + cpupower as `linux-cpupower` — and apt's install transaction is all-or-nothing, so the one unknown + name failed the ENTIRE install: git, build-essential and cmake never arrived either. `setup` now + probes with `apt-cache show` (the same guard the kernel-versioned `linux-tools-$(uname -r)` + package already had) and adds whichever name the distro actually ships; when neither exists it + warns and moves on — cpupower is a tuning aid, never worth losing the compiler toolchain over. + +## [1.13.1] - 2026-08-01 + +### Fixed + +- **`doctor` detects kernel lockdown directly instead of inferring it from MSR failure (#333).** + When kernel lockdown is active (distro kernels — Ubuntu, RHEL, Debian — enable it automatically + under UEFI Secure Boot), every `/dev/cpu/*/msr` write is denied and the MSR prefetcher mod + silently can't apply, costing ~5–15% RandomX. `doctor` only reacted after something downstream had + already failed, and its remedy for the most common cause was wrong: `msr.allow_writes=on` is dead + code under lockdown, because the kernel checks `security_locked_down(LOCKDOWN_MSR)` and returns + before the `allow_writes` filter is ever consulted (`arch/x86/kernel/msr.c`). `doctor` now reads + the actual state from `/sys/kernel/security/lockdown` *before* the MSR probes — cause before + symptom — and flags active lockdown (`integrity` and `confidentiality` both block the write) as a + counted issue naming the level, the cost, and the real fix (disable Secure Boot, with the + board-specific menu path). Reading beats inferring from `mokutil`: Secure-Boot-implies-lockdown is + a distro patch, not upstream behaviour. The three messages that guessed between causes now name + the one that applies, the missing-`msr`-module hint no longer blames Secure Boot (the in-tree + signed module loads fine under it — that's a modprobe/packaging problem), and `secure_boot` joins + the guided `bios` checklist (#80), leading it, verified against the same securityfs probe so + `doctor` and `bios` can't disagree. Unreadable securityfs stays advisory — unknown, never a + manufactured issue. + +## [1.13.0] - 2026-08-01 + +The pithead#797 appliance release: an opt-in setup mode for the read-only appliance image — +baked deps, runtime units, image-owned kernel cmdline — plus a grow-only runtime HugePages +reservation so a co-resident stack's pool is never shrunk out from under it. + +### Added + +- **Appliance mode: `RIGFORGE_APPLIANCE=1` (pithead#797 R1).** One opt-in env flag for running + `setup` on the Pithead appliance image, whose root filesystem is read-only and whose `/etc` is a + volatile overlay — every write there vanishes at reboot, and the image's boot path re-runs setup + each boot instead of relying on persisted state. Under the flag, setup installs no packages (the + toolchain is baked at image build; if a required tool is absent, setup fails naming it), skips the + GRUB leg deliberately (the kernel cmdline, including any 1 GB-hugepage reservation, is + image-owned), renders its systemd units into `/run/systemd/system` and enables them with + `--runtime`, mounts `hugetlbfs` at runtime instead of appending to `fstab`, and skips the + `limits.conf` memlock append (the unit already sets `LimitMEMLOCK=infinity`). Runtime tuning is + unchanged: `modprobe msr`, the grow-only HugePages sysctl (#328), and the performance governor all + work on a read-only root — each decision proven on the pithead#797 R0 bench. `setup --dry-run` + previews the same decisions. Everything else (a normal Linux install, macOS) behaves exactly as + before. + +### Fixed + +- **Runtime HugePages reservation is grow-only (#328).** `tune_kernel`'s runtime sysctl wrote the + miner's computed requirement absolutely, which SHRINKS a pool another consumer already reserved + (a co-hosted pithead stack's p2pool/monerod share the same 2MB pool) down to its in-use floor — + zero free pages on both sides, measured live on the pithead#797 appliance bench. The write is now + availability-based: pages the miner can draw on = free + whatever a running miner already holds; + the pool grows by the shortfall only and never shrinks. Fresh single-purpose rigs see the same + reservation as before; re-runs that find enough available write nothing. This is the runtime half + of #305's co-resident keep-existing guard, and it applies with or without + `hugepages_reserve_extra_mb` set. The `setup --dry-run` plan previews the same decision. + ## [1.12.0] - 2026-07-19 The pithead#597 producer release: the control-upgrade `/status` contract a one-click worker diff --git a/README.md b/README.md index d34b766..77a27a6 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![CI](https://github.com/p2pool-starter-stack/rigforge/actions/workflows/ci.yml/badge.svg)](https://github.com/p2pool-starter-stack/rigforge/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](./LICENSE) -![Platform: Ubuntu 22.04+](https://img.shields.io/badge/Platform-Ubuntu%2022.04%2B-E95420?logo=ubuntu&logoColor=white) +![Platform: Ubuntu 22.04+ / Debian 12](https://img.shields.io/badge/Platform-Ubuntu%2022.04%2B%20%2F%20Debian%2012-E95420?logo=ubuntu&logoColor=white) [![Miner: XMRig](https://img.shields.io/badge/Miner-XMRig-F26822?logo=monero&logoColor=white)](https://github.com/xmrig/xmrig) [![Companion: Pithead](https://img.shields.io/badge/Companion-Pithead-F26822)](https://github.com/p2pool-starter-stack/pithead) @@ -51,8 +51,8 @@ you like to your stack's single endpoint. `autotune` in `config.json` re-tunes on a monthly schedule. - An opt-in `watchdog` restarts a wedged miner (alive but hashing 0 H/s), and with `max_temp_c` set it stops the miner above that temperature and resumes once it cools. -- `doctor` health-checks HugePages, the MSR mod, the governor, and the service; `status` shows live - hashrate, pool, uptime, and shares. +- `doctor` health-checks HugePages, the MSR mod, the governor, the service, and the pool + connection (the miner's own view of it); `status` shows live hashrate, pool, uptime, and shares. - `bios` is a guided, resumable walkthrough of the BIOS/UEFI changes RigForge can't make itself (XMP/EXPO, SMT, PBO/Eco Mode), and re-verifies what took after the reboot. - `backup` and `restore` snapshot your config and tuning; `support-bundle` collects a redacted @@ -147,7 +147,7 @@ sudo ./rigforge.sh apply # regenerate config + restart (no rebuild) git pull && sudo ./rigforge.sh upgrade # rebuild + restart if the XMRig pin moved sudo ./rigforge.sh tune --now # run a live tune now — keep the best prefetch mode -sudo ./rigforge.sh doctor # health check: HugePages, MSR, governor, service +sudo ./rigforge.sh doctor # health check: HugePages, MSR, governor, service, pool connection ./rigforge.sh logs # follow live logs, no root needed (also: status; sudo for start / stop / restart) sudo ./rigforge.sh # provision (or re-provision) the worker — idempotent ``` diff --git a/VERSION b/VERSION index 0eed1a2..141f2e8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.12.0 +1.15.0 diff --git a/docs/README.md b/docs/README.md index d5a2aa2..3ecd081 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,7 +16,7 @@ individual topics once you're up and running. | [Configuration](configuration.md) | Every `config.json` key and default, minimal vs. advanced setups, and how the XMRig config is generated. | | [Operations & Maintenance](operations.md) | The full command reference, service management, logs, upgrades, and troubleshooting. | | [How It Works](how-it-works.md) | What the script actually does: dependencies, compile-from-source, HugePages, MSR, NUMA, the governor, and the systemd service. | -| [Pithead Integration](pithead-integration.md) | The worker ↔ dashboard contract: discovery via `:3333`, the read-only HTTP API on `:8080`, and the token rules. | +| [Pithead Integration](pithead-integration.md) | The worker ↔ dashboard contract: discovery via `:3333`, the read-only API on `:8080` (plus the optional enriched sister API on `:8081`), and the token rules. | | [FAQ](faq.md) | Common questions, plus why RigForge vs. setting XMRig up by hand. | For how RigForge is versioned and released, see [`RELEASING.md`](../RELEASING.md) and @@ -27,7 +27,7 @@ For how RigForge is versioned and released, see [`RELEASING.md`](../RELEASING.md Significant, cross-cutting decisions are recorded as ADRs under [`adr/`](adr/): - [ADR 0001: Writable worker-config control path](adr/0001-writable-worker-config-control-path.md) — *Accepted.* How the stack applies config changes through RigForge without breaking the read-only sister API. -- [ADR 0002: Remote worker-upgrade control path](adr/0002-remote-worker-upgrade.md) — *Proposed.* Opt-in remote RigForge upgrade for the stack (#308), extending ADR 0001 from a tuning surface to a code-update one; hash-only trust, no signing. +- [ADR 0002: Remote worker-upgrade control path](adr/0002-remote-worker-upgrade.md) — *Accepted.* Opt-in remote RigForge upgrade for the stack (#308), extending ADR 0001 from a tuning surface to a code-update one; hash-only trust, no signing. ## Quick links diff --git a/docs/adr/0002-remote-worker-upgrade.md b/docs/adr/0002-remote-worker-upgrade.md index 044fd6a..d6e3ad2 100644 --- a/docs/adr/0002-remote-worker-upgrade.md +++ b/docs/adr/0002-remote-worker-upgrade.md @@ -53,7 +53,7 @@ GitHub is therefore accepted as the trust root. The residual risk is explicit: a ### D6. Fail-closed, throttled, rollback-guarded, with a terminal status -`control-upgrade` is fail-closed and every step is rollback-guarded, reusing the upgrade flow's existing rebuild-only-if-the-pin-changed logic. A **pre-dial throttle stamp** on the rig bounds how often it will reach out to GitHub, so a looping or hostile consumer cannot turn the fleet into a beacon or a request amplifier. Outcomes use the same `/status` surface as control-apply, extended with a **`failed`** terminal (with reason) alongside `applied` / `rolled_back`, so a pre-apply refusal (non-latest, throttled, fetch/build error) surfaces a clear terminal result to the consumer's poll rather than hanging in a non-terminal state. +`control-upgrade` is fail-closed and every step is rollback-guarded, reusing the upgrade flow's existing rebuild-only-if-the-pin-changed logic. A **pre-dial throttle stamp** on the rig bounds how often it will reach out to GitHub, so a looping or hostile consumer cannot turn the fleet into a beacon or a request amplifier. Outcomes use the same `/status` surface as control-apply, extended with a **`failed`** terminal (with reason) alongside `applied` / `rolled_back`, so a pre-apply refusal (target outside the allowed bound, throttled, fetch/build error) surfaces a clear terminal result to the consumer's poll rather than hanging in a non-terminal state. *Amended (#320, consumer feedback from pithead #597):* the status vocabulary grew three additive members so a poller never has to string-match `reason`. A non-terminal **`started`** is written the moment the oneshot claims the intent (D8 move done), so "mid-run" and "oneshot died mid-run" are distinguishable from "queued"; **`noop`** replaces `failed` for the already-on-target refusal (idempotent, not an error); **`throttled`** replaces `failed` for the D6 throttle refusal (retry-later, not an error). `applied`'s `reason` echoes the landed version. All other refusals stay `failed`. @@ -92,7 +92,7 @@ Low-cost integrity controls that need no standing signing key, shoring up the Gi - **Widen the ADR 0001 D3 allowlist to include a `version`/upgrade key.** Rejected: a code update is not an operationally-mutable config knob; folding it into the tuning path would silently grant every existing remote-tuning user a remote-RCE surface. A second opt-in (D1) is the point. - **Sign releases (cosign, as pithead did) and verify before running.** Considered and rejected for this project (D5): signing only helps against a distribution-point compromise and only with an out-of-band pinned/offline key; the key-custody burden isn't warranted, and GitHub is accepted as the trust root. -- **Let the consumer choose the target version.** Rejected: the receiver/intent must not be able to pin a rig to an arbitrary tag (downgrade/rollback-to-vulnerable). The rig re-derives latest and refuses anything else (D4). +- **Let the consumer choose the target version unbounded.** Rejected in that form: the receiver/intent must not be able to pin a rig to an arbitrary tag (downgrade/rollback-to-vulnerable). D4 does accept the dashboard-supplied target — the rig makes no version check of its own — but bounds it with the monotonic anti-rollback and reachable-from-main guards. - **Receiver fetches and runs directly (scoped-privilege network process).** Rejected for the same reason as ADR 0001 D2: the network-facing process stays unprivileged and stages only. - **No network upgrade path; keep SSH + `rigforge.sh upgrade`.** Lowest surface and still fully supported, but does not give pithead #597 a programmatic per-worker upgrade producer, which is the point of #308. diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 130043e..8d380ab 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -209,6 +209,32 @@ XMRig accordingly (and there's no systemd service, so you run the miner yourself --- +## Appliance mode (`RIGFORGE_APPLIANCE=1`) + +The Pithead appliance image runs from a read-only root with a volatile `/etc` overlay: anything +setup writes there disappears at reboot, and the image's boot path re-runs setup every boot +instead. `RIGFORGE_APPLIANCE=1` adapts setup to that contract (pithead#797): + +- No package install. The toolchain is baked into the image at build; setup verifies the tools it + needs and fails naming any that are missing, rather than attempting an install that can't work + on a read-only root. A prebuilt worker needs no compiler at all — the every-boot re-run keeps + mining even without one. +- GRUB skipped deliberately. The kernel cmdline — including any 1 GB-hugepage reservation — is + owned by the image. Without that reservation XMRig falls back to 2 MB pages at 100%: working, + with a small known cost. +- Units in `/run`. The systemd units render into `/run/systemd/system` and are enabled with + `systemctl enable --runtime`, so the enablement lives and dies with the boot, matching the + re-run-every-boot model. +- Runtime mounts only. `hugetlbfs` is mounted directly instead of via `fstab`, and the + `limits.conf` memlock append is skipped — the unit already sets `LimitMEMLOCK=infinity`, and + interactive runs are not an appliance concern. + +Runtime tuning is untouched: `modprobe msr`, the grow-only HugePages sysctl, and the performance +governor all work on a read-only root. `setup --dry-run` previews the appliance decisions with the +same logic. On a normal install, leave the flag unset — nothing changes. + +--- + ## Safety & idempotency RigForge is built to be re-run: diff --git a/docs/operations.md b/docs/operations.md index 85deb8a..ec3b956 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -15,7 +15,7 @@ reference](#commands) is below. | Change a setting (pool, rig name, TLS, failover) | edit `config.json`, then `sudo ./rigforge.sh apply` | Regenerates the live config and restarts. No rebuild. | | Redeploy after a `git pull` | `git pull && sudo ./rigforge.sh upgrade` | Rebuilds + restarts (and re-tunes) if the XMRig pin moved; otherwise a no-op. See [the note below](#upgrading-xmrig-redeploy-after-a-git-pull). | | Run a live tune now | `sudo ./rigforge.sh tune --now` | One live pass against the running miner; keeps the best prefetch mode if it wins. Linux only. | -| Check the worker is healthy | `sudo ./rigforge.sh doctor` | HugePages, MSR, governor, service, with a fix hint for anything off. | +| Check the worker is healthy | `sudo ./rigforge.sh doctor` | HugePages, MSR, governor, service, pool connection, with a fix hint for anything off. | | Watch it mining | `./rigforge.sh logs` | Live logs, no root needed; `Ctrl-C` stops following (the miner keeps running). | | Stop / start / restart | `sudo ./rigforge.sh stop` · `start` · `restart` | Control the miner service. | | Quick speed check | `sudo ./rigforge.sh bench` | One-off offline benchmark; reports H/s. | @@ -54,12 +54,12 @@ Not every verb needs root — the design, in four lines: |---|---| | `setup` *(default)* | Provision the worker: dependencies, build, hardware + kernel tuning, and the service. Idempotent and safe to re-run; skips the recompile when the pinned XMRig is already built. | | `upgrade` | Rebuild and restart only if the pinned XMRig version/commit changed. A no-op when you're already on the pinned build. If periodic autotune is enabled, it also re-tunes the new build (the fastest knobs can shift between versions). `--check` just reports whether a newer RigForge release exists (on-demand GitHub query, always exits 0). | -| `apply` | Re-read `config.json`, regenerate the live XMRig config, and restart, without recompiling. The fast path after editing `config.json`. On Linux it also reconciles the periodic-autotune timer with config (so changing the `autotune` target takes effect) and reports it (efficiency / performance / disabled). | +| `apply` | Re-read `config.json`, regenerate the live XMRig config, and restart, without recompiling. The fast path after editing `config.json`. On Linux it also reconciles the periodic-autotune timer with config (so changing the `autotune` target takes effect) and reports it (efficiency / performance / disabled), then asks the running miner whether a live pool connection came up — and warns when none did (a bad `pools[0].url` otherwise mines nothing while looking applied, #343). Warn only, never a failure: the pool may be legitimately down at apply time. | | `uninstall` | Remove the service and revert all system changes (fstab, limits, modules, GRUB) and the worker build/logs. Leaves `config.json`. Prompts first; add `--yes` to skip. | -| `doctor` | Read-only health check (run with `sudo` for the deepest checks). Critical findings (counted as issues): the service is active, HugePages are reserved, the `msr` module is loaded, and the MSR mod actually applied, confirmed from XMRig's log and, as root, an `rdmsr` register read-back (see [MSR mod verification](#msr-mod-verification)). Advisory findings (hints, not failures): CPU governor, 1 GB HugePages, HugePages 100%-backed (from the XMRig log), hashrate-capping hardware RigForge can't fix but you can (single-channel or slow RAM via `dmidecode`, and a power/boost-capped CPU clock), and BIOS/firmware recommendations (board/BIOS context, plus enable XMP/EXPO/DOCP or SMT when they're off; manual BIOS changes RigForge can't make from the OS). Prints an actionable hint for anything off. Also binary tamper evidence (#141): the on-disk `xmrig` is compared against the SHA-256 recorded at compile time — a deliberate rebuild refreshes the record, anything else warns and counts as an issue. When the [writable control path](#writable-control-path-opt-in) is enabled, it's also a counted issue: `doctor` checks `rigforge-control` is active and its `/status` endpoint answers (#278), staying silent when control is disabled. Exits non-zero when critical issues are found (cron-friendly, matching Pithead's `status`). | +| `doctor` | Read-only health check (run with `sudo` for the deepest checks). Critical findings (counted as issues): the service is active, the miner has a live pool connection (read from the miner's own local API — a running miner with no connection is not mining, #343; with the service stopped, one TCP dial of `pools[0]` stands in), HugePages are reserved, kernel lockdown isn't blocking MSR writes (read from `/sys/kernel/security/lockdown` — active lockdown means the MSR mod cannot apply, and `msr.allow_writes=on` can't override it), the `msr` module is loaded, and the MSR mod actually applied, confirmed from XMRig's log and, as root, an `rdmsr` register read-back (see [MSR mod verification](#msr-mod-verification)). Advisory findings (hints, not failures): CPU governor, 1 GB HugePages, HugePages 100%-backed (from the XMRig log), hashrate-capping hardware RigForge can't fix but you can (single-channel or slow RAM via `dmidecode`, and a power/boost-capped CPU clock), and BIOS/firmware recommendations (board/BIOS context, plus enable XMP/EXPO/DOCP or SMT when they're off; manual BIOS changes RigForge can't make from the OS). Prints an actionable hint for anything off. Also binary tamper evidence (#141): the on-disk `xmrig` is compared against the SHA-256 recorded at compile time — a deliberate rebuild refreshes the record, anything else warns and counts as an issue. When the [writable control path](#writable-control-path-opt-in) is enabled, it's also a counted issue: `doctor` checks `rigforge-control` is active and its `/status` endpoint answers (#278), staying silent when control is disabled. Exits non-zero when critical issues are found (cron-friendly, matching Pithead's `status`). | | `bench` | Run a one-off `xmrig --bench` and report the hashrate (a quick perf/health check; set `BENCH=10M` for a longer run). | | `tune` | The single command for tuning. A bare `tune` measures the fastest CPU-specific knobs (prefetch, `cpu.yield`, thread count) offline and keeps them, an optional, one-time step. Live variants: `--now` / `--short` (a quick prefetch re-tune against the running miner, the *run a live tune now* path), `--now --long` (a full live search of every knob, = `--live`), `--confirm` (A/B-check the winner live). Plus `--efficiency` / `--perf`, `--history`, `--clear`. See [Tuning](#tuning). | -| `bios` | Guided, resumable walk-through of the BIOS/UEFI changes for your hardware — the settings `tune` can't reach from the OS (memory profile XMP/EXPO/DOCP, SMT, PBO/Eco-Mode, NUMA-per-socket NPS on EPYC; `--efficiency` picks the low-power set). Detects the current firmware state via the same probes `doctor` uses, hands you a board-specific checklist one item at a time, saves the pending items, and on the next run re-verifies which changes actually took. RigForge never writes BIOS itself; plan for console access (keyboard/KVM) for the reboot-into-BIOS step. Linux-only. See [Guided BIOS tuning](#guided-bios-tuning). | +| `bios` | Guided, resumable walk-through of the BIOS/UEFI changes for your hardware — the settings `tune` can't reach from the OS (Secure Boot when kernel lockdown is blocking the MSR mod, memory profile XMP/EXPO/DOCP, SMT, PBO/Eco-Mode, NUMA-per-socket NPS on EPYC; `--efficiency` picks the low-power set). Detects the current firmware state via the same probes `doctor` uses, hands you a board-specific checklist one item at a time, saves the pending items, and on the next run re-verifies which changes actually took. RigForge never writes BIOS itself; plan for console access (keyboard/KVM) for the reboot-into-BIOS step. Linux-only. See [Guided BIOS tuning](#guided-bios-tuning). | | `autotune` | The scheduled live tuner. You normally don't type it; `tune --now` is the friendlier spelling for an on-demand run, and the periodic schedule is what this verb is really for: set `"autotune": "performance"` (raw H/s) or `"autotune": "efficiency"` (hashrate-per-watt) in `config.json` and setup installs a systemd timer (also re-tuned on `upgrade`). Conservative: it keeps a change only if it beats the baseline by a margin, else rolls back. Linux-only. See [Live auto-tuning](#live-auto-tuning-opt-in). | | `watchdog` | One health check per run — the scheduled recovery verb, like `autotune`. You normally don't type it; set `"watchdog": "enabled"` in `config.json` and setup installs a timer that runs it every `watchdog_interval_min` minutes: two consecutive checks seeing 0 H/s or a dead API restart the wedged miner, and with `max_temp_c` set it stops the miner above that temperature (starting it again 5 °C below). Linux-only. See [Watchdog](#watchdog-opt-in). | | `backup` | Snapshot `config.json` + the tuning files into a timestamped `tar.gz` under `./backups`. See [Backup & restore](#backup--restore). | @@ -136,13 +136,17 @@ power/efficiency and reservation-aware details are all in ### Guided BIOS tuning `sudo ./rigforge.sh bios` walks the detect → guide → reboot → re-verify loop for the firmware -settings with the biggest RandomX impact: the memory profile (XMP/EXPO/DOCP), SMT, the CPU -power/boost posture (`--efficiency` swaps the boost item for Eco-Mode + Curve Optimizer), and — on -EPYC — NUMA nodes per socket (NPS4, so RandomX gets quadrant-local memory). It reads +settings with the biggest RandomX impact: Secure Boot when kernel lockdown is blocking the MSR mod +(it leads the checklist — the cheapest change, and it gates the ~5–15% MSR mod outright; whether to +make it is the operator's call, `bios` just makes the cost visible), the memory profile +(XMP/EXPO/DOCP), SMT, the CPU power/boost posture (`--efficiency` swaps the boost item for +Eco-Mode + Curve Optimizer), and — on EPYC — NUMA nodes per socket (NPS4, so RandomX gets +quadrant-local memory). It reads the same probes `doctor` reports on, so the two never disagree; pending items are saved to `rigforge-bios.json` (included in `backup`/`restore`) and the next `bios` run re-checks exactly those items against fresh probes — an item only counts as applied when its OS-visible fingerprint -flips (memory running at rated speed, SMT on, loaded clock above the boost threshold). The CPU +flips (lockdown reading `none`, memory running at rated speed, SMT on, loaded clock above the boost +threshold). The CPU boost item needs the miner running to measure; with it stopped, `bios` says so and keeps the item pending rather than guessing. After everything took, re-run `tune --live` — the hardware envelope changed. @@ -156,6 +160,11 @@ pass that sweeps every knob live (threads, yield, 1G-pages, not just prefetch), a BIOS, RAM, or cooling change. `tune --now` is the friendly name for the `autotune` engine; the standalone `autotune` verb still works and is what the scheduled timer below runs. +Live sweeps apply each candidate to the running miner as they measure it. If a sweep is interrupted +(Ctrl+C, a dropped SSH session) or errors out mid-run, RigForge restores the tuning that was in place +before the sweep and restarts the miner on it — a half-measured candidate is never left running or +saved to `tune-overrides.json`. + For a hands-off schedule, set `autotune` in `config.json` to a target and re-run `setup`. RigForge installs a systemd timer that periodically optimizes the prefetch mode against your live miner: @@ -196,9 +205,13 @@ one-time sweep of every knob, run the offline [`tune`](#tuning) above. Linux onl The MSR "RandomX boost" (writing the CPU's prefetcher MSRs) is one of the biggest levers, worth ~10–15%, so `doctor` verifies it actually took effect, not just that the `msr` module loaded: +- Kernel lockdown first: before any MSR probe, `doctor` reads `/sys/kernel/security/lockdown` — active + lockdown (`integrity` or `confidentiality`, which distro kernels enable automatically under UEFI + Secure Boot) denies every `/dev/cpu/*/msr` write before the `msr.allow_writes` filter is even + consulted, so it is flagged as the cause rather than letting the probes below fail mysteriously. - From XMRig's log (always): the `msr register values for "" preset have been set successfully` - line confirms XMRig wrote the per-family preset (e.g. `ryzen_19h_zen4`). A `FAILED` line is flagged, - usually Secure Boot or a missing `msr.allow_writes=on`. + line confirms XMRig wrote the per-family preset (e.g. `ryzen_19h_zen4`). A `FAILED` line is flagged — + attributed to lockdown when lockdown is active, otherwise to `msr.allow_writes=on` or a hypervisor. - Register read-back via `rdmsr` (run `doctor` as root, with `msr-tools` installed; `setup` installs it): `doctor` reads the prefetcher registers back and checks they hold the preset's values, catching a write a hypervisor or kernel lockdown silently dropped even though XMRig reported success. Run without root, @@ -250,7 +263,7 @@ Each timer tick runs one check (`journalctl -u rigforge-watchdog.service` shows design (a loaded EPYC sits near 90 °C Tctl in perfectly normal operation), so a damage-avoidance cutoff belongs well above the everyday reading, not near it. Editing `max_temp_c` takes effect on the next tick — the watchdog re-reads `config.json` every run, no `apply` needed. -- A miner that's simply **not running** is left alone — dead-process recovery is systemd's job. +- A miner that's **not running** is left alone — dead-process recovery is systemd's job. Disable it with `"watchdog": "disabled"` (or removing the key) + `apply`; the units are removed cleanly. `uninstall` removes them too. @@ -501,9 +514,9 @@ If you see MSR errors, see Troubleshooting below. | Symptom | Likely cause & fix | |---|---| | Setup fails during the build | The script names the step that failed and tails the build log. Read the full error in `/build.log` (e.g. `data/worker/build.log`). Common causes: a build dependency you declined to install (re-run and accept), or too little RAM during compilation (the build already caps parallelism by RAM; add swap on very low-memory hosts). Re-run `sudo ./rigforge.sh` once resolved; it resumes without redoing finished work. | -| MSR errors in the log | Secure Boot is blocking the `msr` kernel module. Disable Secure Boot in your BIOS/UEFI, then reboot. | -| `doctor`: "MSR registers don't match the preset" | XMRig's log says the write succeeded but the read-back disagrees: the kernel or hypervisor silently dropped it. Common on VMs/cloud instances and under kernel lockdown. Run RigForge on bare metal, and ensure `msr.allow_writes=on` (RigForge sets this) and that lockdown isn't enforced. | -| `doctor`: "couldn't read the MSRs via rdmsr" | The `msr` module isn't loaded (or `doctor` wasn't run as root). Run `sudo ./rigforge.sh doctor`; if it persists, `sudo modprobe msr` (Secure Boot can block it). This is advisory; XMRig's log already confirms the write. | +| MSR errors in the log | Kernel lockdown is denying the MSR writes (distro kernels enable lockdown automatically under UEFI Secure Boot). `msr.allow_writes=on` cannot override it — the kernel checks lockdown before that filter. `doctor` reads `/sys/kernel/security/lockdown` and says which level is active; the fix is disabling Secure Boot in BIOS/UEFI (`bios` gives the board-specific path), then rebooting. | +| `doctor`: "MSR registers don't match the preset" | XMRig's log says the write succeeded but the read-back disagrees: something silently dropped it. When kernel lockdown is active, `doctor` names it as the cause (see the lockdown item it prints first); otherwise a hypervisor is the usual culprit — common on VMs/cloud instances, run RigForge on bare metal. `msr.allow_writes=on` (RigForge sets this) only matters when lockdown is *not* enforced. | +| `doctor`: "couldn't read the MSRs via rdmsr" | The `msr` module isn't loaded (or `doctor` wasn't run as root). Run `sudo ./rigforge.sh doctor`; if it persists, `sudo modprobe msr` and check `/etc/modules-load.d/msr.conf` — a missing module is a modprobe/packaging problem, not Secure Boot (the in-tree signed module loads fine under it). This is advisory; XMRig's log already confirms the write. | | `HugePages_Total` is 0 | The kernel tuning needs a reboot to take effect (GRUB change). Reboot, then re-check `grep Huge /proc/meminfo`. | | HugePages still 0 after reboot | Not enough contiguous memory was reservable, or another tool changed GRUB. Re-run `sudo ./rigforge.sh`; RigForge merges its kernel parameters into `GRUB_CMDLINE_LINUX_DEFAULT` rather than overwriting, so other params are preserved. | | Low hashrate / few threads | RandomX is L3-bound (~2 MB per thread). A CPU with little L3 runs fewer effective threads; this is expected. See [Hardware › L3 cache](hardware.md#a-note-on-l3-cache). | @@ -526,8 +539,9 @@ second read-only HTTP endpoint (default `:8081`, keys `api_port`/`api_bind`) wit `:8080` XMRig API has **plus** the data only RigForge knows: applied tune knobs and the last tune run (`/tune`), hashrate-per-watt from RAPL, the doctor probes — HugePages, MSR state, governor, RAM channels/speeds, memory-profile and SMT state, throttling — the watchdog's armed/thermal state, -the effective **writable** config (`config`, pool secrets masked; #253), and a config revision + -last-change provenance (`config_meta`; #254) — as JSON (`/health`, or nested under +the effective **writable** config (`config`, pool secrets masked; #253), a config revision + +last-change provenance (`config_meta`; #254), and the last control-path outcome (`control`, +`{change_id, status, reason}` or `null`; #346) — as JSON (`/health`, or nested under `rigforge` in `/1/summary` and `/2/summary`). It follows XMRig's own architecture: one tiny persistent server (python3 stdlib, ~10 MB idle) ships pre-computed bytes, so a request costs microseconds and cannot touch mining performance; a systemd timer recomputes the state every 15 diff --git a/docs/pithead-integration.md b/docs/pithead-integration.md index 68e22dd..70c028b 100644 --- a/docs/pithead-integration.md +++ b/docs/pithead-integration.md @@ -82,7 +82,10 @@ pithead#235): config — exactly the control-path allowlist, pool secrets masked; see the prefill note in §3), and `config_meta` (`{revision, changed_at, source, last_change_id}` — `revision` is a content hash of the writable config that changes iff that config changes, so a poller can detect a change made directly - on the rig; `source` is `control`/`local`/`restore`; see §3). + on the rig; `source` is `control`/`local`/`restore`; see §3), and `control` (the last control-path + outcome, `{change_id, status, reason}` mirrored from the control status file, so a poller that + missed a slow rollback catches the terminal outcome here without dialing the control port; `null` + when the rig has never taken a control change or the status file is unreadable). - `GET /health` and `GET /tune` — the `rigforge.health` / `rigforge.tune` objects bare. - When XMRig's own API is unreachable the response is still `200` with `"rigforge": {..., "xmrig_api": "unreachable"}` — a down miner is exactly when the health data @@ -100,36 +103,46 @@ pre-computed state (a request costs microseconds — polling cannot shave hashra ### Stratum over TLS (optional) -Needs stack-side support that hasn't shipped yet (Pithead is tracking it as -[pithead#261](https://github.com/p2pool-starter-stack/pithead/issues/261)) — but everything below -already works today against **any** TLS stratum endpoint, e.g. a public pool's TLS port. +The stack can serve TLS on the **same** `:3333` +([`p2pool.stratum_tls`](https://github.com/p2pool-starter-stack/pithead/blob/main/docs/workers.md#stratum-over-tls), +default off): its proxy detects TLS vs plain per connection, so nothing re-points and a mixed +fleet migrates one rig at a time — rigs still on cleartext keep mining throughout. On the first +`pithead apply` with TLS on, the stack generates a self-signed certificate and prints its SHA-256 +fingerprint (64 lowercase hex chars); `pithead status` repeats it. That fingerprint is what the +rig pins: ```jsonc -// config.json — TLS on, server cert pinned by its SHA-256 fingerprint +// config.json — TLS on, the stack's cert pinned by its SHA-256 fingerprint { "pools": [ - { "url": "your-stack:3334", "tls": true, "tls-fingerprint": "<64 hex chars>" } + { "url": "your-stack:3333", "tls": true, "tls-fingerprint": "" } ] } ``` -Then `sudo ./rigforge.sh apply`. (The `:3334` port is only an example — use whatever port the stack -documents once pithead#261 fixes its port model.) +Then `sudo ./rigforge.sh apply`. The first-run prompt doesn't ask about TLS — these two fields +are edited into `config.json` by hand. The same fields work against **any** TLS stratum endpoint, +e.g. a public pool's TLS port. **The trust model, plainly:** XMRig does no CA validation on stratum TLS. With `"tls": true` and no fingerprint, the link is encrypted but not authenticated — fine against passive snooping, no defense -against an active man-in-the-middle. The fingerprint pin IS the server authentication. Get it with: +against an active man-in-the-middle. The fingerprint pin IS the server authentication: a pinned rig +refuses anything that doesn't hold the stack's exact certificate. `pithead status` is the canonical +source for the pin; to read it off the wire instead (XMRig compares case-insensitively, so the +uppercase openssl output works as-is): ```bash -echo | openssl s_client -connect your-stack:3334 2>/dev/null \ +echo | openssl s_client -connect your-stack:3333 2>/dev/null \ | openssl x509 -noout -fingerprint -sha256 | cut -d= -f2 | tr -d ':' ``` - TLS is confidentiality; the stratum password (above) is access control. They're orthogonal — set both on an untrusted network. -- Rotation: when the stack rotates its certificate, update `tls-fingerprint` on each rig and run - `apply` (same runbook shape as the password). A stale pin shows up as - `Failed to verify server certificate fingerprint` in the XMRig log. +- Rotation: the operator deletes the two files in the stack's `proxy-tls` data directory and re-runs + `pithead apply` (new certificate, new fingerprint); then update `tls-fingerprint` on each TLS rig + and run `apply` (same runbook shape as the password). A stale pin shows up as + `Failed to verify server certificate fingerprint` in the XMRig log; cleartext rigs are unaffected + throughout. --- @@ -220,7 +233,7 @@ the token in flight, so isolate the mining LAN — see | Symptom | Fix | |---|---| | Rig won't mine / XMRig logs `Permission denied` at login | The stack has stratum authentication on (`p2pool.stratum_password`); set the pool `pass` to that secret. See [Stratum authentication](#stratum-authentication-optional). | -| XMRig logs `Failed to verify server certificate fingerprint` | The `tls-fingerprint` pin doesn't match the server's certificate (rotated cert or a typo). Re-run the openssl one-liner in [Stratum over TLS](#stratum-over-tls-optional) and `apply`. | +| XMRig logs `Failed to verify server certificate fingerprint` | The `tls-fingerprint` pin doesn't match the server's certificate (rotated cert or a typo). Re-pin from `pithead status` (or the openssl one-liner in [Stratum over TLS](#stratum-over-tls-optional)) and `apply`. | | Worker missing from the dashboard | The dashboard discovers rigs from their stratum `user` label; confirm the worker is actually connected to the pool and mining. | | Rig shows as connected but no stats | By default the API is open and the dashboard reads it with no token. If you set an `ACCESS_TOKEN` here, the dashboard must match it (`workers.api_auth: token` + `workers.api_token`, or `name` if the token is the rig name); otherwise clear `ACCESS_TOKEN` and re-run setup. | | Stats unreachable from the stack host | Confirm the worker's `:8080` is reachable from the stack host over the LAN (firewall, correct IP). RigForge binds `0.0.0.0` by default. | diff --git a/rigforge.sh b/rigforge.sh index 276077c..fd20087 100755 --- a/rigforge.sh +++ b/rigforge.sh @@ -90,6 +90,33 @@ XMRIG_COMMIT="${XMRIG_COMMIT:-b2ca72480c58d197e18c885d9fc1a0c8d517e60a}" # recompile and the service restart — making re-runs idempotent (#4). XMRIG_REBUILD=true +# Appliance mode (pithead#797 R1): opt-in via RIGFORGE_APPLIANCE=1 for running setup on the Pithead +# appliance image, whose root filesystem is read-only and whose /etc is a volatile overlay — every +# write there vanishes at reboot, and a pithead boot leg re-runs setup each boot instead. An env +# flag, not a config.json key, because the caller is the image's boot path (not the operator) and +# the mode is a preset bundle over the env-overridable system paths below — the same seam the R0 +# bench drove by hand with GRUB_DEFAULT=/nonexistent. Under the flag, setup: +# - never installs packages (the toolchain is baked at image build; apt cannot run on the RO +# root) — it verifies the tools it needs and fails naming what's missing; +# - skips the GRUB leg deliberately (the kernel cmdline, incl. any 1GB-hugepage reservation, is +# image-owned; update-grub aborted the whole run on the RO /boot); +# - renders systemd units into /run/systemd/system and enables them with --runtime; +# - mounts hugetlbfs at runtime instead of appending to fstab, and skips the limits.conf memlock +# append (the unit already sets LimitMEMLOCK=infinity; interactive-run memlock is not an +# appliance concern). +# Runtime tuning is unchanged: modprobe msr, the grow-only HugePages sysctl (#328), and the +# cpupower governor via ExecStartPre all work on a read-only root. +RIGFORGE_APPLIANCE="${RIGFORGE_APPLIANCE:-0}" # every consumer tests `= 1`; anything else is off +if [ "$RIGFORGE_APPLIANCE" = 1 ]; then + # Preset only — an explicit SYSTEMD_DIR in the environment (the test sandbox) still wins, and + # the non-appliance default below keeps this value because it is now set. + SYSTEMD_DIR="${SYSTEMD_DIR:-/run/systemd/system}" +fi +# Unit-enablement mode: appliance units live in /run, so their wants/ symlinks must too (a plain +# `enable` would write them to the volatile /etc overlay — working until reboot, then gone). +ENABLE_RUNTIME="" +if [ "$RIGFORGE_APPLIANCE" = 1 ]; then ENABLE_RUNTIME="--runtime"; fi + # System paths the script writes to. Overridable so the test suite can redirect them at a sandbox # (the defaults are the real locations, so production behaviour is unchanged). LOGROTATE_DIR="${LOGROTATE_DIR:-/etc/logrotate.d}" @@ -105,9 +132,13 @@ BIN_DIR="${BIN_DIR:-/usr/local/bin}" # Read-only system paths the `doctor` health check inspects (overridable for tests). MEMINFO="${MEMINFO:-/proc/meminfo}" +# CPU flags source for the ISA preflight (#338). Same override name util/proposed-grub.sh already +# uses for its pdpe1gb probe — one knob, and the test harness already isolates it from the host. +CPUINFO="${CPUINFO:-/proc/cpuinfo}" MSR_MODULE_DIR="${MSR_MODULE_DIR:-/sys/module/msr}" GOVERNOR_FILE="${GOVERNOR_FILE:-/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor}" HUGEPAGES_1G_NR="${HUGEPAGES_1G_NR:-/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages}" +NR_HUGEPAGES_FILE="${NR_HUGEPAGES_FILE:-/proc/sys/vm/nr_hugepages}" # Hashrate-capping-hardware diagnostics (#67): RAM layout (dmidecode) + effective CPU clock under load. DMIDECODE="${DMIDECODE:-dmidecode}" RDMSR_BIN="${RDMSR_BIN:-rdmsr}" # msr-tools, for doctor's register-level MSR verification (#66) @@ -118,6 +149,10 @@ MIN_CLOCK_PCT="${MIN_CLOCK_PCT:-75}" # warn when the loaded clock is below this # BIOS/firmware advisory (#78): board/BIOS identity + SMT state (both world-readable from sysfs). DMI_DIR="${DMI_DIR:-/sys/class/dmi/id}" SMT_CONTROL="${SMT_CONTROL:-/sys/devices/system/cpu/smt/control}" +# Kernel lockdown state (#333): securityfs exposes the active level, world-readable. Distro kernels +# (Ubuntu >= 20.04, RHEL, Debian) turn lockdown on automatically under UEFI Secure Boot, and lockdown +# blocks every /dev/cpu/*/msr write — so the MSR mod silently can't apply. Read, never inferred. +LOCKDOWN_FILE="${LOCKDOWN_FILE:-/sys/kernel/security/lockdown}" # systemd service name for the worker. SERVICE_NAME="${SERVICE_NAME:-xmrig}" @@ -227,6 +262,11 @@ _sed_escape_replacement() { # -> escaped value on stdout check_prerequisites() { log "Verifying system prerequisites..." if ! command -v jq &>/dev/null; then + # Appliance mode never installs packages — same contract as install_dependencies: the read-only + # image bakes everything at build, and an install here would silently violate it (pithead#797 R1). + if [ "$RIGFORGE_APPLIANCE" = 1 ]; then + error "jq is required but missing — appliance mode never installs packages; bake jq into the image." + fi if [ "$OS_TYPE" == "Darwin" ]; then if command -v brew &>/dev/null; then log "Installing prerequisite: jq..." @@ -701,7 +741,20 @@ _detect_pkg_manager() { DEP_LIST="git build-essential cmake libuv1-dev libssl-dev libhwloc-dev gettext-base python3" if [ "$OS_TYPE" == "Linux" ]; then # msr-tools (rdmsr): lets `doctor` verify the prefetcher MSR mod actually applied (#66). - DEP_LIST="$DEP_LIST linux-tools-common msr-tools" + DEP_LIST="$DEP_LIST msr-tools" + # cpupower's package name differs by distro: linux-tools-common on Ubuntu, + # linux-cpupower on Debian. apt's install transaction is all-or-nothing, so one unknown + # name would fail the ENTIRE dependency install — gcc/cmake included (#327). Probe with + # apt-cache show (same guard as the kernel-versioned package below) and add only the + # name this distro actually ships; when neither exists, warn and carry on — cpupower is + # a tuning aid, never worth losing the compiler toolchain over. + if apt-cache show linux-tools-common &>/dev/null; then + DEP_LIST="$DEP_LIST linux-tools-common" + elif apt-cache show linux-cpupower &>/dev/null; then + DEP_LIST="$DEP_LIST linux-cpupower" + else + warn "No cpupower package found (tried linux-tools-common, linux-cpupower) — skipping it. Frequency tuning via cpupower may be unavailable." + fi if apt-cache show "linux-tools-$(uname -r)" &>/dev/null; then DEP_LIST="$DEP_LIST linux-tools-$(uname -r)" fi @@ -737,7 +790,36 @@ _missing_deps() { printf '%s' "$missing" } +# Appliance-mode tool check (pithead#797 R1), shared with the setup --dry-run plan (#146 — never a +# second copy to drift). Echoes the tools setup would actually invoke that are absent. `command -v`, +# not the package manager: on the appliance, dpkg/alternatives state rides the volatile /etc overlay +# and can be stale while the tools in /usr are fine (observed on the R0 bench after a reboot). +# envsubst renders the unit on every run; the compiler chain only matters when a build is pending — +# the R0 bench restored mining from a cached build with a half-broken toolchain, and the every-boot +# re-run must keep doing that. +_missing_appliance_tools() { + local t missing="" tools="envsubst" + [ "$XMRIG_REBUILD" = true ] && tools="git cmake make cc envsubst" + for t in $tools; do + command -v "$t" >/dev/null 2>&1 || missing="$missing $t" + done + printf '%s' "$missing" +} + install_dependencies() { + # Appliance mode: the toolchain is baked into the image at build — apt cannot run on the + # read-only root, and even a rw-remounted install leaves its /etc state on the volatile overlay + # (pithead#797 R0 item 1). So never install here: verify and fail naming the missing tools — + # that's an image build bug, not something a rig can fix at runtime. + if [ "$RIGFORGE_APPLIANCE" = 1 ]; then + local _missing + _missing="$(_missing_appliance_tools)" + if [ -n "$_missing" ]; then + error "Appliance mode: required tool(s) missing from the image:$_missing. The toolchain must be baked at image build — package install cannot run on the read-only root." + fi + log "Appliance mode: dependencies are baked into the image — skipping package install." + return 0 + fi if [ "$OS_TYPE" == "Darwin" ]; then log "Installing macOS dependencies..." if command -v brew &>/dev/null; then @@ -922,6 +1004,13 @@ generate_xmrig_config() { # where only one CCD has the V-cache). See issue #44. if [ "$OS_TYPE" != "Darwin" ]; then log "Detected CPU: ${CPU_MODEL:-unknown} — using XMRig auto-tuning (threads, asm, MSR, NUMA auto-detected)." + # ISA preflight (#338): surface missing AES-NI/AVX2 here, at the moment the rig is being + # configured, instead of letting unsupported hardware mine silently slow. Warn, never abort: + # XMRig still runs (soft AES / non-AVX2 init) and a knowingly-old rig is a valid choice. + local _missing_isa + _missing_isa=$(_cpu_missing_isa) + [[ " $_missing_isa " == *" aes "* ]] && warn "This CPU has no AES-NI: RandomX falls back to soft AES, roughly 4x slower. Mining will work, but expect a fraction of a modern CPU's rate." + [[ " $_missing_isa " == *" avx2 "* ]] && warn "This CPU has no AVX2: dataset init will be slower (steady-state hashrate is unaffected)." fi # Rig label for the pool `user` field (#22): any pool entry that didn't set its own `user` gets the @@ -1018,6 +1107,13 @@ generate_xmrig_config() { chmod 600 config.json if [ "$OS_TYPE" == "Linux" ]; then + # Appliance mode: no logrotate drop-in — /etc is a volatile overlay and the image does not run + # logrotate; log policy on the appliance belongs to the integration layer (pithead#797 R2), + # which re-renders the miner's config every boot and can cap or journald-route the log there. + if [ "$RIGFORGE_APPLIANCE" = 1 ]; then + log "Appliance mode: skipping the logrotate policy (image-owned logging)." + return + fi log "Configuring log rotation policy..." # Install logrotate configuration sudo tee "$LOGROTATE_DIR/xmrig" >/dev/null </dev/null sudo systemctl daemon-reload - sudo systemctl enable --now rigforge-autotune.timer 2>/dev/null || true + sudo systemctl enable ${ENABLE_RUNTIME:+"$ENABLE_RUNTIME"} --now rigforge-autotune.timer 2>/dev/null || true } # Install (or remove) the systemd timer that runs the miner watchdog periodically, based on the @@ -1153,7 +1249,7 @@ install_watchdog() { envsubst '$WATCHDOG_INTERVAL_MIN' \ <"$SCRIPT_DIR/systemd/rigforge-watchdog.timer.template" | sudo tee "$tmr" >/dev/null sudo systemctl daemon-reload - sudo systemctl enable --now rigforge-watchdog.timer 2>/dev/null || true + sudo systemctl enable ${ENABLE_RUNTIME:+"$ENABLE_RUNTIME"} --now rigforge-watchdog.timer 2>/dev/null || true } # Sister API (#99/#164, xmrig-model): one tiny persistent python3-stdlib server ships pre-computed @@ -1227,8 +1323,8 @@ install_api() { RIGFORGE_OPERATOR="$REAL_USER" SCRIPT_DIR="$SCRIPT_DIR" envsubst '$RIGFORGE_OPERATOR $SCRIPT_DIR' <"$SCRIPT_DIR/systemd/rigforge-api-refresh.service.template" | sudo tee "$rsvc" >/dev/null sudo tee "$rtmr" <"$SCRIPT_DIR/systemd/rigforge-api-refresh.timer.template" >/dev/null sudo systemctl daemon-reload - sudo systemctl enable --now rigforge-api-refresh.timer 2>/dev/null || true - sudo systemctl enable rigforge-api.service 2>/dev/null || true + sudo systemctl enable ${ENABLE_RUNTIME:+"$ENABLE_RUNTIME"} --now rigforge-api-refresh.timer 2>/dev/null || true + sudo systemctl enable ${ENABLE_RUNTIME:+"$ENABLE_RUNTIME"} rigforge-api.service 2>/dev/null || true # restart, not just enable --now: a bind/port/token change must be re-read (restart also starts). sudo systemctl restart rigforge-api.service 2>/dev/null || true # Prime the state files so the first poll isn't a 503 for a whole timer period. @@ -1280,11 +1376,11 @@ install_control() { sudo rm -f "$usvc" "$upath" fi sudo systemctl daemon-reload - sudo systemctl enable --now rigforge-control-apply.path 2>/dev/null || true + sudo systemctl enable ${ENABLE_RUNTIME:+"$ENABLE_RUNTIME"} --now rigforge-control-apply.path 2>/dev/null || true if [ "${CONTROL_UPGRADE:-disabled}" = "enabled" ]; then - sudo systemctl enable --now rigforge-control-upgrade.path 2>/dev/null || true + sudo systemctl enable ${ENABLE_RUNTIME:+"$ENABLE_RUNTIME"} --now rigforge-control-upgrade.path 2>/dev/null || true fi - sudo systemctl enable rigforge-control.service 2>/dev/null || true + sudo systemctl enable ${ENABLE_RUNTIME:+"$ENABLE_RUNTIME"} rigforge-control.service 2>/dev/null || true # restart, not just enable --now: a bind/port/token/upgrade-flag change must be re-read (restart also starts). sudo systemctl restart rigforge-control.service 2>/dev/null || true } @@ -1337,6 +1433,48 @@ _cmdline_reserved_2mb() { # -> reserved 2MB-equivalent pages awk '{ cur=""; def=""; t=0; for (i=1;i<=NF;i++) { if ($i ~ /^hugepagesz=/) { split($i,a,"="); cur=a[2] } else if ($i ~ /^default_hugepagesz=/) { split($i,a,"="); def=a[2] } else if ($i ~ /^hugepages=/) { split($i,a,"="); n=a[2]+0; s=(cur!=""?cur:def); if (s=="1G"||s=="1073741824"||s=="1048576K") t+=n*512; else t+=n } } print t+0 }' <<<"$1" } +# 2MB pages the running miner already holds (kB -> pages), 0 when stopped: pages it holds now are +# pages it re-uses across a restart, so they count as available when sizing the pool. HugetlbPages +# spans every page size, so a dataset sitting in 1GB pages overstates this credit — in exactly the +# case where the 2MB pool is not needed for the dataset, so nothing actually used goes unreserved. +_miner_held_hugepages() { + local pid kb + pid=$(systemctl show "$SERVICE_NAME.service" -p MainPID --value 2>/dev/null) || pid="" + if [ -n "$pid" ] && [ "$pid" -gt 0 ] 2>/dev/null && [ -r "/proc/$pid/status" ]; then + kb=$(awk '/^HugetlbPages:/ { print $2; exit }' "/proc/$pid/status" 2>/dev/null) + echo $((${kb:-0} / 2048)) + return + fi + echo 0 +} + +# Grow-only runtime reservation (#328, the runtime half of #305's keep-existing guard): another +# consumer may hold pages from the same pool (a co-hosted pithead stack's p2pool/monerod), and +# writing our raw requirement shrinks that pool to its in-use floor — 0 free pages on both sides, +# measured live on the pithead#797 appliance bench. Ensure the miner's pages are AVAILABLE (free +# now, or already held by the running miner) and grow the pool by the shortfall only; never shrink +# a reservation someone else made. Idempotent: a re-run that finds enough available writes nothing. +# ponytail: with headroom configured AND the co-resident already running, its live pages and the +# headroom both count toward the target — a bounded over-reserve in the safe direction; tighten to +# max(headroom, live use) if a real box ever needs the difference. +_hugepages_avail() { # -> 2MB pages the miner could draw on right now (free + already held) + local free held + free=$(awk '/^HugePages_Free:/ { print $2; exit }' "$MEMINFO" 2>/dev/null) || free="" + held=$(_miner_held_hugepages) + echo $((${free:-0} + held)) +} + +_ensure_hugepages() { # + local required=$1 current avail + current=$(cat "$NR_HUGEPAGES_FILE" 2>/dev/null) || current=0 + avail=$(_hugepages_avail) + if [ "$avail" -lt "$required" ]; then + sudo sysctl -w vm.nr_hugepages=$((current + required - avail)) + else + log "HugePages pool already covers the miner ($avail of $required pages available; pool: $current) — leaving it as-is (#328)." + fi +} + tune_kernel() { if [ "$OS_TYPE" != "Linux" ]; then log "Skipping kernel tuning (Not supported on $OS_TYPE)." @@ -1346,7 +1484,13 @@ tune_kernel() { if [[ "$(uname -m)" == "x86_64" || "$(uname -m)" == "i686" ]]; then log "Enabling MSR module for hardware prefetcher tuning..." sudo modprobe msr 2>/dev/null || true - if [ -d "$MODULES_LOAD_DIR" ]; then + if [ "$RIGFORGE_APPLIANCE" = 1 ]; then + # The modprobe above is the whole job on the appliance: setup re-runs every boot (the + # pithead boot leg), so a modules-load drop-in is persistence machinery for a + # persistence layer the box doesn't have. Smaller than writing /run/modules-load.d + # for the same effect — that dir is only read at boot, before this run exists. + log "Appliance mode: msr loaded at runtime only (no modules-load drop-in — setup re-runs each boot)." + elif [ -d "$MODULES_LOAD_DIR" ]; then echo "msr" | sudo tee "$MODULES_LOAD_DIR/msr.conf" >/dev/null elif [ -f "$MODULES_FILE" ]; then append_once "$MODULES_FILE" "msr" @@ -1361,17 +1505,24 @@ tune_kernel() { # Calculate exact requirement based on hardware, the tuned thread count, and 1GB page status REQUIRED_PAGES=$(RX_THREADS="$RX_SETUP_THREADS" RESERVE_EXTRA_MB="${HUGEPAGES_RESERVE_EXTRA_MB:-0}" THREADS_CAP="${THREADS_CAP:-}" "$SCRIPT_DIR/util/proposed-grub.sh" --runtime) log "Hardware-optimized HugePages: $REQUIRED_PAGES (2MB pages) calculated." - sudo sysctl -w vm.nr_hugepages="$REQUIRED_PAGES" + _ensure_hugepages "$REQUIRED_PAGES" else # Fallback when proposed-grub.sh is missing: 3072 × 2MB = 6 GB of huge pages — enough for the # ~2.3 GB RandomX dataset plus per-thread scratchpads on a large desktop/server, without over- # reserving on smaller hosts. proposed-grub.sh computes an exact, hardware-sized value instead. warn "Utility script not found. Fallback to safe default (3072)." - sudo sysctl -w vm.nr_hugepages=3072 + _ensure_hugepages 3072 fi log "Configuring bootloader (GRUB) for persistent HugePages..." - if [ -f "$SCRIPT_DIR/util/proposed-grub.sh" ] && [ -f "$GRUB_DEFAULT" ]; then + if [ "$RIGFORGE_APPLIANCE" = 1 ]; then + # A deliberate skip, not the utility-not-found fallback below: on the appliance /boot is + # read-only and the kernel cmdline (incl. any 1GB-hugepage reservation) is image-owned — + # an unguarded update-grub took the whole R0 run down after a completed compile. Without + # the 1G reservation XMRig falls back to 2MB pages at 100%: working, small known cost + # (pithead#797 R0 item 2). + log "Appliance mode: skipping GRUB updates — the kernel cmdline is image-owned." + elif [ -f "$SCRIPT_DIR/util/proposed-grub.sh" ] && [ -f "$GRUB_DEFAULT" ]; then # proposed-grub.sh prints a generic "quiet splash" prefix plus the HugePage/MSR params we # manage. Keep only the params we manage and MERGE them into the existing cmdline so we don't # clobber other kernel parameters the user/distro set (#19 — boot-safety). @@ -1402,6 +1553,22 @@ configure_limits() { return fi + # Appliance mode: mount hugetlbfs at RUNTIME only. The fstab lines would land on the volatile + # /etc overlay and vanish at reboot (proven on the R0 bench) — setup re-runs each boot and just + # mounts again. The limits.conf memlock append covers interactive runs only, which are not an + # appliance concern: the unit already sets LimitMEMLOCK=infinity. + if [ "$RIGFORGE_APPLIANCE" = 1 ]; then + log "Appliance mode: mounting hugetlbfs at runtime (no fstab or limits.conf writes)..." + sudo mkdir -p "$HUGEPAGES_1G_DIR" + mountpoint -q /dev/hugepages 2>/dev/null || + sudo mount -t hugetlbfs hugetlbfs /dev/hugepages || + warn "Could not mount /dev/hugepages. Check 'dmesg' for details." + mountpoint -q "$HUGEPAGES_1G_DIR" 2>/dev/null || + sudo mount -t hugetlbfs -o pagesize=1G hugetlbfs_1g "$HUGEPAGES_1G_DIR" || + warn "Could not mount $HUGEPAGES_1G_DIR (1G pages need an image-owned cmdline reservation)." + return 0 + fi + log "Configuring persistent HugePage mounts and memory limits..." sudo mkdir -p "$HUGEPAGES_1G_DIR" @@ -1540,7 +1707,16 @@ _setup_plan() { _p "checking the build" "skip the build — XMRig $XMRIG_VERSION already built at the pinned commit" fi _p "preparing workspace" "workspace at $WORKER_ROOT (an existing prior install would be archived first)" - if [ "$OS_TYPE" = "Darwin" ]; then + if [ "$RIGFORGE_APPLIANCE" = 1 ]; then + # Same check install_dependencies runs (shared _missing_appliance_tools — no second copy). + local _mt + _mt="$(_missing_appliance_tools)" + if [ -n "$_mt" ]; then + _p "installing dependencies" "appliance mode: FAIL — required tool(s) missing from the image:$_mt (the toolchain must be baked at image build)" + else + _p "installing dependencies" "appliance mode: baked into the image — no package install" + fi + elif [ "$OS_TYPE" = "Darwin" ]; then _p "installing dependencies" "install/verify via brew: cmake libuv openssl hwloc" elif _detect_pkg_manager; then local _md @@ -1567,10 +1743,16 @@ _setup_plan() { local _msr _pages="(proposed-grub.sh missing — fallback 3072)" _grubline="GRUB: will check at run time" _reboot="" _msr="write msr to $MODULES_LOAD_DIR/msr.conf (module autoload)" [ -e "$MODULES_LOAD_DIR/msr.conf" ] && _msr="msr module already configured" + # Appliance arms: the same flag checks tune_kernel makes, previewed (pithead#797 R1). The + # GRUB probe/diff below is also gated off — the appliance skip line must stand as-is. + if [ "$RIGFORGE_APPLIANCE" = 1 ]; then + _msr="modprobe msr at runtime only (appliance: no modules-load drop-in)" + _grubline="skipping GRUB updates (appliance: the kernel cmdline is image-owned)" + fi if [ -f "$SCRIPT_DIR/util/proposed-grub.sh" ]; then _rx_setup_threads _pages=$(RX_THREADS="$RX_SETUP_THREADS" RESERVE_EXTRA_MB="${HUGEPAGES_RESERVE_EXTRA_MB:-0}" THREADS_CAP="${THREADS_CAP:-}" "$SCRIPT_DIR/util/proposed-grub.sh" --runtime 2>/dev/null) || _pages="?" - if [ -f "$GRUB_DEFAULT" ]; then + if [ "$RIGFORGE_APPLIANCE" != 1 ] && [ -f "$GRUB_DEFAULT" ]; then _grub_proposed if [ "$CURRENT" = "$MERGED" ]; then _grubline="GRUB already configured (no reboot needed for it)" @@ -1580,16 +1762,25 @@ _setup_plan() { fi fi fi - _p "tuning the kernel" "$_msr; reserve $_pages 2MB HugePages (runtime sysctl); $_grubline$_reboot" - local _f1="hugetlbfs /dev/hugepages hugetlbfs defaults 0 0" _f2="hugetlbfs_1g $HUGEPAGES_1G_DIR hugetlbfs pagesize=1G 0 0" _add="" - grep -qxF "$_f1" "$FSTAB" 2>/dev/null || _add=" '$_f1'" - grep -qxF "$_f2" "$FSTAB" 2>/dev/null || _add="$_add '$_f2'" - if [ -n "$_add" ]; then - _p "configuring limits" "append to $FSTAB:$_add; memlock unlimited for $REAL_USER in $LIMITS_CONF" + # Grow-only preview (#328): same availability check _ensure_hugepages runs, read-only. + local _hpline="grow the pool so $_pages 2MB HugePages are available (grow-only runtime sysctl)" + if [[ "$_pages" =~ ^[0-9]+$ ]] && [ "$(_hugepages_avail)" -ge "$_pages" ] 2>/dev/null; then + _hpline="HugePages pool already covers the miner ($_pages pages needed) — no change" + fi + _p "tuning the kernel" "$_msr; $_hpline; $_grubline$_reboot" + if [ "$RIGFORGE_APPLIANCE" = 1 ]; then + _p "configuring limits" "appliance mode: mount hugetlbfs at runtime — no $FSTAB or $LIMITS_CONF writes (volatile /etc)" else - _p "configuring limits" "fstab already configured; memlock unlimited for $REAL_USER in $LIMITS_CONF" + local _f1="hugetlbfs /dev/hugepages hugetlbfs defaults 0 0" _f2="hugetlbfs_1g $HUGEPAGES_1G_DIR hugetlbfs pagesize=1G 0 0" _add="" + grep -qxF "$_f1" "$FSTAB" 2>/dev/null || _add=" '$_f1'" + grep -qxF "$_f2" "$FSTAB" 2>/dev/null || _add="$_add '$_f2'" + if [ -n "$_add" ]; then + _p "configuring limits" "append to $FSTAB:$_add; memlock unlimited for $REAL_USER in $LIMITS_CONF" + else + _p "configuring limits" "fstab already configured; memlock unlimited for $REAL_USER in $LIMITS_CONF" + fi fi - _p "installing the service" "render systemd/xmrig.service.template -> $SYSTEMD_DIR/$SERVICE_NAME.service (User=${MINER_USER:-root}), daemon-reload, enable --now" + _p "installing the service" "render systemd/xmrig.service.template -> $SYSTEMD_DIR/$SERVICE_NAME.service (User=${MINER_USER:-root}), daemon-reload, enable${ENABLE_RUNTIME:+ $ENABLE_RUNTIME} --now" fi case "$AUTOTUNE_MODE" in disabled) _p "configuring autotune" "no periodic timer (autotune disabled) — an installed one would be removed" ;; @@ -1975,16 +2166,35 @@ S_y="" S_t="" S_g="" S_pr="" -S_hj="" # cpu.huge-pages-jit (off by default; swept only if TUNE_HPJIT lists >1 value) -S_cq="" # randomx.cache_qos (off by default; swept only if TUNE_CACHEQOS lists >1 value) -S_wr="" # randomx.wrmsr (off by default; swept only if TUNE_WRMSR lists >1 value) (#66) -HILL_BEST="" # set by _hillclimb (its result is returned via this global, not stdout — see below) -HP_CAP_THREADS="" # #65: max thread count whose 2MB-page need fits the reservation (empty = check off) -_TUNE_SVC_STOPPED=0 # set by tune() when it stops the live service for a --bench run (#2) +S_hj="" # cpu.huge-pages-jit (off by default; swept only if TUNE_HPJIT lists >1 value) +S_cq="" # randomx.cache_qos (off by default; swept only if TUNE_CACHEQOS lists >1 value) +S_wr="" # randomx.wrmsr (off by default; swept only if TUNE_WRMSR lists >1 value) (#66) +HILL_BEST="" # set by _hillclimb (its result is returned via this global, not stdout — see below) +HP_CAP_THREADS="" # #65: max thread count whose 2MB-page need fits the reservation (empty = check off) +_TUNE_SVC_STOPPED=0 # set by tune() when it stops the live service for a --bench run (#2) +_TUNE_LIVE_DIRTY=0 # #347: a live sweep is mid-flight — the overrides file holds a candidate, not a decision +_TUNE_PRE_OVERRIDES="" # #347: the overrides content from before the live sweep ("" = no file existed) + +# Put a previous overrides snapshot back — write it (or remove the file when none existed) and re-apply +# it to the running miner. Shared by the A/B confirm's revert leg (#64) and the aborted-live-sweep trap +# (#347), so the abort path restores exactly the way a completed run does. +_restore_overrides() { # + if [ -n "$1" ]; then printf '%s\n' "$1" | sudo tee "$TUNE_OVERRIDES" >/dev/null; else sudo rm -f "$TUNE_OVERRIDES"; fi + _apply_runtime >/dev/null 2>&1 || true +} # Restart the miner that a --bench run stopped, and clean the temp dir. Installed as an EXIT trap by # tune() so the service comes back even if the run errors or is interrupted. _tune_bench_cleanup() { + # #347: an aborted LIVE sweep would otherwise persist the mid-sweep candidate — _measure_live applies + # each one straight into tune-overrides.json and restarts the miner on it. Restore the pre-sweep + # overrides FIRST: the restore itself restarts the service, and any restart must re-read the restored + # file, never the candidate. tune() clears the flag once the winner deliberately lands in the file. + if [ "${_TUNE_LIVE_DIRTY:-0}" = 1 ]; then + _TUNE_LIVE_DIRTY=0 + log "Live sweep interrupted — restoring the pre-tune overrides and restarting the miner on them." + _restore_overrides "$_TUNE_PRE_OVERRIDES" + fi [ -n "${TUNE_TMP:-}" ] && rm -rf "$TUNE_TMP" 2>/dev/null if [ "${_TUNE_SVC_STOPPED:-0}" = 1 ]; then _TUNE_SVC_STOPPED=0 @@ -2831,6 +3041,11 @@ tune() { : >"$RESULTS_FILE" if [ "$TUNE_MODE" = live ]; then + # #347: from here every candidate lands in tune-overrides.json and restarts the miner on it + # (_measure_live). Mark the file dirty so the EXIT trap above restores the pre-sweep overrides + # if the sweep doesn't reach its deliberate winner write below. + _TUNE_PRE_OVERRIDES="$pre_overrides" + _TUNE_LIVE_DIRTY=1 log "Auto-tuning LIVE against the running miner (warmup ${TUNE_LIVE_WARMUP:-60}s, ${TUNE_LIVE_SAMPLES:-3} samples) — search=$TUNE_SEARCH, knobs={$ACTIVE_KNOBS}." else log "Auto-tuning via 'xmrig --bench=$TUNE_BENCH' (median of $TUNE_ITERS) — search=$TUNE_SEARCH, knobs={$ACTIVE_KNOBS}, min-delta=$TUNE_MIN_DELTA." @@ -2899,6 +3114,7 @@ tune() { case " $ACTIVE_KNOBS " in *" cacheqos "*) ovr=$(printf '%s' "$ovr" | jq --argjson cq "$G_cq" '.randomx.cache_qos = $cq') ;; esac case " $ACTIVE_KNOBS " in *" wrmsr "*) ovr=$(printf '%s' "$ovr" | jq --argjson wr "$G_wr" '.randomx.wrmsr = $wr') ;; esac printf '%s\n' "$ovr" >"$TUNE_TMP/ovr.json" && sudo cp "$TUNE_TMP/ovr.json" "$TUNE_OVERRIDES" + _TUNE_LIVE_DIRTY=0 # #347: the winner — a completed decision — now owns the file; the abort-restore stands down # Assemble the full search log: the winner, the search parameters, and every measured candidate. jq -s --argjson p "$G_p" --argjson y "$G_y" --arg t "$G_t" --argjson g "$G_g" --argjson pr "$G_pr" \ @@ -2953,8 +3169,7 @@ _tune_confirm_live() { # sleep "$warm" win_hr=$(_sample_api_median "$n" "$iv") [ -n "$win_hr" ] || win_hr=0 - if [ -n "$pre_ovr" ]; then printf '%s\n' "$pre_ovr" | sudo tee "$TUNE_OVERRIDES" >/dev/null; else sudo rm -f "$TUNE_OVERRIDES"; fi - _apply_runtime >/dev/null 2>&1 || true + _restore_overrides "$pre_ovr" sleep "$warm" base_hr=$(_sample_api_median "$n" "$iv") [ -n "$base_hr" ] || base_hr=0 @@ -2981,6 +3196,26 @@ _autotune_set_prefetch() { # rm -f "$tmp" } +# #347: autotune's abort protection. Each trial merges a candidate prefetch mode into the overrides file +# and restarts the miner on it — an abort mid-sweep would otherwise persist that candidate. The EXIT trap +# restores the pre-sweep mode through the SAME settle step the clean path ends with, so the two cannot +# diverge. The merge-based restore (_autotune_set_prefetch) preserves any offline-`tune` knobs, exactly +# like the sweep's own writes. +_AUTOTUNE_DIRTY=0 +_AUTOTUNE_PRE_MODE="" +_AUTOTUNE_OVR_FILE="" +_autotune_settle() { # — merge the mode into the overrides and leave the miner running on it + _autotune_set_prefetch "$_AUTOTUNE_OVR_FILE" "$1" + _apply_runtime >/dev/null 2>&1 || true +} +_autotune_cleanup() { + if [ "${_AUTOTUNE_DIRTY:-0}" = 1 ]; then + _AUTOTUNE_DIRTY=0 + log "autotune: sweep interrupted — restoring prefetch_mode=$_AUTOTUNE_PRE_MODE." + _autotune_settle "$_AUTOTUNE_PRE_MODE" + fi +} + # Sample the live miner for one candidate: median H/s over the window and, for the efficiency target, # the average package watts over that SAME window — prints "hrwatts" (watts empty for perf or when # no power source). RAPL brackets the sampling window; a TUNE_POWER_CMD override is polled once per @@ -3083,6 +3318,12 @@ autotune() { last_applied="$cur" log "autotune: optimizing for $(_autotune_desc "$target"); live-sweeping prefetch modes [$modes] against the running miner; baseline mode=$cur at $(_autotune_fmt "$target" "$base_hr" "$base_w") (median of $n)." + # #347: arm the abort-restore before the first candidate lands in the overrides file. + _AUTOTUNE_PRE_MODE="$cur" + _AUTOTUNE_OVR_FILE="$overrides" + _AUTOTUNE_DIRTY=1 + trap '_autotune_cleanup' EXIT + # Try every OTHER mode once, live; track the running best by the target's score. for m in $modes; do [ "$m" = "$cur" ] && continue @@ -3111,9 +3352,9 @@ autotune() { fi # Leave the chosen mode running (the sweep may have ended on a different one). if [ "$last_applied" != "$best_mode" ]; then - _autotune_set_prefetch "$overrides" "$best_mode" - _apply_runtime >/dev/null 2>&1 || true + _autotune_settle "$best_mode" fi + _AUTOTUNE_DIRTY=0 # #347: a deliberate final mode is in place; the abort-restore stands down } # Read the current total hashrate from the worker's HTTP API (empty if unreachable). Overridable for @@ -3497,10 +3738,11 @@ _status_api_summary() { echo "RigForge: worker API not reachable at 127.0.0.1:8080 (miner stopped or still starting)." return 0 fi - # One jq fork for every field, tab-separated (bash-3.2-safe read into locals). + # One jq fork for every field, tab-separated (bash-3.2-safe read into locals). /2/summary reports + # hugepages as a [loaded, total] pages array — @tsv rejects nested arrays (exit 5, killing the whole + # row on every healthy rig, #341), so serialize it; scalar/absent shapes pass through untouched. IFS=$(printf '\t') read -r hs pool up acc rej hp < <(printf '%s' "$body" | - jq -r '[(.hashrate.total[0] // 0), (.connection.pool // "?"), (.uptime // 0), - (.connection.accepted // 0), (.connection.rejected // 0), (.hugepages // "")] | @tsv' 2>/dev/null) || true + jq -r '[(.hashrate.total[0] // 0), (.connection.pool // "?"), (.uptime // 0), (.connection.accepted // 0), (.connection.rejected // 0), (.hugepages // "" | if type == "array" then join("/") else . end)] | @tsv' 2>/dev/null) || true [ -n "${hs:-}" ] || return 0 # half-up API / unparseable body: stay quiet, platform block follows printf ' %-10s %s H/s\n' "Hashrate:" "$hs" printf ' %-10s %s\n' "Pool:" "$pool" @@ -3567,7 +3809,7 @@ svc_enable() { mac_enable return } - sudo systemctl enable "$SERVICE_NAME" && log "Enabled $SERVICE_NAME (starts on boot)." + sudo systemctl enable ${ENABLE_RUNTIME:+"$ENABLE_RUNTIME"} "$SERVICE_NAME" && log "Enabled $SERVICE_NAME (starts on boot)." } svc_disable() { [ "$OS_TYPE" = "Linux" ] || { @@ -3713,6 +3955,32 @@ apply() { # config actually changed (so tune/autotune restarts, which reuse _apply_runtime not apply(), and # a re-apply of the same config, never bump the revision). _stamp_config_meta "${RIGFORGE_CONFIG_SOURCE:-local}" "${RIGFORGE_CONFIG_CHANGE_ID:-}" + _apply_pool_check +} + +# #343: apply's honesty check. "Applied config and restarted" used to be the last word even when the +# rig then sat at 0 H/s forever (an unresolvable pools[0].url just loops on DNS errors, silently) — +# so after the restart + reconcile, ask the miner itself: poll its local API briefly for a live pool +# connection and WARN when none appears. Warn, never refuse: the pool may be legitimately down at +# apply time and the config is already applied, so the exit code stays 0 (doctor is the judgmental +# verb). Only apply() calls this — tune/autotune restart via _apply_runtime and verify liveness +# their own way (_wait_miner_live). Tries/interval overridable for the suite. +_apply_pool_check() { + [ "$OS_TYPE" = Linux ] || return 0 # no restart happened (macOS apply only regenerates) + systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null || return 0 # stopped service: nothing to verify + local tries="${APPLY_POOL_TRIES:-5}" iv="${APPLY_POOL_IVL:-3}" i st="" pool="" n="" + for i in $(seq 1 "$tries"); do + IFS="$(printf '\t')" read -r st pool n _ < # rigforge-control-upgrade.path unit when the receiver stages an upgrade intent. Fetches the target # RigForge release and applies it, health-gated with rollback to the prior version. Every failure path # returns 0 with a recorded status (served by the receiver's GET /status) — a bad request must not -# wedge the oneshot. The staged version is a CONFIRMATION guard, not a target selector: this verb -# bounds what it will act on (D4/D10) so a compromised trigger can only ever land a real, reachable, -# NEWER release — never an arbitrary tag, a downgrade, or a dangling commit. +# wedge the oneshot. The staged version IS the target (D4) — the dashboard re-derives latest host-side, +# this verb makes no version check of its own: it bounds what it will act on (D4/D10) so a compromised +# trigger can only ever land a real, reachable, NEWER release — never an arbitrary tag, a downgrade, or +# a dangling commit. control_upgrade() { [ "$OS_TYPE" != "Linux" ] && error "control-upgrade is driven by the rigforge-control-upgrade.path unit and is Linux-only." parse_config # need API_PORT etc. so the post-build liveness check can read the miner @@ -4206,6 +4475,40 @@ _cpu_eff_khz() { if [ "$n" -gt 0 ]; then echo $((sum / n)); fi # always exit 0 (empty output when no data) } +# Kernel lockdown level (#333). securityfs prints every level with the ACTIVE one in square brackets, +# e.g. "none [integrity] confidentiality" (security/lockdown/lockdown.c:lockdown_read). Echoes just the +# bracketed level; empty when the file is absent or unreadable (lockdown LSM not built in, securityfs +# not mounted, or a non-root reader) — "unknown", never "off". Always exits 0: an unreadable probe is +# missing information, not a failure, and must not trip the ERR trap. +_lockdown_state() { # -> none|integrity|confidentiality, or empty when unknown + [ -r "$LOCKDOWN_FILE" ] || return 0 + sed -nE 's/.*\[([a-z]+)\].*/\1/p' "$LOCKDOWN_FILE" 2>/dev/null | head -1 + return 0 +} + +# RandomX ISA preflight (#338, the last acceptance criterion from #1): echo which of aes / avx2 the +# CPU lacks, from the kernel's flags line. RandomX without AES-NI silently falls back to XMRig's +# soft-AES path (~4x slower) and without AVX2 dataset init slows — neither aborts anything, so a rig +# on unsupported hardware "works" at a mysteriously bad rate unless somebody says why. Judged ONLY +# when an x86-style "flags" line exists: no flags line (macOS has no /proc, ARM cpuinfo says +# "Features", sandboxes stub the file) means unknown, and unknown never manufactures an issue — the +# same stance as #333's lockdown probe. `-w` so vaes/avx2_vnni style neighbors can't false-match. +_cpu_missing_isa() { # -> "aes", "avx2", "aes avx2", or empty when all present / undeterminable + grep -q '^flags' "$CPUINFO" 2>/dev/null || return 0 + local missing="" + grep -qw aes "$CPUINFO" || missing="aes" + grep -qw avx2 "$CPUINFO" || missing="$missing${missing:+ }avx2" + printf '%s' "$missing" + return 0 +} + +# True when lockdown is at a level that blocks MSR writes (#333). LOCKDOWN_MSR sits below +# LOCKDOWN_INTEGRITY_MAX in enum lockdown_reason (include/linux/security.h), so both `integrity` and +# `confidentiality` deny the write; only `none` permits it. +_lockdown_blocks_msr() { # -> 0 when MSR writes are denied + case "$1" in integrity | confidentiality) return 0 ;; *) return 1 ;; esac +} + # #66 MSR-verification helpers. doctor confirms the prefetcher MSR mod actually took effect — not just # that the `msr` module loaded — in two layers: XMRig's own log line (always available) and an rdmsr # read-back (when msr-tools is installed), which catches a write a hypervisor / kernel-lockdown silently @@ -4496,8 +4799,17 @@ _api_config_meta_json() { fi } +# #346: the last control outcome mirrored into the feed as `rigforge.control`. Pithead's host runner +# caps its synchronous /status poll after a worker-apply, so a slow rollback outruns it; rather than a +# new authenticated dial to the control port, the rig mirrors {change_id, status, reason} from the +# status.json _control_status already writes 644, and the next routine poll catches up. Missing, +# unreadable, or malformed status.json -> null — the mirror must never break the feed. +_api_control_json() { + jq -c '{change_id, status, reason}' "${RIGFORGE_CONTROL_STATE:-/var/lib/rigforge-control}/status.json" 2>/dev/null || echo null +} + _api_rigforge_block() { # - jq -n --arg v "$(cat "$SCRIPT_DIR/VERSION" 2>/dev/null || echo unknown)" --arg xv "$XMRIG_VERSION" --arg xc "$XMRIG_COMMIT" --argjson tune "$(_api_tune_json)" --argjson power "$(_api_power_json "$1")" --argjson health "$(_health_json)" --argjson watchdog "$(_watchdog_json)" --argjson config "$(_api_config_json)" --argjson config_meta "$(_api_config_meta_json)" '{version: $v, xmrig_version: $xv, xmrig_commit: $xc, tune: $tune, power: $power, health: $health, watchdog: $watchdog, config: $config, config_meta: $config_meta}' + jq -n --arg v "$(cat "$SCRIPT_DIR/VERSION" 2>/dev/null || echo unknown)" --arg xv "$XMRIG_VERSION" --arg xc "$XMRIG_COMMIT" --argjson tune "$(_api_tune_json)" --argjson power "$(_api_power_json "$1")" --argjson health "$(_health_json)" --argjson watchdog "$(_watchdog_json)" --argjson config "$(_api_config_json)" --argjson config_meta "$(_api_config_meta_json)" --argjson control "$(_api_control_json)" '{version: $v, xmrig_version: $xv, xmrig_commit: $xc, tune: $tune, power: $power, health: $health, watchdog: $watchdog, config: $config, config_meta: $config_meta, control: $control}' } # Produce the sister API's response bodies: compute once, write atomically (tmp + rename, the @@ -4524,6 +4836,50 @@ api_refresh() { # --- Doctor: one-stop health check --- +# --- Pool-connection probe (#343), shared by doctor and apply --- + +# The miner's own verdict on its pool connection, read from the local /2/summary (API_CMD test hook +# + Bearer discipline via _xmrig_summary_json). One TSV line: +# connected — a stratum connection is live +# disconnected — miner answers, but no live connection +# api-down — no parseable summary (API unreachable) +# connection.uptime is XMRig's seconds-since-connect: positive exactly while a pool connection is +# live, 0 while it retries (DNS failure, refused, pool down) — the signal #343 found missing. We +# read the miner's view rather than dialing the pool ourselves: a second dial from this script can +# disagree with the miner's (proxied, TLS) one, and the miner is the party that has to be connected. +_pool_conn_status() { + local body pool cup fails acc + body=$(_xmrig_summary_json) + if ! printf '%s' "$body" | jq -e '.connection' >/dev/null 2>&1; then + echo api-down + return 0 + fi + # The "?" placeholder is jq-side, not bash-side: a disconnected xmrig reports pool "", and an + # empty FIRST tsv field would be swallowed by read (tab is IFS *whitespace*, so leading tabs + # strip) — every later field would shift one left and "failures: 2" would parse as a live + # connection. Emitting a non-empty field for every column keeps the read aligned. + IFS="$(printf '\t')" read -r pool cup fails acc </dev/null) +EOF + [ -n "${pool:-}" ] || pool="?" # belt (jq guarantees non-empty) and braces (an unparseable body) + if [ "${cup:-0}" -gt 0 ] 2>/dev/null; then + printf 'connected\t%s\t%s\t%s\n' "$pool" "$cup" "${acc:-0}" + else + printf 'disconnected\t%s\t%s\n' "$pool" "${fails:-0}" + fi +} + +# One guarded TCP dial, rc-only — doctor's fallback when the miner isn't running to testify itself. +# Host and port ride in as positional args (never interpolated into the -c string, so a hostile +# config.json can't inject shell), and `timeout` bounds the hang a filtered port would cause. +_tcp_probe() { # + if command -v timeout >/dev/null 2>&1; then + timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1"' "$1" "$2" 2>/dev/null + else + bash -c 'exec 3<>"/dev/tcp/$0/$1"' "$1" "$2" 2>/dev/null + fi +} + doctor() { cmd_version if [ "$OS_TYPE" != "Linux" ]; then @@ -4534,13 +4890,56 @@ doctor() { local issues=0 # Service active? + local svc_up=n if systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then + svc_up=y _ck_ok "service '$SERVICE_NAME' is active" else _ck_warn "service '$SERVICE_NAME' is not active — start it with: sudo $0 start" issues=$((issues + 1)) fi + # Pool connection (#343): the check that says whether the rig is doing its job at all — every + # other probe here can pass while a bad pools[0].url loops on DNS errors and mines nothing. + # Service running -> ask the miner itself (see _pool_conn_status); API silent -> advisory only + # (it may still be starting, and the service check already judges the service); service + # stopped -> one guarded TCP dial of pools[0], so a parked rig still learns whether its pool + # would even answer. + if [ -f "$CONFIG_JSON" ]; then + local pc_st="" pc_pool="" pc_n1="" pc_n2="" pool0 ph pp + pool0=$(jq -r '.pools[0].url // empty' "$CONFIG_JSON" 2>/dev/null || true) + if [ "$svc_up" = y ]; then + IFS="$(printf '\t')" read -r pc_st pc_pool pc_n1 pc_n2 </dev/null || true)" _pool_conn_status) +EOF + case "$pc_st" in + connected) + _ck_ok "pool connection live: $pc_pool (up $(_render_duration "${pc_n1:-0}"), ${pc_n2:-0} share(s) accepted)" + ;; + disconnected) + { [ -n "$pc_pool" ] && [ "$pc_pool" != "?" ]; } || pc_pool="${pool0:-?}" + _ck_warn "miner is running but has NO live pool connection (pool: $pc_pool, ${pc_n1:-0} failed attempt(s)) — the rig is not mining; check pools[0].url in config.json and '$0 logs'" + issues=$((issues + 1)) + ;; + *) + _ck_info "worker API not reachable at 127.0.0.1:8080 — can't verify the pool connection (the miner may still be starting; re-run doctor shortly)" + ;; + esac + elif [ -n "$pool0" ]; then + ph=${pool0#*://} + pp=${ph##*:} + ph=${ph%:*} + if [[ "$pp" =~ ^[0-9]+$ ]]; then + if _tcp_probe "$ph" "$pp"; then + _ck_info "pool $ph:$pp accepts TCP connections (miner stopped — the live check runs once it's started)" + else + _ck_warn "pool $ph:$pp is unreachable (TCP connect failed) — the rig cannot mine until pools[0].url points at a reachable pool" + issues=$((issues + 1)) + fi + fi + fi + fi + # HugePages reserved? (the single biggest lever; needs a reboot after setup) local hp hp=$(awk '/^HugePages_Total:/ {print $2; exit}' "$MEMINFO" 2>/dev/null) @@ -4560,6 +4959,21 @@ doctor() { _ck_warn "1GB HugePages not reserved (optional; needs a pdpe1gb CPU + reboot)" fi + # CPU ISA support (#338): a rig without AES-NI mines at soft-AES speed (~4x slower) with no + # error anywhere — exactly the "silently failing" #1's acceptance criterion forbids, so it's a + # counted issue. Missing AVX2 only slows dataset init: advisory. Quiet when there's no x86 + # flags line to judge (unknown, not unsupported). + local miss_isa + miss_isa=$(_cpu_missing_isa) + if [[ " $miss_isa " == *" aes "* ]]; then + _ck_warn "CPU has no AES-NI — RandomX runs soft AES, roughly 4x slower; this hardware cannot mine at a competitive rate" + issues=$((issues + 1)) + else + # The ok line only when there IS a flags line to have judged; no flags line = unknown, say nothing. + grep -q '^flags' "$CPUINFO" 2>/dev/null && _ck_ok "CPU supports AES-NI (hardware RandomX path)" || true + fi + [[ " $miss_isa " == *" avx2 "* ]] && _ck_info "CPU has no AVX2 — dataset init is slower (steady-state hashrate unaffected)" + # Resolve the worker's xmrig.log once — the MSR-applied (#66) and HUGE PAGES checks both read it. local wr="" log_file="" if [ -f "$CONFIG_JSON" ]; then @@ -4678,10 +5092,29 @@ doctor() { # MSR mod applied? (#66) The ~10-15% RandomX gain needs three things, checked in order: the msr # module loadable, XMRig's own log line confirming it WROTE the prefetcher preset, and — when rdmsr # (msr-tools) is present — a register read-back that catches a write a hypervisor/lockdown dropped. + # Kernel lockdown (#333) — checked BEFORE the MSR probes below, because it is the CAUSE and they are + # the symptom. Under lockdown, msr_write()/msr_ioctl() call security_locked_down(LOCKDOWN_MSR) and + # return -EPERM *before* consulting the allow_writes filter (arch/x86/kernel/msr.c), so the + # msr.allow_writes=on we put on the cmdline cannot override it. Reading the state beats inferring it + # from Secure Boot: whether Secure Boot implies lockdown is a distro patch, not upstream behaviour. + local ld + ld=$(_lockdown_state) + if [ -z "$ld" ]; then + _ck_info "kernel lockdown state unknown ($LOCKDOWN_FILE unreadable) — run 'doctor' as root, or the lockdown LSM isn't built in" + elif _lockdown_blocks_msr "$ld"; then + _ck_warn "kernel lockdown is ACTIVE ('$ld') — it denies every /dev/cpu/*/msr write, so the MSR mod cannot apply (worth ~5-15% RandomX). msr.allow_writes=on does NOT override it: disable Secure Boot in BIOS ($(_bios_menu "$(_dmi board_vendor)" secure_boot "${TUNE_TARGET:-perf}")), then re-run 'sudo $0 doctor'." + issues=$((issues + 1)) + else + _ck_ok "kernel lockdown: none — MSR writes are permitted" + fi + if [ -d "$MSR_MODULE_DIR" ]; then _ck_ok "msr kernel module loaded" else - _ck_warn "msr module not loaded — the MSR mod won't apply; if it persists, disable Secure Boot" + # Secure Boot does NOT stop the in-tree, distro-signed `msr` module from loading, so the old + # "disable Secure Boot" hint pointed the wrong way here (#333). A missing module is a modprobe / + # packaging problem; lockdown is reported separately above. + _ck_warn "msr module not loaded — the MSR mod won't apply; check 'sudo modprobe msr' and $MODULES_LOAD_DIR/msr.conf" issues=$((issues + 1)) fi if [ -n "$log_file" ] && [ -f "$log_file" ]; then @@ -4706,7 +5139,13 @@ EOF _ck_ok "MSR registers verified via rdmsr ($_MSR_OK/$_MSR_TOTAL match the $preset preset)" elif [ -n "$_MSR_BAD" ]; then # A genuine value mismatch — the write didn't take. This is the real failure (#66). - _ck_warn "MSR registers don't match the $preset preset ($_MSR_OK/$_MSR_TOTAL ok;${_MSR_BAD}) — a hypervisor/lockdown may have dropped the write, or XMRig changed its preset" + # #333: when lockdown is active we already know which of the three causes it was — + # say so instead of listing all of them. + if _lockdown_blocks_msr "$ld"; then + _ck_warn "MSR registers don't match the $preset preset ($_MSR_OK/$_MSR_TOTAL ok;${_MSR_BAD}) — kernel lockdown ('$ld') dropped the write; see the lockdown item above" + else + _ck_warn "MSR registers don't match the $preset preset ($_MSR_OK/$_MSR_TOTAL ok;${_MSR_BAD}) — a hypervisor may have dropped the write, or XMRig changed its preset" + fi issues=$((issues + 1)) else # No mismatch, but some/all registers were unreadable (e.g. msr module not loaded). Advisory. @@ -4715,7 +5154,13 @@ EOF fi ;; fail) - _ck_warn "XMRig reports the MSR preset FAILED to set — check Secure Boot / msr.allow_writes=on" + # #333: msr.allow_writes=on is only the answer when lockdown is NOT the blocker — under + # lockdown the kernel rejects the write before the filter ever runs. + if _lockdown_blocks_msr "$ld"; then + _ck_warn "XMRig reports the MSR preset FAILED to set — kernel lockdown ('$ld') is denying the write; see the lockdown item above" + else + _ck_warn "XMRig reports the MSR preset FAILED to set — check msr.allow_writes=on and that no hypervisor is masking MSR access" + fi issues=$((issues + 1)) ;; *) : ;; # no msr line yet (the miner may not have started a RandomX job) — stay quiet @@ -4857,6 +5302,21 @@ _bios_detect() { local mem pop nch spd rated smt effk maxk pct cpu_m B_MEM_STATUS=unknown B_MEM_BEFORE="" B_SMT_STATUS=unknown B_SMT_BEFORE="" B_BOOST_STATUS=unknown B_BOOST_BEFORE="" B_NPS_STATUS=unknown B_NPS_BEFORE="" + B_SB_STATUS=unknown B_SB_BEFORE="" + # Secure Boot / lockdown (#333). Same probe doctor uses (the #80 rule: they can never disagree). + # `unknown` when securityfs isn't readable — never guessed from Secure Boot state, which only + # implies lockdown on distro kernels, not upstream. + local ld_state + ld_state=$(_lockdown_state) + if [ -n "$ld_state" ]; then + if _lockdown_blocks_msr "$ld_state"; then + B_SB_STATUS=pending + B_SB_BEFORE="lockdown=$ld_state (MSR writes denied)" + else + B_SB_STATUS=ok + B_SB_BEFORE="lockdown=none" + fi + fi cpu_m=$(lscpu 2>/dev/null | awk -F: '/^Model name:/ {gsub(/^[ \t]+/, "", $2); print $2; exit}' || true) case "$cpu_m" in *EPYC*) @@ -4918,6 +5378,13 @@ _bios_menu() { # -> menu-path line(s) on stdou memory_profile:msi) echo "OC ▸ A-XMP / EXPO ▸ Profile 1" ;; memory_profile:*) echo "look for the memory profile setting (XMP / EXPO / DOCP) and enable profile 1" ;; smt:*) echo "Advanced ▸ CPU Configuration ▸ SMT / Hyper-Threading ▸ Enabled" ;; + # #333: Secure Boot itself is the lever — distro kernels enable lockdown from it, and lockdown is + # what denies the MSR writes. On most boards the toggle only appears once the OS type is "Other OS". + secure_boot:asus) echo "Boot ▸ Secure Boot ▸ OS Type ▸ Other OS (then Secure Boot state reads 'Disabled')" ;; + secure_boot:asrock) echo "Security ▸ Secure Boot ▸ Disabled" ;; + secure_boot:gigabyte) echo "Boot ▸ Secure Boot ▸ Secure Boot Enable ▸ Disabled" ;; + secure_boot:msi) echo "Settings ▸ Advanced ▸ Windows OS Configuration ▸ Secure Boot ▸ Disabled" ;; + secure_boot:*) echo "find the Secure Boot setting (usually under Boot or Security) and disable it" ;; numa_nps:*) echo "Advanced ▸ AMD CBS ▸ DF Common Options ▸ Memory Addressing ▸ NUMA nodes per socket ▸ NPS4" ;; power_boost:*) if [ "$3" = efficiency ]; then @@ -4938,6 +5405,7 @@ _bios_state_write() { # ; reads the B_* globals + TUNE_TARGET if [ "$B_SMT_STATUS" = pending ]; then items=$(jq -c --argjson a "$items" --arg b "$B_SMT_BEFORE" --arg m "$(_bios_menu "$vendor" smt "$TUNE_TARGET")" -n '$a + [{id: "smt", status: "pending", before: $b, menu: $m}]'); fi if [ "$B_BOOST_STATUS" = pending ]; then items=$(jq -c --argjson a "$items" --arg b "$B_BOOST_BEFORE" --arg m "$(_bios_menu "$vendor" power_boost "$TUNE_TARGET")" -n '$a + [{id: "power_boost", status: "pending", before: $b, menu: $m}]'); fi if [ "$B_NPS_STATUS" = pending ]; then items=$(jq -c --argjson a "$items" --arg b "$B_NPS_BEFORE" --arg m "$(_bios_menu "$vendor" numa_nps "$TUNE_TARGET")" -n '$a + [{id: "numa_nps", status: "pending", before: $b, menu: $m}]'); fi + if [ "$B_SB_STATUS" = pending ]; then items=$(jq -c --argjson a "$items" --arg b "$B_SB_BEFORE" --arg m "$(_bios_menu "$vendor" secure_boot "$TUNE_TARGET")" -n '$a + [{id: "secure_boot", status: "pending", before: $b, menu: $m}]'); fi jq -n --arg t "$TUNE_TARGET" --arg when "$(date '+%Y-%m-%d %H:%M')" --argjson items "$items" '{target: $t, saved: $when, items: $items}' >"$f" log "Saved $(jq -r '.items | length' "$f") pending item(s) to $f." } @@ -4948,6 +5416,7 @@ _bios_item_label() { # -> human label smt) printf 'SMT / Hyper-Threading' ;; power_boost) printf 'CPU boost / power' ;; numa_nps) printf 'NUMA per socket (NPS)' ;; + secure_boot) printf 'Secure Boot (kernel lockdown)' ;; esac } @@ -4970,12 +5439,17 @@ _bios_guide() { # if [ "$B_BOOST_STATUS" = unknown ]; then _ck_info "CPU boost not checked — the miner isn't running (start it and re-run bios to include the power/boost item)"; fi if [ "$B_NPS_STATUS" = ok ]; then _ck_ok "NUMA per socket (NPS): $B_NPS_BEFORE"; fi if [ "$B_NPS_STATUS" = pending ]; then _ck_warn "NUMA per socket (NPS): $B_NPS_BEFORE — set NUMA nodes per socket to NPS4 in BIOS so RandomX gets quadrant-local memory."; fi - for id in memory_profile smt power_boost numa_nps; do + if [ "$B_SB_STATUS" = ok ]; then _ck_ok "Secure Boot / kernel lockdown: $B_SB_BEFORE — the MSR mod can apply"; fi + if [ "$B_SB_STATUS" = pending ]; then _ck_warn "Secure Boot / kernel lockdown: $B_SB_BEFORE — the MSR mod (~5-15% RandomX) cannot apply until Secure Boot is off."; fi + if [ "$B_SB_STATUS" = unknown ]; then _ck_info "Secure Boot / kernel lockdown: can't verify — $LOCKDOWN_FILE isn't readable (run as root, or the lockdown LSM isn't built in)"; fi + # Secure Boot leads: it's the cheapest change and it gates the MSR mod outright. + for id in secure_boot memory_profile smt power_boost numa_nps; do case "$id" in memory_profile) if [ "$B_MEM_STATUS" = pending ]; then pending="$pending $id"; fi ;; smt) if [ "$B_SMT_STATUS" = pending ]; then pending="$pending $id"; fi ;; power_boost) if [ "$B_BOOST_STATUS" = pending ]; then pending="$pending $id"; fi ;; numa_nps) if [ "$B_NPS_STATUS" = pending ]; then pending="$pending $id"; fi ;; + secure_boot) if [ "$B_SB_STATUS" = pending ]; then pending="$pending $id"; fi ;; esac done if [ -z "$pending" ]; then @@ -5033,6 +5507,10 @@ _bios_verify() { # fresh_status="$B_NPS_STATUS" fresh_before="$B_NPS_BEFORE" ;; + secure_boot) + fresh_status="$B_SB_STATUS" + fresh_before="$B_SB_BEFORE" + ;; *) continue ;; esac if [ "$fresh_status" = ok ]; then @@ -5049,6 +5527,7 @@ _bios_verify() { # memory_profile) _ck_warn "$(_bios_item_label "$id") — can't verify (run as root so dmidecode can read the RAM state)." ;; smt) _ck_warn "$(_bios_item_label "$id") — can't verify (no SMT control exposed in sysfs on this system)." ;; numa_nps) _ck_warn "$(_bios_item_label "$id") — can't verify (lscpu didn't report an EPYC CPU model, so the NUMA-node count can't be checked)." ;; + secure_boot) _ck_warn "$(_bios_item_label "$id") — can't verify ($LOCKDOWN_FILE isn't readable; re-run as root)." ;; esac kept="$kept $id" fi @@ -5065,10 +5544,12 @@ _bios_verify() { # case " $kept " in *" smt "*) : ;; *) B_SMT_STATUS="done" ;; esac case " $kept " in *" power_boost "*) : ;; *) B_BOOST_STATUS="done" ;; esac case " $kept " in *" numa_nps "*) : ;; *) B_NPS_STATUS="done" ;; esac + case " $kept " in *" secure_boot "*) : ;; *) B_SB_STATUS="done" ;; esac if [ "$B_BOOST_STATUS" = unknown ]; then B_BOOST_STATUS=pending; fi # keep it resumable if [ "$B_MEM_STATUS" = unknown ]; then B_MEM_STATUS=pending; fi if [ "$B_SMT_STATUS" = unknown ]; then B_SMT_STATUS=pending; fi if [ "$B_NPS_STATUS" = unknown ]; then B_NPS_STATUS=pending; fi + if [ "$B_SB_STATUS" = unknown ]; then B_SB_STATUS=pending; fi _bios_state_write "$state" log "$applied of $total applied, $(jq -r '.items | length' "$state") still pending. Reboot into BIOS to finish, then run 'sudo $0 bios' again." fi diff --git a/systemd/rigforge-control-upgrade.service.template b/systemd/rigforge-control-upgrade.service.template index 831d60e..3162a91 100644 --- a/systemd/rigforge-control-upgrade.service.template +++ b/systemd/rigforge-control-upgrade.service.template @@ -11,9 +11,9 @@ Type=oneshot # and systemctl), so ProtectSystem / ProtectHome / NoNewPrivileges / an empty CapabilityBoundingSet # would each break it — the same reason the control-apply oneshot isn't sandboxed. The security for # this path lives in the verb, not in unit flags: it treats the staged file as untrusted (root-owned -# handoff, strict version whitelist), re-derives the real latest release and refuses any non-latest, -# unreachable, or older target, and throttles repeat runs (D4/D6/D8/D10). RIGFORGE_OPERATOR (baked at -# setup) is who the updated checkout and build get handed back to. +# handoff, strict version whitelist), bounds the staged target — refusing anything that is not a real, +# reachable release newer than the one installed — and throttles repeat runs (D4/D6/D8/D10). +# RIGFORGE_OPERATOR (baked at setup) is who the updated checkout and build get handed back to. Environment=RIGFORGE_OPERATOR=$RIGFORGE_OPERATOR ExecStart=$SCRIPT_DIR/rigforge.sh control-upgrade Nice=19 diff --git a/tests/README.md b/tests/README.md index 0f6d9ee..716b8ae 100644 --- a/tests/README.md +++ b/tests/README.md @@ -13,7 +13,7 @@ compiles XMRig and mines. Each layer covers what the one below it has to stub. | Layer | File | Runs | What it proves | How to run | |---|---|---|---|---| | **Unit + black-box suite** | [`run.sh`](run.sh) | Any host (macOS/Linux), no Docker. **In CI.** | Config parsing, the XMRig-config generation matrix (every CPU/OS profile, simulated via PATH stubs), GRUB/HugePages math, the command surface, tune search, doctor: everything that doesn't need a real `/etc` or real hardware. The bulk of coverage. | `make test` (lint + suite) or `bash tests/run.sh` | -| **Linux container e2e** | [`e2e/linux.sh`](e2e/linux.sh) → [`e2e/in-container.sh`](e2e/in-container.sh) | Disposable Ubuntu container, **needs Docker**. **In CI.** | The genuine Linux deploy path against a real (throwaway) `/etc` with real GNU tools (`sed -i`, `tee`, `envsubst`) + idempotency on re-run. Only the heavy/privileged bits (compile, package install, `systemctl`/`mount`) are stubbed. | `make test-e2e` | +| **Linux container e2e** | [`e2e/linux.sh`](e2e/linux.sh) → [`e2e/in-container.sh`](e2e/in-container.sh) | Disposable Ubuntu container, **needs Docker**. **In CI.** | The genuine Linux deploy path against a real (throwaway) `/etc` with real GNU tools (`sed -i`, `tee`, `envsubst`) + idempotency on re-run. Only the heavy/privileged bits (compile, package install, `systemctl`/`mount`) are stubbed. A second pass re-runs the container with `RIGFORGE_APPLIANCE=1` (#348) and proves the appliance contracts on that real `/etc`: units land in `/run/systemd/system`, no package installs, no fstab/limits.conf/logrotate writes, every enable carries `--runtime`. | `make test-e2e` | | **Native macOS e2e** | [`e2e/macos.sh`](e2e/macos.sh) | A real Mac, **CI-only** (runs as a step in the macOS job). | The macOS deploy path with genuine BSD tools the Linux CI can only stub: BSD `sed`, the macOS config profile, `mac_*` process control (real `nohup` + PID file), the launchd login agent, `backup`/`restore`. | `make test-e2e-macos` | | **Coverage gate** | [`coverage.sh`](coverage.sh) | kcov in **Docker**. **In CI.** | Line coverage of `rigforge.sh` + `util/proposed-grub.sh` by running `run.sh` under kcov; enforces the committed floor ([`coverage-floor.txt`](coverage-floor.txt)) plus a patch-coverage gate (diff-cover) on changed lines. | `make coverage` | | **Release smoke (quick)** | [`smoke.sh`](smoke.sh) | Real Linux rig, **manual**. Not in CI. | The compiled binary actually starts and hashes (`xmrig --bench`, fully offline). Fast pre-tag confidence that the worker we ship runs. | `make smoke` | diff --git a/tests/e2e/in-container.sh b/tests/e2e/in-container.sh index b935255..ea23f3d 100755 --- a/tests/e2e/in-container.sh +++ b/tests/e2e/in-container.sh @@ -2,7 +2,10 @@ # # Runs INSIDE a disposable Linux container (invoked by tests/e2e/linux.sh). Provisions a writable copy # of the repo, runs the real rigforge.sh twice against the container's real /etc, and asserts the -# Linux deploy path + idempotency. Exits non-zero on any failed assertion. +# Linux deploy path + idempotency. With RIGFORGE_APPLIANCE=1 in the environment (linux.sh's second +# pass, #348) it instead asserts the appliance contracts against the same real /etc: units in +# /run/systemd/system, no package installs, no fstab/limits.conf/logrotate writes, --runtime enables. +# Exits non-zero on any failed assertion. # set -uo pipefail @@ -20,6 +23,16 @@ assert_rc() { if [ "$2" = "$3" ]; then ok "$1"; else bad "$1" "expected rc $3, g assert_eq() { if [ "$2" = "$3" ]; then ok "$1"; else bad "$1" "expected [$3], got [$2]"; fi; } assert_contains() { case "$2" in *"$3"*) ok "$1" ;; *) bad "$1" "[$2] missing [$3]" ;; esac } assert_absent() { case "$2" in *"$3"*) bad "$1" "[$2] unexpectedly contains [$3]" ;; *) ok "$1" ;; esac } +summarize() { # print the tally and exit (non-zero if any assertion failed) + echo "" + printf 'in-container: \033[1;32m%d passed\033[0m, ' "$PASS" + if [ "$FAIL" -gt 0 ]; then + printf '\033[1;31m%d failed\033[0m\n' "$FAIL" + exit 1 + fi + printf '0 failed\n' + exit 0 +} # 1. Real prerequisites: jq + envsubst (gettext). Installed with the REAL apt before stubs go on # PATH, so the script's own dependency step is the only thing we stub out. @@ -61,6 +74,7 @@ exec "$@" X cat >"$STUBS/git" <<'X' #!/usr/bin/env bash +echo "[git] $*" >>"${CALL_LOG:-/dev/null}" case "$*" in *rev-parse*) echo "${XMRIG_COMMIT:-}" ;; # #18 verifies the cloned commit *clone*) mkdir -p xmrig/src; printf 'static int DonateLevel = 1;\n' > xmrig/src/donate.h ;; @@ -75,9 +89,11 @@ echo "Socket(s): 2" X printf '#!/usr/bin/env bash\necho 8\n' >"$STUBS/nproc" printf '#!/usr/bin/env bash\necho poolbox\n' >"$STUBS/hostname" -# No-op the rest (sysctl -w / mount / systemctl etc. cannot run unprivileged in a container). -for c in cmake make systemctl modprobe mount cpupower update-grub sysctl dpkg nft; do - printf '#!/usr/bin/env bash\nexit 0\n' >"$STUBS/$c" +# No-op the rest (sysctl -w / mount / systemctl etc. cannot run unprivileged in a container). Each +# stub logs "[cmd] args" to $CALL_LOG (run.sh's idiom) so call-shape assertions read real evidence +# instead of an always-empty file. cc exists only for the appliance pass's baked-toolchain probe. +for c in cmake make systemctl modprobe mount cpupower update-grub sysctl dpkg nft cc; do + printf '#!/usr/bin/env bash\necho "[%s] $*" >>"${CALL_LOG:-/dev/null}"\nexit 0\n' "$c" >"$STUBS/$c" done chmod +x "$STUBS"/* export PATH="$STUBS:$PATH" @@ -94,6 +110,78 @@ EOF BUILD="$WORK/data-home/worker/xmrig/build" ARCH="$(uname -m)" +# Appliance pass (#348): RIGFORGE_APPLIANCE=1 in the environment (linux.sh's second pass) asserts +# the appliance contracts against the same real container /etc, then exits — tests/run.sh already +# covers the mode's per-function branches with PATH stubs; this proves the /etc side for real. +# What is REAL here: the filesystem (/etc and /run), the unit renders (envsubst | tee), sed, jq, +# useradd, and the mountpoint probe. What stays STUBBED — so those contracts are proven at the +# argument level only: systemctl (no pid-1 systemd in a container; --runtime is asserted on the +# logged args), mount (needs privileges), the compile toolchain, and apt-get (stubbed to LOG so a +# wrongful install attempt becomes assertion evidence instead of a real package install). +if [ "${RIGFORGE_APPLIANCE:-0}" = 1 ]; then + cat >"$STUBS/apt-get" <<'X' +#!/usr/bin/env bash +echo "[apt-get] $*" >>"${CALL_LOG:-/dev/null}" +exit 0 +X + chmod +x "$STUBS/apt-get" + # systemd owns /run/systemd/system on the real appliance; the container has no pid-1 systemd. + mkdir -p /run/systemd/system + # Byte-identical before/after is the contract: appliance mode writes NOTHING to these files. + # (Grepping for e.g. "memlock" would false-fail — the stock limits.conf documents it in comments.) + grub_before="$(cat /etc/default/grub)" + fstab_before="$(cat /etc/fstab)" + limits_before="$(cat /etc/security/limits.conf)" + + echo "== appliance run (the real /etc must stay untouched) ==" + aout="$(CALL_LOG="$WORK/appliance-calls.log" ./rigforge.sh &1)" + arc=$? + assert_rc "appliance run exits 0" "$arc" "0" + [ "$arc" = 0 ] || printf '%s\n' "$aout" | tail -20 + acalls="$(cat "$WORK/appliance-calls.log" 2>/dev/null)" + # Units land in /run/systemd/system, never on the volatile /etc overlay. + assert_eq "appliance: xmrig unit rendered into /run/systemd/system" "$([ -f /run/systemd/system/xmrig.service ] && echo y || echo n)" "y" + assert_eq "appliance: no xmrig unit in /etc/systemd/system" "$([ -e /etc/systemd/system/xmrig.service ] && echo present || echo absent)" "absent" + assert_eq "appliance: sister API server unit in /run (#99)" "$([ -f /run/systemd/system/rigforge-api.service ] && echo y || echo n)" "y" + assert_eq "appliance: API refresh timer in /run (#99)" "$([ -f /run/systemd/system/rigforge-api-refresh.timer ] && echo y || echo n)" "y" + assert_eq "appliance: no rigforge/xmrig unit anywhere under /etc/systemd/system" "$(find /etc/systemd/system \( -name '*.service' -o -name '*.timer' -o -name '*.path' \) 2>/dev/null | grep -c 'xmrig\|rigforge')" "0" + assert_contains "appliance: unit rendered by real envsubst" "$(cat /run/systemd/system/xmrig.service 2>/dev/null)" "ReadWritePaths=$WORK/data-home/worker" + assert_absent "appliance: server unit fully rendered (no unexpanded vars)" "$(cat /run/systemd/system/rigforge-api.service 2>/dev/null)" '$SCRIPT_DIR' + # Never installs packages: the toolchain reads baked (stub cc + git/cmake/make on PATH). + assert_contains "appliance: deps declared baked, no install" "$aout" "dependencies are baked into the image" + assert_absent "appliance: apt-get never invoked" "$acalls" "[apt-get]" + # /etc stays byte-identical: no fstab/limits/GRUB/modules-load/logrotate writes. + assert_eq "appliance: fstab byte-identical" "$(cat /etc/fstab)" "$fstab_before" + assert_eq "appliance: limits.conf byte-identical" "$(cat /etc/security/limits.conf)" "$limits_before" + assert_eq "appliance: GRUB byte-identical" "$(cat /etc/default/grub)" "$grub_before" + assert_eq "appliance: no GRUB backup written" "$([ -e /etc/default/grub.bak ] && echo present || echo absent)" "absent" + assert_contains "appliance: GRUB skip is deliberate (image-owned cmdline)" "$aout" "the kernel cmdline is image-owned" + assert_eq "appliance: no modules-load drop-in" "$([ -e /etc/modules-load.d/msr.conf ] && echo present || echo absent)" "absent" + assert_eq "appliance: no logrotate drop-in" "$([ -e /etc/logrotate.d/xmrig ] && echo present || echo absent)" "absent" + # Enablement is transient. The xmrig assert pins one real site (non-vacuous), the count guards + # every other enable site (timers, api, control) against a forgotten ${ENABLE_RUNTIME:+...}. + assert_contains "appliance: xmrig enable carries --runtime" "$acalls" "[systemctl] enable --runtime xmrig.service" + assert_eq "appliance: every systemctl enable is --runtime" "$(grep -F "[systemctl] enable" "$WORK/appliance-calls.log" | grep -cv -- --runtime)" "0" + # hugetlbfs is mounted at runtime instead of via fstab (mount is stubbed: argument-level proof; + # the real mountpoint probe reports not-mounted in a fresh container, so both mounts must fire). + assert_contains "appliance: runtime 2MB hugetlbfs mount" "$acalls" "[mount] -t hugetlbfs hugetlbfs /dev/hugepages" + assert_contains "appliance: runtime 1G hugetlbfs mount" "$acalls" "[mount] -t hugetlbfs -o pagesize=1G hugetlbfs_1g /dev/hugepages1G" + assert_eq "appliance: 1G mountpoint dir created for real" "$([ -d /dev/hugepages1G ] && echo y || echo n)" "y" + + echo "== appliance second run (the every-boot path accretes no /etc state) ==" + aout2="$(CALL_LOG="$WORK/appliance-calls2.log" ./rigforge.sh &1)" + arc2=$? + assert_rc "appliance re-run exits 0" "$arc2" "0" + [ "$arc2" = 0 ] || printf '%s\n' "$aout2" | tail -20 + assert_eq "appliance re-run: fstab still byte-identical" "$(cat /etc/fstab)" "$fstab_before" + assert_eq "appliance re-run: limits.conf still byte-identical" "$(cat /etc/security/limits.conf)" "$limits_before" + assert_eq "appliance re-run: GRUB still byte-identical" "$(cat /etc/default/grub)" "$grub_before" + assert_contains "appliance re-run: xmrig re-enabled --runtime" "$(cat "$WORK/appliance-calls2.log" 2>/dev/null)" "[systemctl] enable --runtime xmrig.service" + assert_eq "appliance re-run: every systemctl enable is --runtime" "$(grep -F "[systemctl] enable" "$WORK/appliance-calls2.log" | grep -cv -- --runtime)" "0" + + summarize +fi + # #146: the dry-run plan, against the REAL container: real dpkg probe, real /proc for the # HugePages count, real proposed-grub.sh for the exact GRUB before -> after diff. Run BEFORE the # real setup so the plan shows the fresh-box actions — and prove it changed nothing. @@ -273,10 +361,4 @@ assert_eq "uninstall: removed the 'rigforge' command from PATH" "$([ -L /usr/loc ./rigforge.sh uninstall --yes /dev/null 2>&1 assert_rc "uninstall is idempotent" "$?" "0" -echo "" -printf 'in-container: \033[1;32m%d passed\033[0m, ' "$PASS" -if [ "$FAIL" -gt 0 ]; then - printf '\033[1;31m%d failed\033[0m\n' "$FAIL" - exit 1 -fi -printf '0 failed\n' +summarize diff --git a/tests/e2e/linux.sh b/tests/e2e/linux.sh index 451dc96..4bc5dfc 100644 --- a/tests/e2e/linux.sh +++ b/tests/e2e/linux.sh @@ -9,7 +9,9 @@ # compile (git/cmake/make), the package install (dpkg reports "already present"), and the host-only # bits (systemctl/modprobe/mount/sysctl). Hardware detection is stubbed so the CPU profile is # deterministic. We force linux/amd64 so the x86-only MSR path actually fires (emulated on Apple -# Silicon). Run: tests/e2e/linux.sh (or: make test-e2e) +# Silicon). Two passes, a fresh container each: the standard deploy, then RIGFORGE_APPLIANCE=1 +# (#348) — appliance mode against a real /etc, which must stay untouched while units land in /run. +# Run: tests/e2e/linux.sh (or: make test-e2e) # set -uo pipefail @@ -28,14 +30,24 @@ fi # multi-arch index digest; --platform below selects linux/amd64 from it. Refresh with: # docker buildx imagetools inspect ubuntu:24.04 --format '{{.Manifest.Digest}}' IMAGE="ubuntu:24.04@sha256:786a8b558f7be160c6c8c4a54f9a57274f3b4fb1491cf65146521ae77ff1dc54" -echo "=================== E2E: $IMAGE (linux/amd64) ===================" -if docker run --rm --platform linux/amd64 \ - -v "$ROOT:/src:ro" \ - "$IMAGE" bash /src/tests/e2e/in-container.sh; then - echo "" - echo "rigforge e2e: $IMAGE passed" + +run_pass() { #