diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ad07a7..633ae2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,16 @@ All notable changes to RigForge are documented here. The format is based on ### Fixed +- **An interrupted live sweep no longer persists the mid-sweep candidate (#347).** `tune --live` + (and `tune --now --long`) applies every candidate straight into `tune-overrides.json` and restarts + the miner on it, but the cleanup trap only restored the temp dir and the service — a Ctrl+C, a + dropped SSH session, or an error mid-sweep left an arbitrary half-measured candidate saved and + running, silently costing hashrate until the next re-tune. The trap now restores the pre-sweep + overrides and restarts the miner on them, through the same restore step the `--confirm` revert leg + uses. `autotune` (and `tune --now`) had the same gap with its per-trial prefetch merges; its trap + restores the pre-sweep mode via the same merge the sweep uses, so knobs pinned by an offline + `tune` survive the abort too. + - **`status` no longer aborts on a healthy rig (#341).** XMRig's `/2/summary` reports `"hugepages"` as an array (`[pages, total]`); the status renderer fed it to jq's `@tsv`, which rejects arrays — so the operator's first command printed an abort line instead of diff --git a/docs/operations.md b/docs/operations.md index ce51634..67b4e7e 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -160,6 +160,11 @@ pass that sweeps every knob live (threads, yield, 1G-pages, not just prefetch), a BIOS, RAM, or cooling change. `tune --now` is the friendly name for the `autotune` engine; the standalone `autotune` verb still works and is what the scheduled timer below runs. +Live sweeps apply each candidate to the running miner as they measure it. If a sweep is interrupted +(Ctrl+C, a dropped SSH session) or errors out mid-run, RigForge restores the tuning that was in place +before the sweep and restarts the miner on it — a half-measured candidate is never left running or +saved to `tune-overrides.json`. + For a hands-off schedule, set `autotune` in `config.json` to a target and re-run `setup`. RigForge installs a systemd timer that periodically optimizes the prefetch mode against your live miner: diff --git a/rigforge.sh b/rigforge.sh index e01413b..bbf3cb6 100755 --- a/rigforge.sh +++ b/rigforge.sh @@ -2166,16 +2166,35 @@ S_y="" S_t="" S_g="" S_pr="" -S_hj="" # cpu.huge-pages-jit (off by default; swept only if TUNE_HPJIT lists >1 value) -S_cq="" # randomx.cache_qos (off by default; swept only if TUNE_CACHEQOS lists >1 value) -S_wr="" # randomx.wrmsr (off by default; swept only if TUNE_WRMSR lists >1 value) (#66) -HILL_BEST="" # set by _hillclimb (its result is returned via this global, not stdout — see below) -HP_CAP_THREADS="" # #65: max thread count whose 2MB-page need fits the reservation (empty = check off) -_TUNE_SVC_STOPPED=0 # set by tune() when it stops the live service for a --bench run (#2) +S_hj="" # cpu.huge-pages-jit (off by default; swept only if TUNE_HPJIT lists >1 value) +S_cq="" # randomx.cache_qos (off by default; swept only if TUNE_CACHEQOS lists >1 value) +S_wr="" # randomx.wrmsr (off by default; swept only if TUNE_WRMSR lists >1 value) (#66) +HILL_BEST="" # set by _hillclimb (its result is returned via this global, not stdout — see below) +HP_CAP_THREADS="" # #65: max thread count whose 2MB-page need fits the reservation (empty = check off) +_TUNE_SVC_STOPPED=0 # set by tune() when it stops the live service for a --bench run (#2) +_TUNE_LIVE_DIRTY=0 # #347: a live sweep is mid-flight — the overrides file holds a candidate, not a decision +_TUNE_PRE_OVERRIDES="" # #347: the overrides content from before the live sweep ("" = no file existed) + +# Put a previous overrides snapshot back — write it (or remove the file when none existed) and re-apply +# it to the running miner. Shared by the A/B confirm's revert leg (#64) and the aborted-live-sweep trap +# (#347), so the abort path restores exactly the way a completed run does. +_restore_overrides() { # + if [ -n "$1" ]; then printf '%s\n' "$1" | sudo tee "$TUNE_OVERRIDES" >/dev/null; else sudo rm -f "$TUNE_OVERRIDES"; fi + _apply_runtime >/dev/null 2>&1 || true +} # Restart the miner that a --bench run stopped, and clean the temp dir. Installed as an EXIT trap by # tune() so the service comes back even if the run errors or is interrupted. _tune_bench_cleanup() { + # #347: an aborted LIVE sweep would otherwise persist the mid-sweep candidate — _measure_live applies + # each one straight into tune-overrides.json and restarts the miner on it. Restore the pre-sweep + # overrides FIRST: the restore itself restarts the service, and any restart must re-read the restored + # file, never the candidate. tune() clears the flag once the winner deliberately lands in the file. + if [ "${_TUNE_LIVE_DIRTY:-0}" = 1 ]; then + _TUNE_LIVE_DIRTY=0 + log "Live sweep interrupted — restoring the pre-tune overrides and restarting the miner on them." + _restore_overrides "$_TUNE_PRE_OVERRIDES" + fi [ -n "${TUNE_TMP:-}" ] && rm -rf "$TUNE_TMP" 2>/dev/null if [ "${_TUNE_SVC_STOPPED:-0}" = 1 ]; then _TUNE_SVC_STOPPED=0 @@ -3022,6 +3041,11 @@ tune() { : >"$RESULTS_FILE" if [ "$TUNE_MODE" = live ]; then + # #347: from here every candidate lands in tune-overrides.json and restarts the miner on it + # (_measure_live). Mark the file dirty so the EXIT trap above restores the pre-sweep overrides + # if the sweep doesn't reach its deliberate winner write below. + _TUNE_PRE_OVERRIDES="$pre_overrides" + _TUNE_LIVE_DIRTY=1 log "Auto-tuning LIVE against the running miner (warmup ${TUNE_LIVE_WARMUP:-60}s, ${TUNE_LIVE_SAMPLES:-3} samples) — search=$TUNE_SEARCH, knobs={$ACTIVE_KNOBS}." else log "Auto-tuning via 'xmrig --bench=$TUNE_BENCH' (median of $TUNE_ITERS) — search=$TUNE_SEARCH, knobs={$ACTIVE_KNOBS}, min-delta=$TUNE_MIN_DELTA." @@ -3090,6 +3114,7 @@ tune() { case " $ACTIVE_KNOBS " in *" cacheqos "*) ovr=$(printf '%s' "$ovr" | jq --argjson cq "$G_cq" '.randomx.cache_qos = $cq') ;; esac case " $ACTIVE_KNOBS " in *" wrmsr "*) ovr=$(printf '%s' "$ovr" | jq --argjson wr "$G_wr" '.randomx.wrmsr = $wr') ;; esac printf '%s\n' "$ovr" >"$TUNE_TMP/ovr.json" && sudo cp "$TUNE_TMP/ovr.json" "$TUNE_OVERRIDES" + _TUNE_LIVE_DIRTY=0 # #347: the winner — a completed decision — now owns the file; the abort-restore stands down # Assemble the full search log: the winner, the search parameters, and every measured candidate. jq -s --argjson p "$G_p" --argjson y "$G_y" --arg t "$G_t" --argjson g "$G_g" --argjson pr "$G_pr" \ @@ -3144,8 +3169,7 @@ _tune_confirm_live() { # sleep "$warm" win_hr=$(_sample_api_median "$n" "$iv") [ -n "$win_hr" ] || win_hr=0 - if [ -n "$pre_ovr" ]; then printf '%s\n' "$pre_ovr" | sudo tee "$TUNE_OVERRIDES" >/dev/null; else sudo rm -f "$TUNE_OVERRIDES"; fi - _apply_runtime >/dev/null 2>&1 || true + _restore_overrides "$pre_ovr" sleep "$warm" base_hr=$(_sample_api_median "$n" "$iv") [ -n "$base_hr" ] || base_hr=0 @@ -3172,6 +3196,26 @@ _autotune_set_prefetch() { # rm -f "$tmp" } +# #347: autotune's abort protection. Each trial merges a candidate prefetch mode into the overrides file +# and restarts the miner on it — an abort mid-sweep would otherwise persist that candidate. The EXIT trap +# restores the pre-sweep mode through the SAME settle step the clean path ends with, so the two cannot +# diverge. The merge-based restore (_autotune_set_prefetch) preserves any offline-`tune` knobs, exactly +# like the sweep's own writes. +_AUTOTUNE_DIRTY=0 +_AUTOTUNE_PRE_MODE="" +_AUTOTUNE_OVR_FILE="" +_autotune_settle() { # — merge the mode into the overrides and leave the miner running on it + _autotune_set_prefetch "$_AUTOTUNE_OVR_FILE" "$1" + _apply_runtime >/dev/null 2>&1 || true +} +_autotune_cleanup() { + if [ "${_AUTOTUNE_DIRTY:-0}" = 1 ]; then + _AUTOTUNE_DIRTY=0 + log "autotune: sweep interrupted — restoring prefetch_mode=$_AUTOTUNE_PRE_MODE." + _autotune_settle "$_AUTOTUNE_PRE_MODE" + fi +} + # Sample the live miner for one candidate: median H/s over the window and, for the efficiency target, # the average package watts over that SAME window — prints "hrwatts" (watts empty for perf or when # no power source). RAPL brackets the sampling window; a TUNE_POWER_CMD override is polled once per @@ -3274,6 +3318,12 @@ autotune() { last_applied="$cur" log "autotune: optimizing for $(_autotune_desc "$target"); live-sweeping prefetch modes [$modes] against the running miner; baseline mode=$cur at $(_autotune_fmt "$target" "$base_hr" "$base_w") (median of $n)." + # #347: arm the abort-restore before the first candidate lands in the overrides file. + _AUTOTUNE_PRE_MODE="$cur" + _AUTOTUNE_OVR_FILE="$overrides" + _AUTOTUNE_DIRTY=1 + trap '_autotune_cleanup' EXIT + # Try every OTHER mode once, live; track the running best by the target's score. for m in $modes; do [ "$m" = "$cur" ] && continue @@ -3302,9 +3352,9 @@ autotune() { fi # Leave the chosen mode running (the sweep may have ended on a different one). if [ "$last_applied" != "$best_mode" ]; then - _autotune_set_prefetch "$overrides" "$best_mode" - _apply_runtime >/dev/null 2>&1 || true + _autotune_settle "$best_mode" fi + _AUTOTUNE_DIRTY=0 # #347: a deliberate final mode is in place; the abort-restore stands down } # Read the current total hashrate from the worker's HTTP API (empty if unreachable). Overridable for diff --git a/tests/run.sh b/tests/run.sh index 4fc5b18..9d86f87 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -4976,6 +4976,45 @@ out="$(cd "$TN" && PATH="$STUBS:$PATH" STUB_UNAME_S=Darwin RIGFORGE_HOME="$PWD" assert_rc "tune --live rejected on non-Linux" "$?" "1" assert_contains "tune --live non-Linux message" "$out" "only supported on Linux" +# #347: an INTERRUPTED live sweep must not persist the mid-sweep candidate. _measure_live applies every +# candidate straight into tune-overrides.json and restarts the miner on it; the EXIT trap must restore +# the pre-sweep overrides and restart the miner on THEM. The fake API touches a flag when a candidate +# window is in flight and then blocks — the test kills tune there (mid-iteration, candidate in the file) +# and asserts the file came back byte-identical and the miner saw a post-kill restart. `exec` makes the +# backgrounded PID the rigforge process itself, so the kill lands on the shell that owns the trap. +echo "== black-box: aborted tune --live restores the pre-sweep overrides (#347) ==" +cat >"$OVR" <<'EOF' +{ "randomx": { "scratchpad_prefetch_mode": 3 }, "cpu": { "rx": 4 } } +EOF +PRE347="$(cat "$OVR")" +FLAG347="$TN/flag347" +CL347="$TN/calls347.log" +rm -f "$FLAG347" +: >"$CL347" +(cd "$TN" && RIGFORGE_HOME="$PWD" PATH="$STUBS:$PATH" CALL_LOG="$CL347" LOGROTATE_DIR="$TN/logrotate" \ + FLAG347="$FLAG347" API_CMD='touch "$FLAG347"; sleep 60; echo 1500' \ + TUNE_LIVE_WARMUP=0 TUNE_LIVE_INTERVAL=0 TUNE_LIVE_SAMPLES=1 \ + TUNE_SEEDS=auto TUNE_PREFETCH_MODES="0 1" TUNE_YIELDS=false TUNE_THREADS=-1 TUNE_MAX_ROUNDS=1 \ + exec bash "$SCRIPT" tune --live "$TN/abort347.out" 2>&1) & +PID347=$! +for _ in $(seq 1 100); do + [ -f "$FLAG347" ] && break + sleep 0.1 +done +assert_eq "abort test reached a live candidate window (#347)" "$([ -f "$FLAG347" ] && echo y || echo n)" "y" +# The candidate is in the file RIGHT NOW — prove the pre-state was actually clobbered mid-sweep, so the +# restore assert below can't pass vacuously. +assert_eq "mid-sweep: overrides hold the candidate, not the pre-state (#347)" \ + "$([ "$(cat "$OVR")" = "$PRE347" ] && echo same || echo differs)" "differs" +RESTARTS_PRE347="$(grep -c '\[systemctl\] restart' "$CL347")" +kill "$PID347" 2>/dev/null +wait "$PID347" 2>/dev/null +assert_contains "aborted live sweep announces the restore (#347)" "$(cat "$TN/abort347.out")" "restoring the pre-tune overrides" +assert_eq "aborted live sweep: overrides byte-identical to pre-sweep (#347)" "$(cat "$OVR")" "$PRE347" +RESTARTS_POST347="$(grep -c '\[systemctl\] restart' "$CL347")" +assert_eq "aborted live sweep: the miner saw a final restart on the restored config (#347)" \ + "$([ "$RESTARTS_POST347" -gt "$RESTARTS_PRE347" ] && echo y || echo n)" "y" + # #64: --confirm A/B-checks the bench winner against the previous config on the live miner, keeping it # only if it genuinely wins live (else reverting). The bench search picks the winner (prefetch=2); a fake # API then drives the live A/B — a counter returns the winner window first, the previous-config window @@ -5623,6 +5662,45 @@ assert_eq "autotune left prefetch at the current mode (#46)" "$(J "$OVR" '.rando out="$(cd "$TN" && PATH="$STUBS:$PATH" STUB_UNAME_S=Darwin RIGFORGE_HOME="$PWD" bash "$SCRIPT" autotune &1)" assert_rc "autotune rejected on non-Linux" "$?" "1" +# #347: an INTERRUPTED autotune sweep must not persist the mid-sweep prefetch candidate. Each trial +# merges a candidate mode into the overrides and restarts the miner on it; the EXIT trap must restore +# the pre-sweep mode — via the same MERGE the sweep uses, so offline-`tune` knobs survive the abort too. +# The fake API serves the baseline read fast, then flags + blocks on the first candidate window; the +# test kills autotune there and asserts prefetch is back at the pre-sweep mode with the merged knobs +# intact and a post-kill restart. +echo "== black-box: aborted autotune restores the pre-sweep prefetch mode (#347) ==" +cat >"$OVR" <<'EOF' +{ "randomx": { "scratchpad_prefetch_mode": 1 }, "cpu": { "rx": 4, "yield": false } } +EOF +FLAGAT347="$TN/flagat347" +CLAT347="$TN/callsat347.log" +ACTR347="$TN/actr347" +rm -f "$FLAGAT347" "$ACTR347" +: >"$CLAT347" +(cd "$TN" && RIGFORGE_HOME="$PWD" PATH="$STUBS:$PATH" CALL_LOG="$CLAT347" LOGROTATE_DIR="$TN/logrotate" \ + FLAGAT347="$FLAGAT347" ACTR347="$ACTR347" \ + API_CMD='c=$(cat "$ACTR347" 2>/dev/null||echo 0);c=$((c+1));echo "$c">"$ACTR347";if [ "$c" -ge 2 ]; then touch "$FLAGAT347"; sleep 60; fi; echo 1200' \ + AUTOTUNE_WARMUP=0 AUTOTUNE_SAMPLES=1 AUTOTUNE_INTERVAL=0 \ + exec bash "$SCRIPT" autotune "$TN/abortat347.out" 2>&1) & +PIDAT347=$! +for _ in $(seq 1 100); do + [ -f "$FLAGAT347" ] && break + sleep 0.1 +done +assert_eq "autotune abort test reached a candidate window (#347)" "$([ -f "$FLAGAT347" ] && echo y || echo n)" "y" +assert_eq "mid-sweep: overrides hold the candidate mode, not the baseline (#347)" \ + "$(J "$OVR" '.randomx.scratchpad_prefetch_mode')" "0" +RESTARTS_PREAT347="$(grep -c '\[systemctl\] restart' "$CLAT347")" +kill "$PIDAT347" 2>/dev/null +wait "$PIDAT347" 2>/dev/null +assert_contains "aborted autotune announces the restore (#347)" "$(cat "$TN/abortat347.out")" "sweep interrupted — restoring prefetch_mode=1" +assert_eq "aborted autotune: prefetch back at the pre-sweep mode (#347)" "$(J "$OVR" '.randomx.scratchpad_prefetch_mode')" "1" +assert_eq "aborted autotune: merged threads survive the abort restore (#347)" "$(J "$OVR" '.cpu.rx')" "4" +assert_eq "aborted autotune: merged yield survives the abort restore (#347)" "$(J "$OVR" '.cpu.yield')" "false" +RESTARTS_POSTAT347="$(grep -c '\[systemctl\] restart' "$CLAT347")" +assert_eq "aborted autotune: the miner saw a final restart on the restored mode (#347)" \ + "$([ "$RESTARTS_POSTAT347" -gt "$RESTARTS_PREAT347" ] && echo y || echo n)" "y" + # #6: grid search exhaustively tries every knob combination (TUNE_SEARCH=grid). Reset the base + a # prefetch-rewarding fake; only prefetch is active (4 values), so grid measures 4 combos and finds 2. echo "== black-box: tune grid search (#6) =="