fix(tune): an interrupted live sweep restores the pre-sweep config (#347)

CHANGELOG.md

@@ -21,6 +21,16 @@ All notable changes to RigForge are documented here. The format is based on
 
 ### Fixed
 
+- **An interrupted live sweep no longer persists the mid-sweep candidate (#347).** `tune --live`
+  (and `tune --now --long`) applies every candidate straight into `tune-overrides.json` and restarts
+  the miner on it, but the cleanup trap only restored the temp dir and the service — a Ctrl+C, a
+  dropped SSH session, or an error mid-sweep left an arbitrary half-measured candidate saved and
+  running, silently costing hashrate until the next re-tune. The trap now restores the pre-sweep
+  overrides and restarts the miner on them, through the same restore step the `--confirm` revert leg
+  uses. `autotune` (and `tune --now`) had the same gap with its per-trial prefetch merges; its trap
+  restores the pre-sweep mode via the same merge the sweep uses, so knobs pinned by an offline
+  `tune` survive the abort too.
+
 - **`status` no longer aborts on a healthy rig (#341).** XMRig's `/2/summary` reports
   `"hugepages"` as an array (`[pages, total]`); the status renderer fed it to jq's `@tsv`,
   which rejects arrays — so the operator's first command printed an abort line instead of

docs/operations.md

@@ -160,6 +160,11 @@ pass that sweeps every knob live (threads, yield, 1G-pages, not just prefetch),
 a BIOS, RAM, or cooling change. `tune --now` is the friendly name for the `autotune` engine; the
 standalone `autotune` verb still works and is what the scheduled timer below runs.
 
+Live sweeps apply each candidate to the running miner as they measure it. If a sweep is interrupted
+(Ctrl+C, a dropped SSH session) or errors out mid-run, RigForge restores the tuning that was in place
+before the sweep and restarts the miner on it — a half-measured candidate is never left running or
+saved to `tune-overrides.json`.
+
 For a hands-off schedule, set `autotune` in `config.json` to a target and re-run `setup`. RigForge
 installs a systemd timer that periodically optimizes the prefetch mode against your live miner:
 

rigforge.sh

@@ -2166,16 +2166,35 @@ S_y=""
 S_t=""
 S_g=""
 S_pr=""
-S_hj=""             # cpu.huge-pages-jit (off by default; swept only if TUNE_HPJIT lists >1 value)
-S_cq=""             # randomx.cache_qos  (off by default; swept only if TUNE_CACHEQOS lists >1 value)
-S_wr=""             # randomx.wrmsr      (off by default; swept only if TUNE_WRMSR lists >1 value) (#66)
-HILL_BEST=""        # set by _hillclimb (its result is returned via this global, not stdout — see below)
-HP_CAP_THREADS=""   # #65: max thread count whose 2MB-page need fits the reservation (empty = check off)
-_TUNE_SVC_STOPPED=0 # set by tune() when it stops the live service for a --bench run (#2)
+S_hj=""                # cpu.huge-pages-jit (off by default; swept only if TUNE_HPJIT lists >1 value)
+S_cq=""                # randomx.cache_qos  (off by default; swept only if TUNE_CACHEQOS lists >1 value)
+S_wr=""                # randomx.wrmsr      (off by default; swept only if TUNE_WRMSR lists >1 value) (#66)
+HILL_BEST=""           # set by _hillclimb (its result is returned via this global, not stdout — see below)
+HP_CAP_THREADS=""      # #65: max thread count whose 2MB-page need fits the reservation (empty = check off)
+_TUNE_SVC_STOPPED=0    # set by tune() when it stops the live service for a --bench run (#2)
+_TUNE_LIVE_DIRTY=0     # #347: a live sweep is mid-flight — the overrides file holds a candidate, not a decision
+_TUNE_PRE_OVERRIDES="" # #347: the overrides content from before the live sweep ("" = no file existed)
+
+# Put a previous overrides snapshot back — write it (or remove the file when none existed) and re-apply
+# it to the running miner. Shared by the A/B confirm's revert leg (#64) and the aborted-live-sweep trap
+# (#347), so the abort path restores exactly the way a completed run does.
+_restore_overrides() { # <previous_overrides_json>
+    if [ -n "$1" ]; then printf '%s\n' "$1" | sudo tee "$TUNE_OVERRIDES" >/dev/null; else sudo rm -f "$TUNE_OVERRIDES"; fi
+    _apply_runtime >/dev/null 2>&1 || true
+}
 
 # Restart the miner that a --bench run stopped, and clean the temp dir. Installed as an EXIT trap by
 # tune() so the service comes back even if the run errors or is interrupted.
 _tune_bench_cleanup() {
+    # #347: an aborted LIVE sweep would otherwise persist the mid-sweep candidate — _measure_live applies
+    # each one straight into tune-overrides.json and restarts the miner on it. Restore the pre-sweep
+    # overrides FIRST: the restore itself restarts the service, and any restart must re-read the restored
+    # file, never the candidate. tune() clears the flag once the winner deliberately lands in the file.
+    if [ "${_TUNE_LIVE_DIRTY:-0}" = 1 ]; then
+        _TUNE_LIVE_DIRTY=0
+        log "Live sweep interrupted — restoring the pre-tune overrides and restarting the miner on them."
+        _restore_overrides "$_TUNE_PRE_OVERRIDES"
+    fi
     [ -n "${TUNE_TMP:-}" ] && rm -rf "$TUNE_TMP" 2>/dev/null
     if [ "${_TUNE_SVC_STOPPED:-0}" = 1 ]; then
         _TUNE_SVC_STOPPED=0
@@ -3022,6 +3041,11 @@ tune() {
     : >"$RESULTS_FILE"
 
     if [ "$TUNE_MODE" = live ]; then
+        # #347: from here every candidate lands in tune-overrides.json and restarts the miner on it
+        # (_measure_live). Mark the file dirty so the EXIT trap above restores the pre-sweep overrides
+        # if the sweep doesn't reach its deliberate winner write below.
+        _TUNE_PRE_OVERRIDES="$pre_overrides"
+        _TUNE_LIVE_DIRTY=1
         log "Auto-tuning LIVE against the running miner (warmup ${TUNE_LIVE_WARMUP:-60}s, ${TUNE_LIVE_SAMPLES:-3} samples) — search=$TUNE_SEARCH, knobs={$ACTIVE_KNOBS}."
     else
         log "Auto-tuning via 'xmrig --bench=$TUNE_BENCH' (median of $TUNE_ITERS) — search=$TUNE_SEARCH, knobs={$ACTIVE_KNOBS}, min-delta=$TUNE_MIN_DELTA."
@@ -3090,6 +3114,7 @@ tune() {
     case " $ACTIVE_KNOBS " in *" cacheqos "*) ovr=$(printf '%s' "$ovr" | jq --argjson cq "$G_cq" '.randomx.cache_qos = $cq') ;; esac
     case " $ACTIVE_KNOBS " in *" wrmsr "*) ovr=$(printf '%s' "$ovr" | jq --argjson wr "$G_wr" '.randomx.wrmsr = $wr') ;; esac
     printf '%s\n' "$ovr" >"$TUNE_TMP/ovr.json" && sudo cp "$TUNE_TMP/ovr.json" "$TUNE_OVERRIDES"
+    _TUNE_LIVE_DIRTY=0 # #347: the winner — a completed decision — now owns the file; the abort-restore stands down
 
     # Assemble the full search log: the winner, the search parameters, and every measured candidate.
     jq -s --argjson p "$G_p" --argjson y "$G_y" --arg t "$G_t" --argjson g "$G_g" --argjson pr "$G_pr" \
@@ -3144,8 +3169,7 @@ _tune_confirm_live() { # <winner_overrides_json> <previous_overrides_json>
     sleep "$warm"
     win_hr=$(_sample_api_median "$n" "$iv")
     [ -n "$win_hr" ] || win_hr=0
-    if [ -n "$pre_ovr" ]; then printf '%s\n' "$pre_ovr" | sudo tee "$TUNE_OVERRIDES" >/dev/null; else sudo rm -f "$TUNE_OVERRIDES"; fi
-    _apply_runtime >/dev/null 2>&1 || true
+    _restore_overrides "$pre_ovr"
     sleep "$warm"
     base_hr=$(_sample_api_median "$n" "$iv")
     [ -n "$base_hr" ] || base_hr=0
@@ -3172,6 +3196,26 @@ _autotune_set_prefetch() { # <overrides_file> <mode>
     rm -f "$tmp"
 }
 
+# #347: autotune's abort protection. Each trial merges a candidate prefetch mode into the overrides file
+# and restarts the miner on it — an abort mid-sweep would otherwise persist that candidate. The EXIT trap
+# restores the pre-sweep mode through the SAME settle step the clean path ends with, so the two cannot
+# diverge. The merge-based restore (_autotune_set_prefetch) preserves any offline-`tune` knobs, exactly
+# like the sweep's own writes.
+_AUTOTUNE_DIRTY=0
+_AUTOTUNE_PRE_MODE=""
+_AUTOTUNE_OVR_FILE=""
+_autotune_settle() { # <mode> — merge the mode into the overrides and leave the miner running on it
+    _autotune_set_prefetch "$_AUTOTUNE_OVR_FILE" "$1"
+    _apply_runtime >/dev/null 2>&1 || true
+}
+_autotune_cleanup() {
+    if [ "${_AUTOTUNE_DIRTY:-0}" = 1 ]; then
+        _AUTOTUNE_DIRTY=0
+        log "autotune: sweep interrupted — restoring prefetch_mode=$_AUTOTUNE_PRE_MODE."
+        _autotune_settle "$_AUTOTUNE_PRE_MODE"
+    fi
+}
+
 # Sample the live miner for one candidate: median H/s over the window and, for the efficiency target,
 # the average package watts over that SAME window — prints "hr<TAB>watts" (watts empty for perf or when
 # no power source). RAPL brackets the sampling window; a TUNE_POWER_CMD override is polled once per
@@ -3274,6 +3318,12 @@ autotune() {
     last_applied="$cur"
     log "autotune: optimizing for $(_autotune_desc "$target"); live-sweeping prefetch modes [$modes] against the running miner; baseline mode=$cur at $(_autotune_fmt "$target" "$base_hr" "$base_w") (median of $n)."
 
+    # #347: arm the abort-restore before the first candidate lands in the overrides file.
+    _AUTOTUNE_PRE_MODE="$cur"
+    _AUTOTUNE_OVR_FILE="$overrides"
+    _AUTOTUNE_DIRTY=1
+    trap '_autotune_cleanup' EXIT
+
     # Try every OTHER mode once, live; track the running best by the target's score.
     for m in $modes; do
         [ "$m" = "$cur" ] && continue
@@ -3302,9 +3352,9 @@ autotune() {
     fi
     # Leave the chosen mode running (the sweep may have ended on a different one).
     if [ "$last_applied" != "$best_mode" ]; then
-        _autotune_set_prefetch "$overrides" "$best_mode"
-        _apply_runtime >/dev/null 2>&1 || true
+        _autotune_settle "$best_mode"
     fi
+    _AUTOTUNE_DIRTY=0 # #347: a deliberate final mode is in place; the abort-restore stands down
 }
 
 # Read the current total hashrate from the worker's HTTP API (empty if unreachable). Overridable for

tests/run.sh

@@ -4976,6 +4976,45 @@ out="$(cd "$TN" && PATH="$STUBS:$PATH" STUB_UNAME_S=Darwin RIGFORGE_HOME="$PWD"
 assert_rc "tune --live rejected on non-Linux" "$?" "1"
 assert_contains "tune --live non-Linux message" "$out" "only supported on Linux"
 
+# #347: an INTERRUPTED live sweep must not persist the mid-sweep candidate. _measure_live applies every
+# candidate straight into tune-overrides.json and restarts the miner on it; the EXIT trap must restore
+# the pre-sweep overrides and restart the miner on THEM. The fake API touches a flag when a candidate
+# window is in flight and then blocks — the test kills tune there (mid-iteration, candidate in the file)
+# and asserts the file came back byte-identical and the miner saw a post-kill restart. `exec` makes the
+# backgrounded PID the rigforge process itself, so the kill lands on the shell that owns the trap.
+echo "== black-box: aborted tune --live restores the pre-sweep overrides (#347) =="
+cat >"$OVR" <<'EOF'
+{ "randomx": { "scratchpad_prefetch_mode": 3 }, "cpu": { "rx": 4 } }
+EOF
+PRE347="$(cat "$OVR")"
+FLAG347="$TN/flag347"
+CL347="$TN/calls347.log"
+rm -f "$FLAG347"
+: >"$CL347"
+(cd "$TN" && RIGFORGE_HOME="$PWD" PATH="$STUBS:$PATH" CALL_LOG="$CL347" LOGROTATE_DIR="$TN/logrotate" \
+    FLAG347="$FLAG347" API_CMD='touch "$FLAG347"; sleep 60; echo 1500' \
+    TUNE_LIVE_WARMUP=0 TUNE_LIVE_INTERVAL=0 TUNE_LIVE_SAMPLES=1 \
+    TUNE_SEEDS=auto TUNE_PREFETCH_MODES="0 1" TUNE_YIELDS=false TUNE_THREADS=-1 TUNE_MAX_ROUNDS=1 \
+    exec bash "$SCRIPT" tune --live </dev/null >"$TN/abort347.out" 2>&1) &
+PID347=$!
+for _ in $(seq 1 100); do
+    [ -f "$FLAG347" ] && break
+    sleep 0.1
+done
+assert_eq "abort test reached a live candidate window (#347)" "$([ -f "$FLAG347" ] && echo y || echo n)" "y"
+# The candidate is in the file RIGHT NOW — prove the pre-state was actually clobbered mid-sweep, so the
+# restore assert below can't pass vacuously.
+assert_eq "mid-sweep: overrides hold the candidate, not the pre-state (#347)" \
+    "$([ "$(cat "$OVR")" = "$PRE347" ] && echo same || echo differs)" "differs"
+RESTARTS_PRE347="$(grep -c '\[systemctl\] restart' "$CL347")"
+kill "$PID347" 2>/dev/null
+wait "$PID347" 2>/dev/null
+assert_contains "aborted live sweep announces the restore (#347)" "$(cat "$TN/abort347.out")" "restoring the pre-tune overrides"
+assert_eq "aborted live sweep: overrides byte-identical to pre-sweep (#347)" "$(cat "$OVR")" "$PRE347"
+RESTARTS_POST347="$(grep -c '\[systemctl\] restart' "$CL347")"
+assert_eq "aborted live sweep: the miner saw a final restart on the restored config (#347)" \
+    "$([ "$RESTARTS_POST347" -gt "$RESTARTS_PRE347" ] && echo y || echo n)" "y"
+
 # #64: --confirm A/B-checks the bench winner against the previous config on the live miner, keeping it
 # only if it genuinely wins live (else reverting). The bench search picks the winner (prefetch=2); a fake
 # API then drives the live A/B — a counter returns the winner window first, the previous-config window
@@ -5623,6 +5662,45 @@ assert_eq "autotune left prefetch at the current mode (#46)" "$(J "$OVR" '.rando
 out="$(cd "$TN" && PATH="$STUBS:$PATH" STUB_UNAME_S=Darwin RIGFORGE_HOME="$PWD" bash "$SCRIPT" autotune </dev/null 2>&1)"
 assert_rc "autotune rejected on non-Linux" "$?" "1"
 
+# #347: an INTERRUPTED autotune sweep must not persist the mid-sweep prefetch candidate. Each trial
+# merges a candidate mode into the overrides and restarts the miner on it; the EXIT trap must restore
+# the pre-sweep mode — via the same MERGE the sweep uses, so offline-`tune` knobs survive the abort too.
+# The fake API serves the baseline read fast, then flags + blocks on the first candidate window; the
+# test kills autotune there and asserts prefetch is back at the pre-sweep mode with the merged knobs
+# intact and a post-kill restart.
+echo "== black-box: aborted autotune restores the pre-sweep prefetch mode (#347) =="
+cat >"$OVR" <<'EOF'
+{ "randomx": { "scratchpad_prefetch_mode": 1 }, "cpu": { "rx": 4, "yield": false } }
+EOF
+FLAGAT347="$TN/flagat347"
+CLAT347="$TN/callsat347.log"
+ACTR347="$TN/actr347"
+rm -f "$FLAGAT347" "$ACTR347"
+: >"$CLAT347"
+(cd "$TN" && RIGFORGE_HOME="$PWD" PATH="$STUBS:$PATH" CALL_LOG="$CLAT347" LOGROTATE_DIR="$TN/logrotate" \
+    FLAGAT347="$FLAGAT347" ACTR347="$ACTR347" \
+    API_CMD='c=$(cat "$ACTR347" 2>/dev/null||echo 0);c=$((c+1));echo "$c">"$ACTR347";if [ "$c" -ge 2 ]; then touch "$FLAGAT347"; sleep 60; fi; echo 1200' \
+    AUTOTUNE_WARMUP=0 AUTOTUNE_SAMPLES=1 AUTOTUNE_INTERVAL=0 \
+    exec bash "$SCRIPT" autotune </dev/null >"$TN/abortat347.out" 2>&1) &
+PIDAT347=$!
+for _ in $(seq 1 100); do
+    [ -f "$FLAGAT347" ] && break
+    sleep 0.1
+done
+assert_eq "autotune abort test reached a candidate window (#347)" "$([ -f "$FLAGAT347" ] && echo y || echo n)" "y"
+assert_eq "mid-sweep: overrides hold the candidate mode, not the baseline (#347)" \
+    "$(J "$OVR" '.randomx.scratchpad_prefetch_mode')" "0"
+RESTARTS_PREAT347="$(grep -c '\[systemctl\] restart' "$CLAT347")"
+kill "$PIDAT347" 2>/dev/null
+wait "$PIDAT347" 2>/dev/null
+assert_contains "aborted autotune announces the restore (#347)" "$(cat "$TN/abortat347.out")" "sweep interrupted — restoring prefetch_mode=1"
+assert_eq "aborted autotune: prefetch back at the pre-sweep mode (#347)" "$(J "$OVR" '.randomx.scratchpad_prefetch_mode')" "1"
+assert_eq "aborted autotune: merged threads survive the abort restore (#347)" "$(J "$OVR" '.cpu.rx')" "4"
+assert_eq "aborted autotune: merged yield survives the abort restore (#347)" "$(J "$OVR" '.cpu.yield')" "false"
+RESTARTS_POSTAT347="$(grep -c '\[systemctl\] restart' "$CLAT347")"
+assert_eq "aborted autotune: the miner saw a final restart on the restored mode (#347)" \
+    "$([ "$RESTARTS_POSTAT347" -gt "$RESTARTS_PREAT347" ] && echo y || echo n)" "y"
+
 # #6: grid search exhaustively tries every knob combination (TUNE_SEARCH=grid). Reset the base + a
 # prefetch-rewarding fake; only prefetch is active (4 values), so grid measures 4 combos and finds 2.
 echo "== black-box: tune grid search (#6) =="