Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 118 additions & 7 deletions crates/cli/src/commands/admin/heal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ use crate::exit_code::ExitCode;
use crate::output::Formatter;
use rc_core::admin::{AdminApi, HealScanMode, HealStartRequest, HealStatus};

const HEAL_STOP_SUCCESS_MESSAGE: &str = "Heal operation stopped successfully";
const HEAL_STOP_STILL_RUNNING_MESSAGE: &str =
"Heal stop request was accepted, but heal status is still in progress";

/// Heal subcommands
#[derive(Subcommand, Debug)]
pub enum HealCommands {
Expand Down Expand Up @@ -73,6 +77,11 @@ struct HealStatusOutput {
healing: bool,
bucket: String,
object: String,
#[serde(skip_serializing_if = "Option::is_none")]
scan_mode: Option<HealScanMode>,
scan_cycle: u64,
heal_queue_length: u64,
heal_active_tasks: u64,
items_scanned: u64,
items_healed: u64,
items_failed: u64,
Expand All @@ -91,6 +100,10 @@ impl From<&HealStatus> for HealStatusOutput {
healing: status.healing,
bucket: status.bucket.clone(),
object: status.object.clone(),
scan_mode: status.scan_mode,
scan_cycle: status.scan_cycle,
heal_queue_length: status.heal_queue_length,
heal_active_tasks: status.heal_active_tasks,
items_scanned: status.items_scanned,
items_healed: status.items_healed,
items_failed: status.items_failed,
Expand All @@ -107,6 +120,10 @@ fn has_heal_status_details(status: &HealStatus) -> bool {
|| !status.heal_id.is_empty()
|| !status.bucket.is_empty()
|| !status.object.is_empty()
|| status.scan_mode.is_some()
|| status.scan_cycle > 0
|| status.heal_queue_length > 0
|| status.heal_active_tasks > 0
|| status.items_scanned > 0
|| status.items_healed > 0
|| status.items_failed > 0
Expand All @@ -125,6 +142,28 @@ struct HealOperationOutput {
status: Option<HealStatusOutput>,
}

fn heal_stop_output(status: &HealStatus) -> (ExitCode, HealOperationOutput) {
let success = !status.healing;
let message = if success {
HEAL_STOP_SUCCESS_MESSAGE
} else {
HEAL_STOP_STILL_RUNNING_MESSAGE
};

(
if success {
ExitCode::Success
} else {
ExitCode::GeneralError
},
HealOperationOutput {
success,
message: message.to_string(),
status: has_heal_status_details(status).then(|| HealStatusOutput::from(status)),
},
)
}

/// Execute a heal subcommand
pub async fn execute(cmd: HealCommands, formatter: &Formatter) -> ExitCode {
match cmd {
Expand Down Expand Up @@ -182,6 +221,19 @@ fn print_heal_status(status: &HealStatus, formatter: &Formatter) {
));
}

if let Some(scan_mode) = status.scan_mode {
formatter.println(&format!(" Scan Mode: {scan_mode}"));
}

if status.scan_cycle > 0 {
formatter.println(&format!(" Scan Cycle: {}", status.scan_cycle));
}

formatter.println(&format!(
" Tasks: {} queued, {} active",
status.heal_queue_length, status.heal_active_tasks
));

formatter.println(&format!(
" Items: {} scanned, {} healed, {} failed",
status.items_scanned, status.items_healed, status.items_failed
Expand Down Expand Up @@ -268,17 +320,27 @@ async fn execute_stop(args: StopArgs, formatter: &Formatter) -> ExitCode {

match client.heal_stop().await {
Ok(()) => {
let status = match client.heal_status().await {
Ok(status) => status,
Err(e) => {
formatter.error(&format!(
"Heal stop request was accepted, but failed to verify heal status: {e}"
));
return ExitCode::GeneralError;
}
};
let (exit_code, output) = heal_stop_output(&status);

if formatter.is_json() {
let output = HealOperationOutput {
success: true,
message: "Heal operation stopped successfully".to_string(),
status: None,
};
formatter.json(&output);
} else if output.success {
formatter.success(&format!("{}.", output.message));
} else {
formatter.success("Heal operation stopped successfully.");
formatter.error(&format!("{}.", output.message));
formatter.println("");
print_heal_status(&status, formatter);
}
ExitCode::Success
exit_code
}
Err(e) => {
formatter.error(&format!("Failed to stop heal operation: {e}"));
Expand Down Expand Up @@ -326,6 +388,10 @@ mod tests {
healing: true,
bucket: "test-bucket".to_string(),
object: "test/object.txt".to_string(),
scan_mode: Some(HealScanMode::Deep),
scan_cycle: 42,
heal_queue_length: 3,
heal_active_tasks: 1,
items_scanned: 1000,
items_healed: 50,
items_failed: 5,
Expand All @@ -348,16 +414,52 @@ mod tests {
.as_object()
.expect("status is object");
assert!(status_value.get("healId").is_some());
assert!(status_value.get("scanMode").is_some());
assert!(status_value.get("healQueueLength").is_some());
assert!(status_value.get("healActiveTasks").is_some());
assert!(status_value.get("itemsScanned").is_some());
}

#[test]
fn test_heal_stop_output_reports_still_running_status() {
let status = HealStatus {
healing: true,
heal_active_tasks: 1,
started: Some("2026-06-15T10:11:18Z".to_string()),
..Default::default()
};

let (exit_code, output) = heal_stop_output(&status);

assert_eq!(exit_code, ExitCode::GeneralError);
assert!(!output.success);
assert_eq!(output.message, HEAL_STOP_STILL_RUNNING_MESSAGE);
assert!(output.status.is_some());
}

#[test]
fn test_heal_stop_output_reports_success_for_idle_status() {
let status = HealStatus::default();

let (exit_code, output) = heal_stop_output(&status);

assert_eq!(exit_code, ExitCode::Success);
assert!(output.success);
assert_eq!(output.message, HEAL_STOP_SUCCESS_MESSAGE);
assert!(output.status.is_none());
}

#[test]
fn test_heal_status_output_from() {
let status = HealStatus {
heal_id: "heal-123".to_string(),
healing: true,
bucket: "test-bucket".to_string(),
object: "test/object.txt".to_string(),
scan_mode: Some(HealScanMode::Deep),
scan_cycle: 42,
heal_queue_length: 3,
heal_active_tasks: 1,
items_scanned: 1000,
items_healed: 50,
items_failed: 5,
Expand All @@ -371,6 +473,10 @@ mod tests {
assert_eq!(output.heal_id, "heal-123");
assert!(output.healing);
assert_eq!(output.bucket, "test-bucket");
assert_eq!(output.scan_mode, Some(HealScanMode::Deep));
assert_eq!(output.scan_cycle, 42);
assert_eq!(output.heal_queue_length, 3);
assert_eq!(output.heal_active_tasks, 1);
assert_eq!(output.items_scanned, 1000);
assert_eq!(output.items_healed, 50);
}
Expand All @@ -388,5 +494,10 @@ mod tests {
started: Some("2024-01-01T10:00:00Z".to_string()),
..Default::default()
}));

assert!(has_heal_status_details(&HealStatus {
heal_active_tasks: 1,
..Default::default()
}));
}
}
20 changes: 20 additions & 0 deletions crates/core/src/admin/cluster.rs
Original file line number Diff line number Diff line change
Expand Up @@ -578,6 +578,22 @@ pub struct HealStatus {
#[serde(default)]
pub object: String,

/// Current scan mode reported by background healing
#[serde(default, skip_serializing_if = "Option::is_none")]
pub scan_mode: Option<HealScanMode>,

/// Background heal scan cycle
#[serde(default)]
pub scan_cycle: u64,

/// Number of queued heal tasks
#[serde(default)]
pub heal_queue_length: u64,

/// Number of active heal tasks
#[serde(default)]
pub heal_active_tasks: u64,

/// Number of items scanned
#[serde(default)]
pub items_scanned: u64,
Expand Down Expand Up @@ -929,6 +945,10 @@ mod tests {
let status = HealStatus::default();
assert!(status.heal_id.is_empty());
assert!(!status.healing);
assert!(status.scan_mode.is_none());
assert_eq!(status.scan_cycle, 0);
assert_eq!(status.heal_queue_length, 0);
assert_eq!(status.heal_active_tasks, 0);
assert_eq!(status.items_scanned, 0);
}

Expand Down
79 changes: 76 additions & 3 deletions crates/s3/src/admin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -392,18 +392,43 @@ struct ServiceAccountInfo {
struct BackgroundHealStatusResponse {
#[serde(default)]
bitrot_start_time: Option<String>,
#[serde(default)]
bitrot_start_cycle: u64,
#[serde(default)]
current_scan_mode: Option<u8>,
#[serde(default)]
heal_queue_length: u64,
#[serde(default)]
heal_active_tasks: u64,
}

impl From<BackgroundHealStatusResponse> for HealStatus {
fn from(response: BackgroundHealStatusResponse) -> Self {
let scan_mode = background_heal_scan_mode(response.current_scan_mode);
let legacy_healing = scan_mode.is_none() && response.bitrot_start_time.is_some();
Self {
healing: response.bitrot_start_time.is_some(),
healing: matches!(scan_mode, Some(HealScanMode::Deep))
|| response.heal_queue_length > 0
|| response.heal_active_tasks > 0
|| legacy_healing,
started: response.bitrot_start_time,
scan_mode,
scan_cycle: response.bitrot_start_cycle,
heal_queue_length: response.heal_queue_length,
heal_active_tasks: response.heal_active_tasks,
..Default::default()
}
}
}

fn background_heal_scan_mode(scan_mode: Option<u8>) -> Option<HealScanMode> {
match scan_mode {
Some(1) => Some(HealScanMode::Normal),
Some(2) => Some(HealScanMode::Deep),
_ => None,
}
}

#[derive(Debug, Serialize)]
struct RustfsHealOptions {
recursive: bool,
Expand Down Expand Up @@ -1198,16 +1223,58 @@ mod tests {
fn test_background_heal_status_response_maps_to_heal_status() {
let status = HealStatus::from(BackgroundHealStatusResponse {
bitrot_start_time: Some("2026-04-19T10:00:00Z".to_string()),
bitrot_start_cycle: 42,
current_scan_mode: Some(2),
heal_queue_length: 3,
heal_active_tasks: 1,
});

assert!(status.healing);
assert_eq!(status.started.as_deref(), Some("2026-04-19T10:00:00Z"));
assert_eq!(status.scan_mode, Some(HealScanMode::Deep));
assert_eq!(status.scan_cycle, 42);
assert_eq!(status.heal_queue_length, 3);
assert_eq!(status.heal_active_tasks, 1);

let idle = HealStatus::from(BackgroundHealStatusResponse {
bitrot_start_time: None,
bitrot_start_cycle: 0,
current_scan_mode: Some(1),
heal_queue_length: 0,
heal_active_tasks: 0,
});
assert!(!idle.healing);
assert_eq!(idle.scan_mode, Some(HealScanMode::Normal));
assert!(idle.started.is_none());

let completed = HealStatus::from(BackgroundHealStatusResponse {
bitrot_start_time: Some("2026-04-19T10:00:00Z".to_string()),
bitrot_start_cycle: 42,
current_scan_mode: Some(1),
heal_queue_length: 0,
heal_active_tasks: 0,
});
assert!(!completed.healing);
assert_eq!(completed.scan_mode, Some(HealScanMode::Normal));
assert_eq!(completed.started.as_deref(), Some("2026-04-19T10:00:00Z"));

let legacy = HealStatus::from(BackgroundHealStatusResponse {
bitrot_start_time: Some("2026-04-19T10:00:00Z".to_string()),
bitrot_start_cycle: 0,
current_scan_mode: None,
heal_queue_length: 0,
heal_active_tasks: 0,
});
assert!(legacy.healing);

let active = HealStatus::from(BackgroundHealStatusResponse {
bitrot_start_time: None,
bitrot_start_cycle: 0,
current_scan_mode: None,
heal_queue_length: 0,
heal_active_tasks: 1,
});
assert!(active.healing);
}

#[test]
Expand All @@ -1222,14 +1289,20 @@ mod tests {

#[tokio::test]
async fn test_heal_status_uses_background_heal_status_endpoint() {
let (endpoint, receiver, handle) =
start_admin_test_server("200 OK", r#"{"bitrotStartTime":"2026-04-19T10:00:00Z"}"#);
let (endpoint, receiver, handle) = start_admin_test_server(
"200 OK",
r#"{"bitrotStartTime":"2026-04-19T10:00:00Z","bitrotStartCycle":42,"currentScanMode":2,"healQueueLength":3,"healActiveTasks":1}"#,
);
let client = admin_client_for_endpoint(&endpoint);

let status = client.heal_status().await.expect("heal status request");

assert!(status.healing);
assert_eq!(status.started.as_deref(), Some("2026-04-19T10:00:00Z"));
assert_eq!(status.scan_mode, Some(HealScanMode::Deep));
assert_eq!(status.scan_cycle, 42);
assert_eq!(status.heal_queue_length, 3);
assert_eq!(status.heal_active_tasks, 1);

let request = receiver.recv().expect("captured request");
assert_eq!(request.method, "POST");
Expand Down
Loading