From fcbf58b346a119017886288c0aa6ea2cf1192256 Mon Sep 17 00:00:00 2001 From: GatewayJ <18332154+GatewayJ@users.noreply.github.com> Date: Sat, 20 Jun 2026 14:59:56 +0800 Subject: [PATCH 1/9] refactor(e2e): model RustFS fault plans --- e2e/src/framework/fault_plan.rs | 372 ++++++++++++++++++++++++++++++++ e2e/src/framework/mod.rs | 1 + e2e/tests/faults.rs | 271 +++++++++++++++-------- 3 files changed, 554 insertions(+), 90 deletions(-) create mode 100644 e2e/src/framework/fault_plan.rs diff --git a/e2e/src/framework/fault_plan.rs b/e2e/src/framework/fault_plan.rs new file mode 100644 index 0000000..cdb5b4a --- /dev/null +++ b/e2e/src/framework/fault_plan.rs @@ -0,0 +1,372 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Result, bail, ensure}; +use std::time::Duration; + +use crate::framework::fault_scenarios::{ + DISK_FULL_SCENARIO, DM_FLAKEY_SCENARIO, FaultBackend, FaultScenario, FaultScenarioSpec, + IO_EIO_SCENARIO, IO_READ_MISTAKE_SCENARIO, NETWORK_PARTITION_ONE_SCENARIO, + POD_KILL_ONE_SCENARIO, WARP_UNDER_CHAOS_SCENARIO, +}; + +pub const DEFAULT_RUSTFS_DATA_VOLUME: &str = "/data/rustfs0"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FaultWorkloadMode { + S3Mixed, + S3MixedWithWarp, +} + +impl FaultWorkloadMode { + pub fn runs_warp(self) -> bool { + matches!(self, Self::S3MixedWithWarp) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FaultKind { + RustfsVolumeIoError, + RustfsVolumeReadMistake, + RustfsVolumeEnospc, + RustfsServerPodKill, + RustfsServerNetworkPartition, + RustfsBlockDeviceFlakey, +} + +impl FaultKind { + pub fn as_str(self) -> &'static str { + match self { + Self::RustfsVolumeIoError => "rustfs_volume_io_error", + Self::RustfsVolumeReadMistake => "rustfs_volume_read_mistake", + Self::RustfsVolumeEnospc => "rustfs_volume_enospc", + Self::RustfsServerPodKill => "rustfs_server_pod_kill", + Self::RustfsServerNetworkPartition => "rustfs_server_network_partition", + Self::RustfsBlockDeviceFlakey => "rustfs_block_device_flakey", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FaultTarget { + RustfsVolume { path: &'static str }, + RustfsServerPod, + RustfsServerPeerNetwork, + DedicatedBlockDevice, +} + +impl FaultTarget { + pub fn summary(self) -> String { + match self { + Self::RustfsVolume { path } => format!("one RustFS volume at {path}"), + Self::RustfsServerPod => "one RustFS server Pod".to_string(), + Self::RustfsServerPeerNetwork => { + "one RustFS server Pod partitioned from its peers".to_string() + } + Self::DedicatedBlockDevice => "one dedicated block-device-backed PV".to_string(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FaultInjection { + pub kind: FaultKind, + pub backend: FaultBackend, + pub target: FaultTarget, + pub percent: u8, + pub duration: Duration, +} + +impl FaultInjection { + pub fn new( + kind: FaultKind, + backend: FaultBackend, + target: FaultTarget, + percent: u8, + duration: Duration, + ) -> Result { + ensure!( + (1..=100).contains(&percent), + "fault percent must be in 1..=100, got {percent}" + ); + ensure!(duration > Duration::ZERO, "fault duration must be positive"); + + Ok(Self { + kind, + backend, + target, + percent, + duration, + }) + } + + pub fn rustfs_volume_path(&self) -> Result<&'static str> { + match self.target { + FaultTarget::RustfsVolume { path } => Ok(path), + other => bail!( + "fault kind {} requires a RustFS volume target, got {:?}", + self.kind.as_str(), + other + ), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FaultPlan { + pub scenario: String, + pub case_name: &'static str, + pub workload_mode: FaultWorkloadMode, + faults: Vec, +} + +impl FaultPlan { + pub fn new( + scenario: impl Into, + case_name: &'static str, + workload_mode: FaultWorkloadMode, + faults: Vec, + ) -> Result { + ensure!( + !faults.is_empty(), + "fault plan must contain at least one fault" + ); + + Ok(Self { + scenario: scenario.into(), + case_name, + workload_mode, + faults, + }) + } + + pub fn from_scenario(scenario: &FaultScenario, spec: &FaultScenarioSpec) -> Result { + ensure!( + scenario.name == spec.scenario, + "fault scenario/spec mismatch: scenario={}, spec={}", + scenario.name, + spec.scenario + ); + + let workload_mode = if spec.backend == FaultBackend::MinioWarpWithChaos { + FaultWorkloadMode::S3MixedWithWarp + } else { + FaultWorkloadMode::S3Mixed + }; + let fault = match scenario.name.as_str() { + IO_EIO_SCENARIO => volume_fault(FaultKind::RustfsVolumeIoError, spec, scenario)?, + POD_KILL_ONE_SCENARIO => FaultInjection::new( + FaultKind::RustfsServerPodKill, + spec.backend, + FaultTarget::RustfsServerPod, + scenario.percent, + scenario.duration, + )?, + NETWORK_PARTITION_ONE_SCENARIO => FaultInjection::new( + FaultKind::RustfsServerNetworkPartition, + spec.backend, + FaultTarget::RustfsServerPeerNetwork, + scenario.percent, + scenario.duration, + )?, + IO_READ_MISTAKE_SCENARIO => { + volume_fault(FaultKind::RustfsVolumeReadMistake, spec, scenario)? + } + DISK_FULL_SCENARIO => volume_fault(FaultKind::RustfsVolumeEnospc, spec, scenario)?, + DM_FLAKEY_SCENARIO => FaultInjection::new( + FaultKind::RustfsBlockDeviceFlakey, + spec.backend, + FaultTarget::DedicatedBlockDevice, + scenario.percent, + scenario.duration, + )?, + WARP_UNDER_CHAOS_SCENARIO => { + volume_fault(FaultKind::RustfsVolumeIoError, spec, scenario)? + } + other => bail!("scenario {other:?} has no fault plan mapping"), + }; + + Self::new( + scenario.name.clone(), + scenario.case_name, + workload_mode, + vec![fault], + ) + } + + pub fn faults(&self) -> &[FaultInjection] { + &self.faults + } + + pub fn required_backends(&self) -> Vec { + let mut backends = Vec::new(); + for fault in &self.faults { + if !backends.contains(&fault.backend) { + backends.push(fault.backend); + } + } + backends + } + + pub fn requires_static_storage(&self) -> bool { + self.faults + .iter() + .any(|fault| fault.backend == FaultBackend::DeviceMapper) + } + + pub fn backend_summary(&self) -> String { + self.required_backends() + .into_iter() + .map(|backend| format!("{backend:?}")) + .collect::>() + .join(" + ") + } + + pub fn target_summary(&self) -> String { + self.faults + .iter() + .map(|fault| fault.target.summary()) + .collect::>() + .join(" + ") + } +} + +fn volume_fault( + kind: FaultKind, + spec: &FaultScenarioSpec, + scenario: &FaultScenario, +) -> Result { + FaultInjection::new( + kind, + spec.backend, + FaultTarget::RustfsVolume { + path: DEFAULT_RUSTFS_DATA_VOLUME, + }, + scenario.percent, + scenario.duration, + ) +} + +#[cfg(test)] +mod tests { + use super::{ + DEFAULT_RUSTFS_DATA_VOLUME, FaultInjection, FaultKind, FaultPlan, FaultTarget, + FaultWorkloadMode, + }; + use crate::framework::{ + fault_config::FaultTestConfig, + fault_scenarios::{ + FaultBackend, FaultScenario, WARP_UNDER_CHAOS_SCENARIO, scenario_catalog, scenario_spec, + }, + }; + use std::time::Duration; + + #[test] + fn scenario_plan_maps_io_eio_to_rustfs_volume_fault() { + let config = FaultTestConfig::for_test("real-cluster", "fast-csi"); + let scenario = FaultScenario::from_config(&config).expect("scenario"); + let spec = scenario_spec(&scenario.name).expect("spec"); + + let plan = FaultPlan::from_scenario(&scenario, spec).expect("plan"); + + assert_eq!(plan.workload_mode, FaultWorkloadMode::S3Mixed); + assert_eq!( + plan.required_backends(), + vec![FaultBackend::ChaosMeshIoChaos] + ); + assert_eq!(plan.faults().len(), 1); + assert_eq!(plan.faults()[0].kind, FaultKind::RustfsVolumeIoError); + assert_eq!( + plan.faults()[0].target, + FaultTarget::RustfsVolume { + path: DEFAULT_RUSTFS_DATA_VOLUME + } + ); + } + + #[test] + fn warp_scenario_keeps_performance_mode_out_of_fault_kind() { + let mut config = FaultTestConfig::for_test("real-cluster", "fast-csi"); + config.scenario = WARP_UNDER_CHAOS_SCENARIO.to_string(); + let scenario = FaultScenario::from_config(&config).expect("scenario"); + let spec = scenario_spec(&scenario.name).expect("spec"); + + let plan = FaultPlan::from_scenario(&scenario, spec).expect("plan"); + + assert!(plan.workload_mode.runs_warp()); + assert_eq!(plan.faults()[0].kind, FaultKind::RustfsVolumeIoError); + assert_eq!( + plan.required_backends(), + vec![FaultBackend::MinioWarpWithChaos] + ); + } + + #[test] + fn every_cataloged_scenario_has_one_current_fault_plan() { + let mut config = FaultTestConfig::for_test("real-cluster", "fast-csi"); + + for spec in scenario_catalog() { + config.scenario = spec.scenario.to_string(); + let scenario = FaultScenario::from_config(&config).expect("scenario"); + let plan = FaultPlan::from_scenario(&scenario, spec).expect("plan"); + + assert_eq!( + plan.faults().len(), + 1, + "{} should remain an independent single-fault scenario", + spec.scenario + ); + } + } + + #[test] + fn plan_contract_allows_multiple_faults_for_future_composition() { + let first = FaultInjection::new( + FaultKind::RustfsVolumeIoError, + FaultBackend::ChaosMeshIoChaos, + FaultTarget::RustfsVolume { + path: DEFAULT_RUSTFS_DATA_VOLUME, + }, + 20, + Duration::from_secs(60), + ) + .expect("first fault"); + let second = FaultInjection::new( + FaultKind::RustfsServerNetworkPartition, + FaultBackend::ChaosMeshNetworkChaos, + FaultTarget::RustfsServerPeerNetwork, + 100, + Duration::from_secs(60), + ) + .expect("second fault"); + + let plan = FaultPlan::new( + "composite", + "fault_composite", + FaultWorkloadMode::S3Mixed, + vec![first, second], + ) + .expect("composite plan"); + + assert_eq!(plan.faults().len(), 2); + assert_eq!( + plan.required_backends(), + vec![ + FaultBackend::ChaosMeshIoChaos, + FaultBackend::ChaosMeshNetworkChaos + ] + ); + assert!(plan.target_summary().contains(" + ")); + } +} diff --git a/e2e/src/framework/mod.rs b/e2e/src/framework/mod.rs index de2f612..c638034 100644 --- a/e2e/src/framework/mod.rs +++ b/e2e/src/framework/mod.rs @@ -22,6 +22,7 @@ pub mod config; pub mod console_client; pub mod deploy; pub mod fault_config; +pub mod fault_plan; pub mod fault_scenarios; pub mod history; pub mod host_faults; diff --git a/e2e/tests/faults.rs b/e2e/tests/faults.rs index 54cf4de..4ecaf3b 100644 --- a/e2e/tests/faults.rs +++ b/e2e/tests/faults.rs @@ -23,10 +23,8 @@ use rustfs_operator_e2e::framework::{ command::CommandSpec, config::ClusterTestConfig, fault_config::FaultTestConfig, - fault_scenarios::{ - self, DISK_FULL_SCENARIO, FaultBackend, FaultIsolation, FaultScenario, - IO_READ_MISTAKE_SCENARIO, - }, + fault_plan::{FaultInjection, FaultKind, FaultPlan}, + fault_scenarios::{self, FaultBackend, FaultIsolation, FaultScenario}, history::OperationOutcome, history::Recorder, host_faults::{self, DmFlakeyGuard, DmFlakeySpec, DmStatusSnapshot}, @@ -44,7 +42,6 @@ use std::time::{Duration, Instant}; use tokio::time::sleep as async_sleep; use uuid::Uuid; -const RUSTFS_DATA_VOLUME: &str = "/data/rustfs0"; const FAULT_TENANT_POD_COUNT: usize = 4; const RUSTFS_POD_STABLE_WINDOW: Duration = Duration::from_secs(60); @@ -54,16 +51,17 @@ async fn fault_selected_scenario() -> Result<()> { let config = FaultTestConfig::from_env()?; let scenario = FaultScenario::from_config(&config)?; let spec = fault_scenarios::scenario_spec(&scenario.name)?; + let plan = FaultPlan::from_scenario(&scenario, spec)?; config.require_destructive_enabled()?; - config.validate_cluster(spec.backend == FaultBackend::DeviceMapper)?; + config.validate_cluster(plan.requires_static_storage())?; eprintln!( "running destructive RustFS fault scenario {} against real Kubernetes context: {}", scenario.name, config.cluster.context ); let collector = ArtifactCollector::new(&config.cluster.artifacts_dir); - let result = run_fault_case(&config, &collector, &scenario).await; + let result = run_fault_case(&config, &collector, &scenario, &plan).await; if let Err(error) = &result { match collector.collect_kubernetes_snapshot(scenario.case_name, &config.cluster) { @@ -87,10 +85,11 @@ async fn run_fault_case( config: &FaultTestConfig, collector: &ArtifactCollector, scenario: &FaultScenario, + plan: &FaultPlan, ) -> Result<()> { let spec = fault_scenarios::scenario_spec(&scenario.name)?; - require_fault_backend(config, spec.backend)?; - cleanup_fault_backend(config, spec.backend)?; + require_fault_backends(config, plan)?; + cleanup_fault_backends(config, plan)?; prepare_fault_fixture(&config.cluster, spec.isolation)?; wait_for_ready_tenant(&config.cluster).await?; @@ -147,7 +146,7 @@ async fn run_fault_case( ) .await?; let pods_before = rustfs_pod_identities(cluster)?; - let mut fault = AppliedFault::apply(config, collector, scenario, spec.backend, &run_id)?; + let mut fault = AppliedFaults::apply(config, collector, scenario, plan, &run_id)?; if let Err(error) = fault.wait_active(cluster.timeout) { collect_fault_artifacts(collector, scenario.case_name, &fault, "wait-active-failed")?; @@ -160,7 +159,7 @@ async fn run_fault_case( return Err(error); } - if spec.backend == FaultBackend::MinioWarpWithChaos { + if plan.workload_mode.runs_warp() { let warp_bucket = warp_bucket_name(&run_id); if let Err(error) = host_faults::run_warp_mixed( config.warp_duration, @@ -260,8 +259,8 @@ async fn run_fault_case( )?; let evidence = FaultEvidence { scenario: scenario.name.clone(), - backend: format!("{:?}", spec.backend), - target: spec.target.to_string(), + backend: plan.backend_summary(), + target: plan.target_summary(), injected: true, active_during_workload: true, recovered: report.tenant_recovered, @@ -290,6 +289,13 @@ async fn run_fault_case( Ok(()) } +fn require_fault_backends(config: &FaultTestConfig, plan: &FaultPlan) -> Result<()> { + for backend in plan.required_backends() { + require_fault_backend(config, backend)?; + } + Ok(()) +} + fn require_fault_backend(config: &FaultTestConfig, backend: FaultBackend) -> Result<()> { let cluster = &config.cluster; match backend { @@ -336,6 +342,13 @@ fn require_dm_flakey_preflight(config: &FaultTestConfig) -> Result<()> { Ok(()) } +fn cleanup_fault_backends(config: &FaultTestConfig, plan: &FaultPlan) -> Result<()> { + for backend in plan.required_backends() { + cleanup_fault_backend(config, backend)?; + } + Ok(()) +} + fn cleanup_fault_backend(config: &FaultTestConfig, backend: FaultBackend) -> Result<()> { match backend { FaultBackend::ChaosMeshIoChaos | FaultBackend::MinioWarpWithChaos => { @@ -375,77 +388,156 @@ enum AppliedFault { DmFlakey(Box), } -impl AppliedFault { +struct AppliedFaults { + items: Vec, +} + +impl AppliedFaults { fn apply( config: &FaultTestConfig, collector: &ArtifactCollector, scenario: &FaultScenario, - backend: FaultBackend, + plan: &FaultPlan, + run_id: &str, + ) -> Result { + ensure!( + !plan.faults().is_empty(), + "fault plan {} did not contain any faults", + plan.scenario + ); + + let total = plan.faults().len(); + let mut items = Vec::with_capacity(total); + for (index, injection) in plan.faults().iter().enumerate() { + let manifest_name = chaos_manifest_artifact_name(total, index, injection); + items.push(AppliedFault::apply_one( + config, + collector, + scenario, + injection, + run_id, + &manifest_name, + )?); + } + + Ok(Self { items }) + } + + fn len(&self) -> usize { + self.items.len() + } + + fn wait_active(&self, timeout: Duration) -> Result<()> { + for fault in &self.items { + fault.wait_active(timeout)?; + } + Ok(()) + } + + fn ensure_active(&self, stage: &str) -> Result<()> { + for fault in &self.items { + fault.ensure_active(stage)?; + } + Ok(()) + } + + fn delete(&mut self, timeout: Duration) -> Result<()> { + for fault in self.items.iter_mut().rev() { + fault.delete(timeout)?; + } + Ok(()) + } + + fn snapshot(&self, stage: &str) -> Result { + ensure!( + self.items.len() == 1, + "single fault snapshot requested for {} applied faults", + self.items.len() + ); + self.items[0].snapshot(stage) + } + + fn snapshots(&self, stage: &str) -> Result> { + self.items + .iter() + .map(|fault| fault.snapshot(stage)) + .collect() + } + + fn recovery_dm_snapshot(&self) -> Option { + self.items + .iter() + .find_map(AppliedFault::recovery_dm_snapshot) + } + + fn chaos_guards(&self) -> Vec<&ChaosGuard> { + self.items + .iter() + .filter_map(AppliedFault::chaos_guard) + .collect() + } +} + +impl AppliedFault { + fn apply_one( + config: &FaultTestConfig, + collector: &ArtifactCollector, + scenario: &FaultScenario, + injection: &FaultInjection, run_id: &str, + manifest_name: &str, ) -> Result { let cluster = &config.cluster; - match backend { - FaultBackend::ChaosMeshIoChaos if scenario.name == DISK_FULL_SCENARIO => { + match injection.kind { + FaultKind::RustfsVolumeEnospc => { let chaos = IoChaosSpec::enospc_on_rustfs_volume( cluster, &config.chaos_namespace, run_id, &scenario.name, - RUSTFS_DATA_VOLUME, - scenario.percent, - scenario.duration, - )?; - collector.write_text( - scenario.case_name, - "chaos-manifest.yaml", - &chaos.manifest(), + injection.rustfs_volume_path()?, + injection.percent, + injection.duration, )?; + collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; Ok(Self::Chaos { guard: Box::new(chaos_mesh::apply_iochaos(cluster, &chaos)?), active_required: true, }) } - FaultBackend::ChaosMeshIoChaos if scenario.name == IO_READ_MISTAKE_SCENARIO => { + FaultKind::RustfsVolumeReadMistake => { let chaos = IoChaosSpec::read_mistake_on_rustfs_volume( cluster, &config.chaos_namespace, run_id, &scenario.name, - RUSTFS_DATA_VOLUME, - scenario.percent, - scenario.duration, - )?; - collector.write_text( - scenario.case_name, - "chaos-manifest.yaml", - &chaos.manifest(), + injection.rustfs_volume_path()?, + injection.percent, + injection.duration, )?; + collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; Ok(Self::Chaos { guard: Box::new(chaos_mesh::apply_iochaos(cluster, &chaos)?), active_required: true, }) } - FaultBackend::ChaosMeshIoChaos => { + FaultKind::RustfsVolumeIoError => { let chaos = IoChaosSpec::eio_on_rustfs_volume( cluster, &config.chaos_namespace, run_id, &scenario.name, - RUSTFS_DATA_VOLUME, - scenario.percent, - scenario.duration, - )?; - collector.write_text( - scenario.case_name, - "chaos-manifest.yaml", - &chaos.manifest(), + injection.rustfs_volume_path()?, + injection.percent, + injection.duration, )?; + collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; Ok(Self::Chaos { guard: Box::new(chaos_mesh::apply_iochaos(cluster, &chaos)?), active_required: true, }) } - FaultBackend::ChaosMeshPodChaos => { + FaultKind::RustfsServerPodKill => { let before_pods = rustfs_pod_identities(cluster)?; let chaos = PodChaosSpec::kill_one_rustfs_pod( cluster, @@ -453,36 +545,28 @@ impl AppliedFault { run_id, &scenario.name, ); - collector.write_text( - scenario.case_name, - "chaos-manifest.yaml", - &chaos.manifest(), - )?; + collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; Ok(Self::PodKill { guard: Box::new(chaos_mesh::apply_podchaos(cluster, &chaos)?), before_pods, config: Box::new(cluster.clone()), }) } - FaultBackend::ChaosMeshNetworkChaos => { + FaultKind::RustfsServerNetworkPartition => { let chaos = NetworkChaosSpec::partition_one_rustfs_pod( cluster, &config.chaos_namespace, run_id, &scenario.name, - scenario.duration, - )?; - collector.write_text( - scenario.case_name, - "chaos-manifest.yaml", - &chaos.manifest(), + injection.duration, )?; + collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; Ok(Self::Chaos { guard: Box::new(chaos_mesh::apply_networkchaos(cluster, &chaos)?), active_required: true, }) } - FaultBackend::DeviceMapper => { + FaultKind::RustfsBlockDeviceFlakey => { let name = config .dm_name .as_deref() @@ -514,27 +598,6 @@ impl AppliedFault { scenario.case_name, )?))) } - FaultBackend::MinioWarpWithChaos => { - let chaos = IoChaosSpec::eio_on_rustfs_volume( - cluster, - &config.chaos_namespace, - run_id, - &scenario.name, - RUSTFS_DATA_VOLUME, - scenario.percent, - scenario.duration, - )?; - collector.write_text( - scenario.case_name, - "chaos-manifest.yaml", - &chaos.manifest(), - )?; - let guard = chaos_mesh::apply_iochaos(cluster, &chaos)?; - Ok(Self::Chaos { - guard: Box::new(guard), - active_required: true, - }) - } } } @@ -616,6 +679,14 @@ impl AppliedFault { } } +fn chaos_manifest_artifact_name(total: usize, index: usize, injection: &FaultInjection) -> String { + if total == 1 { + "chaos-manifest.yaml".to_string() + } else { + format!("chaos-manifest-{index:02}-{}.yaml", injection.kind.as_str()) + } +} + #[derive(Debug, Clone, Serialize)] struct FaultStatusSnapshot { stage: String, @@ -645,34 +716,54 @@ struct FaultEvidence { fn collect_fault_artifacts( collector: &ArtifactCollector, case_name: &str, - fault: &AppliedFault, + fault: &AppliedFaults, suffix: &str, ) -> Result<()> { - let status = fault - .snapshot(suffix) - .and_then(|snapshot| serde_json::to_string_pretty(&snapshot).map_err(Into::into)) - .unwrap_or_else(|error| format!("failed to collect fault status: {error}")); + let status = if fault.len() == 1 { + fault + .snapshot(suffix) + .and_then(|snapshot| serde_json::to_string_pretty(&snapshot).map_err(Into::into)) + } else { + fault + .snapshots(suffix) + .and_then(|snapshots| serde_json::to_string_pretty(&snapshots).map_err(Into::into)) + } + .unwrap_or_else(|error| format!("failed to collect fault status: {error}")); collector.write_text(case_name, &format!("fault-status-{suffix}.json"), &status)?; - if let Some(guard) = fault.chaos_guard() { + let guards = fault.chaos_guards(); + for (index, guard) in guards.iter().enumerate() { let describe = guard .describe() .unwrap_or_else(|error| format!("failed to describe chaos before cleanup: {error}")); - collector.write_text( - case_name, - &format!("chaos-describe-{suffix}.txt"), - &describe, - )?; + let describe_name = + chaos_artifact_name(guards.len(), index, "chaos-describe", suffix, "txt"); + collector.write_text(case_name, &describe_name, &describe)?; let yaml = guard .yaml() .unwrap_or_else(|error| format!("failed to get chaos yaml before cleanup: {error}")); - collector.write_text(case_name, &format!("chaos-{suffix}.yaml"), &yaml)?; + let yaml_name = chaos_artifact_name(guards.len(), index, "chaos", suffix, "yaml"); + collector.write_text(case_name, &yaml_name, &yaml)?; } Ok(()) } +fn chaos_artifact_name( + total: usize, + index: usize, + prefix: &str, + suffix: &str, + extension: &str, +) -> String { + if total == 1 { + format!("{prefix}-{suffix}.{extension}") + } else { + format!("{prefix}-{suffix}-{index:02}.{extension}") + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize)] struct PodIdentity { name: String, From db8c0446f6909eeac3741baec419423a96bcc85f Mon Sep 17 00:00:00 2001 From: GatewayJ <18332154+GatewayJ@users.noreply.github.com> Date: Sat, 20 Jun 2026 15:13:36 +0800 Subject: [PATCH 2/9] fix(e2e): harden composite fault handling --- e2e/src/framework/chaos_mesh.rs | 39 ++++++++++++-- e2e/src/framework/fault_plan.rs | 88 ++++++++++++++++++++++++++----- e2e/tests/faults.rs | 91 +++++++++++++++++++++++++-------- 3 files changed, 182 insertions(+), 36 deletions(-) diff --git a/e2e/src/framework/chaos_mesh.rs b/e2e/src/framework/chaos_mesh.rs index a0c1083..6c9dbd7 100644 --- a/e2e/src/framework/chaos_mesh.rs +++ b/e2e/src/framework/chaos_mesh.rs @@ -204,6 +204,11 @@ impl IoChaosSpec { }) } + pub fn with_name_suffix(mut self, suffix: &str) -> Self { + self.name.push_str(suffix); + self + } + pub fn manifest(&self) -> String { let methods = self .methods @@ -298,6 +303,11 @@ impl PodChaosSpec { } } + pub fn with_name_suffix(mut self, suffix: &str) -> Self { + self.name.push_str(suffix); + self + } + pub fn manifest(&self) -> String { format!( r#"apiVersion: chaos-mesh.org/v1alpha1 @@ -358,6 +368,11 @@ impl NetworkChaosSpec { }) } + pub fn with_name_suffix(mut self, suffix: &str) -> Self { + self.name.push_str(suffix); + self + } + pub fn manifest(&self) -> String { let seconds = self.duration.as_secs(); format!( @@ -472,7 +487,6 @@ fn cleanup_managed_kind(config: &ClusterTestConfig, namespace: &str, kind: &str) } pub fn apply_iochaos(config: &ClusterTestConfig, spec: &IoChaosSpec) -> Result { - cleanup_run_kind(config, &spec.namespace, &spec.run_id, "iochaos")?; Kubectl::new(config) .namespaced(&spec.namespace) .apply_yaml_command(spec.manifest()) @@ -488,7 +502,6 @@ pub fn apply_iochaos(config: &ClusterTestConfig, spec: &IoChaosSpec) -> Result Result { - cleanup_run_kind(config, &spec.namespace, &spec.run_id, "podchaos")?; Kubectl::new(config) .namespaced(&spec.namespace) .apply_yaml_command(spec.manifest()) @@ -507,7 +520,6 @@ pub fn apply_networkchaos( config: &ClusterTestConfig, spec: &NetworkChaosSpec, ) -> Result { - cleanup_run_kind(config, &spec.namespace, &spec.run_id, "networkchaos")?; Kubectl::new(config) .namespaced(&spec.namespace) .apply_yaml_command(spec.manifest()) @@ -699,6 +711,27 @@ mod tests { assert!(!manifest.contains(" - READ")); } + #[test] + fn chaos_name_suffix_keeps_run_label_stable() { + let config = FaultTestConfig::for_test("real-cluster", "fast-csi"); + let spec = IoChaosSpec::eio_on_rustfs_volume( + &config.cluster, + "chaos-mesh", + "run-1234567890", + "io-eio", + "/data/rustfs0", + 20, + Duration::from_secs(60), + ) + .expect("valid io chaos") + .with_name_suffix("-01"); + let manifest = spec.manifest(); + + assert_eq!(spec.name, "rustfs-fault-io-eio-run-12345678-01"); + assert!(manifest.contains("name: rustfs-fault-io-eio-run-12345678-01")); + assert!(manifest.contains("rustfs-fault-test/run-id: \"run-1234567890\"")); + } + #[test] fn iochaos_active_requires_selected_and_injected_not_recovered() { let status = r#"{ diff --git a/e2e/src/framework/fault_plan.rs b/e2e/src/framework/fault_plan.rs index cdb5b4a..4dbef31 100644 --- a/e2e/src/framework/fault_plan.rs +++ b/e2e/src/framework/fault_plan.rs @@ -81,11 +81,11 @@ impl FaultTarget { #[derive(Debug, Clone, PartialEq, Eq)] pub struct FaultInjection { - pub kind: FaultKind, - pub backend: FaultBackend, - pub target: FaultTarget, - pub percent: u8, - pub duration: Duration, + kind: FaultKind, + backend: FaultBackend, + target: FaultTarget, + percent: u8, + duration: Duration, } impl FaultInjection { @@ -96,6 +96,12 @@ impl FaultInjection { percent: u8, duration: Duration, ) -> Result { + ensure!( + fault_kind_accepts_backend(kind, backend), + "fault kind {} cannot run with backend {:?}", + kind.as_str(), + backend + ); ensure!( (1..=100).contains(&percent), "fault percent must be in 1..=100, got {percent}" @@ -111,6 +117,26 @@ impl FaultInjection { }) } + pub fn kind(&self) -> FaultKind { + self.kind + } + + pub fn backend(&self) -> FaultBackend { + self.backend + } + + pub fn target(&self) -> FaultTarget { + self.target + } + + pub fn percent(&self) -> u8 { + self.percent + } + + pub fn duration(&self) -> Duration { + self.duration + } + pub fn rustfs_volume_path(&self) -> Result<&'static str> { match self.target { FaultTarget::RustfsVolume { path } => Ok(path), @@ -123,6 +149,28 @@ impl FaultInjection { } } +fn fault_kind_accepts_backend(kind: FaultKind, backend: FaultBackend) -> bool { + matches!( + (kind, backend), + ( + FaultKind::RustfsVolumeIoError, + FaultBackend::ChaosMeshIoChaos | FaultBackend::MinioWarpWithChaos + ) | ( + FaultKind::RustfsVolumeReadMistake | FaultKind::RustfsVolumeEnospc, + FaultBackend::ChaosMeshIoChaos + ) | ( + FaultKind::RustfsServerPodKill, + FaultBackend::ChaosMeshPodChaos + ) | ( + FaultKind::RustfsServerNetworkPartition, + FaultBackend::ChaosMeshNetworkChaos + ) | ( + FaultKind::RustfsBlockDeviceFlakey, + FaultBackend::DeviceMapper + ) + ) +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct FaultPlan { pub scenario: String, @@ -212,8 +260,9 @@ impl FaultPlan { pub fn required_backends(&self) -> Vec { let mut backends = Vec::new(); for fault in &self.faults { - if !backends.contains(&fault.backend) { - backends.push(fault.backend); + let backend = fault.backend(); + if !backends.contains(&backend) { + backends.push(backend); } } backends @@ -222,7 +271,7 @@ impl FaultPlan { pub fn requires_static_storage(&self) -> bool { self.faults .iter() - .any(|fault| fault.backend == FaultBackend::DeviceMapper) + .any(|fault| fault.backend() == FaultBackend::DeviceMapper) } pub fn backend_summary(&self) -> String { @@ -236,7 +285,7 @@ impl FaultPlan { pub fn target_summary(&self) -> String { self.faults .iter() - .map(|fault| fault.target.summary()) + .map(|fault| fault.target().summary()) .collect::>() .join(" + ") } @@ -286,9 +335,9 @@ mod tests { vec![FaultBackend::ChaosMeshIoChaos] ); assert_eq!(plan.faults().len(), 1); - assert_eq!(plan.faults()[0].kind, FaultKind::RustfsVolumeIoError); + assert_eq!(plan.faults()[0].kind(), FaultKind::RustfsVolumeIoError); assert_eq!( - plan.faults()[0].target, + plan.faults()[0].target(), FaultTarget::RustfsVolume { path: DEFAULT_RUSTFS_DATA_VOLUME } @@ -305,7 +354,7 @@ mod tests { let plan = FaultPlan::from_scenario(&scenario, spec).expect("plan"); assert!(plan.workload_mode.runs_warp()); - assert_eq!(plan.faults()[0].kind, FaultKind::RustfsVolumeIoError); + assert_eq!(plan.faults()[0].kind(), FaultKind::RustfsVolumeIoError); assert_eq!( plan.required_backends(), vec![FaultBackend::MinioWarpWithChaos] @@ -369,4 +418,19 @@ mod tests { ); assert!(plan.target_summary().contains(" + ")); } + + #[test] + fn fault_injection_rejects_backend_kind_mismatch() { + let result = FaultInjection::new( + FaultKind::RustfsVolumeIoError, + FaultBackend::ChaosMeshNetworkChaos, + FaultTarget::RustfsVolume { + path: DEFAULT_RUSTFS_DATA_VOLUME, + }, + 20, + Duration::from_secs(60), + ); + + assert!(result.is_err()); + } } diff --git a/e2e/tests/faults.rs b/e2e/tests/faults.rs index 4ecaf3b..786fab6 100644 --- a/e2e/tests/faults.rs +++ b/e2e/tests/faults.rs @@ -152,7 +152,7 @@ async fn run_fault_case( collect_fault_artifacts(collector, scenario.case_name, &fault, "wait-active-failed")?; return Err(error); } - let active_snapshot = fault.snapshot("active")?; + let active_snapshots = fault.snapshots("active")?; if let Err(error) = ensure_s3_access(&mut port_forward, cluster, &endpoint).await { collect_fault_artifacts(collector, scenario.case_name, &fault, "port-forward-failed")?; @@ -228,7 +228,7 @@ async fn run_fault_case( )?; return Err(error); } - let workload_snapshot = fault.snapshot("after-workload")?; + let workload_snapshots = fault.snapshots("after-workload")?; if let Err(error) = fault.delete(cluster.timeout) { collect_fault_artifacts(collector, scenario.case_name, &fault, "delete-failed")?; @@ -268,8 +268,8 @@ async fn run_fault_case( workload_plan, pods_before, pods_after, - active_snapshot, - workload_snapshot, + active_snapshots, + workload_snapshots, dm_recovery_snapshot: fault.recovery_dm_snapshot(), }; collector.write_text( @@ -410,6 +410,7 @@ impl AppliedFaults { let mut items = Vec::with_capacity(total); for (index, injection) in plan.faults().iter().enumerate() { let manifest_name = chaos_manifest_artifact_name(total, index, injection); + let resource_name_suffix = chaos_resource_name_suffix(total, index); items.push(AppliedFault::apply_one( config, collector, @@ -417,6 +418,7 @@ impl AppliedFaults { injection, run_id, &manifest_name, + &resource_name_suffix, )?); } @@ -486,9 +488,10 @@ impl AppliedFault { injection: &FaultInjection, run_id: &str, manifest_name: &str, + resource_name_suffix: &str, ) -> Result { let cluster = &config.cluster; - match injection.kind { + match injection.kind() { FaultKind::RustfsVolumeEnospc => { let chaos = IoChaosSpec::enospc_on_rustfs_volume( cluster, @@ -496,9 +499,10 @@ impl AppliedFault { run_id, &scenario.name, injection.rustfs_volume_path()?, - injection.percent, - injection.duration, - )?; + injection.percent(), + injection.duration(), + )? + .with_name_suffix(resource_name_suffix); collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; Ok(Self::Chaos { guard: Box::new(chaos_mesh::apply_iochaos(cluster, &chaos)?), @@ -512,9 +516,10 @@ impl AppliedFault { run_id, &scenario.name, injection.rustfs_volume_path()?, - injection.percent, - injection.duration, - )?; + injection.percent(), + injection.duration(), + )? + .with_name_suffix(resource_name_suffix); collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; Ok(Self::Chaos { guard: Box::new(chaos_mesh::apply_iochaos(cluster, &chaos)?), @@ -528,9 +533,10 @@ impl AppliedFault { run_id, &scenario.name, injection.rustfs_volume_path()?, - injection.percent, - injection.duration, - )?; + injection.percent(), + injection.duration(), + )? + .with_name_suffix(resource_name_suffix); collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; Ok(Self::Chaos { guard: Box::new(chaos_mesh::apply_iochaos(cluster, &chaos)?), @@ -544,7 +550,8 @@ impl AppliedFault { &config.chaos_namespace, run_id, &scenario.name, - ); + ) + .with_name_suffix(resource_name_suffix); collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; Ok(Self::PodKill { guard: Box::new(chaos_mesh::apply_podchaos(cluster, &chaos)?), @@ -558,8 +565,9 @@ impl AppliedFault { &config.chaos_namespace, run_id, &scenario.name, - injection.duration, - )?; + injection.duration(), + )? + .with_name_suffix(resource_name_suffix); collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; Ok(Self::Chaos { guard: Box::new(chaos_mesh::apply_networkchaos(cluster, &chaos)?), @@ -683,7 +691,18 @@ fn chaos_manifest_artifact_name(total: usize, index: usize, injection: &FaultInj if total == 1 { "chaos-manifest.yaml".to_string() } else { - format!("chaos-manifest-{index:02}-{}.yaml", injection.kind.as_str()) + format!( + "chaos-manifest-{index:02}-{}.yaml", + injection.kind().as_str() + ) + } +} + +fn chaos_resource_name_suffix(total: usize, index: usize) -> String { + if total == 1 { + String::new() + } else { + format!("-{index:02}") } } @@ -708,8 +727,8 @@ struct FaultEvidence { workload_plan: WorkloadPlan, pods_before: Vec, pods_after: Vec, - active_snapshot: FaultStatusSnapshot, - workload_snapshot: FaultStatusSnapshot, + active_snapshots: Vec, + workload_snapshots: Vec, dm_recovery_snapshot: Option, } @@ -1351,8 +1370,13 @@ fn warp_bucket_name(run_id: &str) -> String { mod tests { use super::{ OutcomeCounts, PodIdentity, PodRuntimeState, WorkloadSummary, bucket_name, - pod_deletion_observed, pod_replacement_observed, stable_pod_fingerprint, warp_bucket_name, + chaos_manifest_artifact_name, chaos_resource_name_suffix, pod_deletion_observed, + pod_replacement_observed, stable_pod_fingerprint, warp_bucket_name, }; + use rustfs_operator_e2e::framework::fault_plan::{ + DEFAULT_RUSTFS_DATA_VOLUME, FaultInjection, FaultKind, FaultTarget, + }; + use rustfs_operator_e2e::framework::fault_scenarios::FaultBackend; use rustfs_operator_e2e::framework::history::OperationOutcome; use rustfs_operator_e2e::framework::s3_workload::WorkloadPlan; @@ -1368,6 +1392,31 @@ mod tests { ); } + #[test] + fn composite_fault_artifacts_and_resource_names_are_indexed() { + let injection = FaultInjection::new( + FaultKind::RustfsVolumeIoError, + FaultBackend::ChaosMeshIoChaos, + FaultTarget::RustfsVolume { + path: DEFAULT_RUSTFS_DATA_VOLUME, + }, + 20, + std::time::Duration::from_secs(60), + ) + .expect("valid fault"); + + assert_eq!( + chaos_manifest_artifact_name(1, 0, &injection), + "chaos-manifest.yaml" + ); + assert_eq!(chaos_resource_name_suffix(1, 0), ""); + assert_eq!( + chaos_manifest_artifact_name(2, 1, &injection), + "chaos-manifest-01-rustfs_volume_io_error.yaml" + ); + assert_eq!(chaos_resource_name_suffix(2, 1), "-01"); + } + #[test] fn workload_summary_counts_disrupted_operations() { let mut summary = WorkloadSummary::new(&WorkloadPlan::seeded(42, 40000, 80)); From bcbfe9622fffe01ade1b807b145309f5c3450266 Mon Sep 17 00:00:00 2001 From: GatewayJ <18332154+GatewayJ@users.noreply.github.com> Date: Sat, 20 Jun 2026 15:40:46 +0800 Subject: [PATCH 3/9] fix: harden fault guards and console fallback --- e2e/scripts/fault-test.sh | 48 ++++++++++++++++++++++++++++----------- src/console/server.rs | 24 ++++++++++++++++---- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/e2e/scripts/fault-test.sh b/e2e/scripts/fault-test.sh index 27e59ef..a8c18e9 100644 --- a/e2e/scripts/fault-test.sh +++ b/e2e/scripts/fault-test.sh @@ -179,14 +179,29 @@ prepare_fault_binary() { echo "fault-test binary ready; pre-existing RustFS Pods remained unchanged for ${BUILD_SETTLE_SECONDS}s" } -require_chaos_ready() { - local deployment_ready daemon_ready - deployment_ready="$(kubectl_ns "$CHAOS_NAMESPACE" get deployment chaos-controller-manager -o json | jq -r ' +chaos_deployment_ready() { + kubectl_ns "$CHAOS_NAMESPACE" get deployment chaos-controller-manager -o json | jq -r ' (.status.readyReplicas // 0) == (.spec.replicas // 0) and (.spec.replicas // 0) > 0 - ')" - daemon_ready="$(kubectl_ns "$CHAOS_NAMESPACE" get daemonset chaos-daemon -o json | jq -r ' + ' +} + +chaos_daemon_ready() { + kubectl_ns "$CHAOS_NAMESPACE" get daemonset chaos-daemon -o json | jq -r ' (.status.numberReady // 0) == (.status.desiredNumberScheduled // 0) and (.status.desiredNumberScheduled // 0) > 0 - ')" + ' +} + +chaos_is_ready() { + local deployment_ready daemon_ready + deployment_ready="$(chaos_deployment_ready 2>/dev/null)" || return 1 + daemon_ready="$(chaos_daemon_ready 2>/dev/null)" || return 1 + [[ "$deployment_ready" == "true" && "$daemon_ready" == "true" ]] +} + +require_chaos_ready() { + local deployment_ready daemon_ready + deployment_ready="$(chaos_deployment_ready)" + daemon_ready="$(chaos_daemon_ready)" [[ "$deployment_ready" == "true" ]] || die "Chaos Mesh controller-manager is not fully Ready" [[ "$daemon_ready" == "true" ]] || die "Chaos Mesh chaos-daemon is not fully Ready" } @@ -320,10 +335,11 @@ capture_fault_logs() { } health_is_safe() { - local baseline_nodes="$1" baseline_tenants="$2" - local current_nodes namespace tenant state - current_nodes="$(kubectl_cluster get nodes -o json 2>/dev/null | jq -r '[.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))] | length' 2>/dev/null || echo 0)" - [[ "$current_nodes" -eq "$baseline_nodes" ]] || return 1 + local baseline_ready_nodes="$1" baseline_tenants="$2" require_chaos="$3" + local current_ready_nodes namespace tenant state + current_ready_nodes="$(kubectl_cluster get nodes -o json 2>/dev/null | jq -r '[.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))] | length' 2>/dev/null || echo 0)" + [[ "$current_ready_nodes" -ge "$baseline_ready_nodes" ]] || return 1 + [[ "$require_chaos" != "true" ]] || chaos_is_ready || return 1 while IFS=$'\t' read -r namespace tenant; do [[ -n "$namespace" ]] || continue @@ -361,6 +377,7 @@ validate_scenario_artifacts() { ] ' "$plan" >/dev/null || die "$scenario workload plan does not match the required profile" jq -e '.injected == true and .active_during_workload == true and .recovered == true' "$evidence" >/dev/null || die "$scenario fault evidence is incomplete" + jq -e '(.active_snapshots | length) > 0 and (.workload_snapshots | length) > 0' "$evidence" >/dev/null || die "$scenario fault evidence snapshots are missing" jq -e --argjson objects "$EXPECTED_OBJECTS" ' .committed_puts == $objects and (.missing_committed_objects | length) == 0 @@ -382,11 +399,16 @@ validate_scenario_artifacts() { run_scenario() { local scenario="$1" run_root="$2" local artifacts="$run_root/$scenario" - local baseline_nodes baseline_tenants test_pid rc current_time health_checks + local baseline_ready_nodes baseline_tenants test_pid rc current_time health_checks require_chaos preflight "$scenario" mkdir -p "$artifacts" - baseline_nodes="$(kubectl_cluster get nodes -o json | jq -r '.items | length')" + baseline_ready_nodes="$(kubectl_cluster get nodes -o json | jq -r '[.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))] | length')" baseline_tenants="$artifacts/baseline-tenants.tsv" + if [[ "$scenario" == "dm-flakey" ]]; then + require_chaos=false + else + require_chaos=true + fi kubectl_cluster get tenants -A -o json | jq -r --arg namespace "$FAULT_NAMESPACE" ' .items[] | select(.metadata.namespace != $namespace) | [.metadata.namespace,.metadata.name] | @tsv ' >"$baseline_tenants" @@ -413,7 +435,7 @@ run_scenario() { while kill -0 "$test_pid" 2>/dev/null; do current_time="$(date -u +%FT%TZ)" health_checks=$((health_checks + 1)) - if health_is_safe "$baseline_nodes" "$baseline_tenants"; then + if health_is_safe "$baseline_ready_nodes" "$baseline_tenants" "$require_chaos"; then echo "$current_time safe=true" >>"$artifacts/health-watch.log" if (( health_checks % 6 == 0 )); then echo "scenario=$scenario running safe=true time=$current_time" diff --git a/src/console/server.rs b/src/console/server.rs index 7ad0b13..8ca81c1 100755 --- a/src/console/server.rs +++ b/src/console/server.rs @@ -14,7 +14,7 @@ use crate::console::{openapi::ApiDoc, routes, state::AppState}; use axum::body::Body; -use axum::http::{HeaderValue, Method, Request, Response, StatusCode, header}; +use axum::http::{HeaderValue, Method, Request, Response, StatusCode, Uri, header}; use axum::{Router, middleware, response::IntoResponse, routing::get}; use k8s_openapi::api::core::v1 as corev1; use kube::{Api, Client, api::ListParams}; @@ -159,14 +159,18 @@ fn static_frontend_service(static_dir: PathBuf) -> StaticFrontendService { let index_path = static_dir.join("index.html"); let static_service = ServeDir::new(static_dir) .append_index_html_on_directories(true) - .fallback(ServeFile::new(index_path)); + .fallback(ServeFile::new(index_path.clone())); - StaticFrontendService { static_service } + StaticFrontendService { + static_service, + index_file: ServeFile::new(index_path), + } } #[derive(Clone)] struct StaticFrontendService { static_service: ServeDir, + index_file: ServeFile, } impl Service> for StaticFrontendService { @@ -185,7 +189,19 @@ impl Service> for StaticFrontendService { } let mut static_service = self.static_service.clone(); - Box::pin(async move { static_service.call(request).await }) + let mut index_file = self.index_file.clone(); + let method = request.method().clone(); + Box::pin(async move { + let response = static_service.call(request).await?; + if response.status() != StatusCode::NOT_FOUND { + return Ok(response); + } + + let mut fallback_request = Request::new(Body::empty()); + *fallback_request.method_mut() = method; + *fallback_request.uri_mut() = Uri::from_static("/"); + index_file.call(fallback_request).await + }) } } From b728221406362565978fd54afab1c347bdc8944e Mon Sep 17 00:00:00 2001 From: GatewayJ <18332154+GatewayJ@users.noreply.github.com> Date: Sat, 20 Jun 2026 16:30:53 +0800 Subject: [PATCH 4/9] fix(e2e): persist recovered fault evidence --- e2e/tests/faults.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/e2e/tests/faults.rs b/e2e/tests/faults.rs index 786fab6..dcf28fe 100644 --- a/e2e/tests/faults.rs +++ b/e2e/tests/faults.rs @@ -239,6 +239,26 @@ async fn run_fault_case( wait_for_stable_rustfs_pods(cluster, RUSTFS_POD_STABLE_WINDOW).await?; let pods_after = rustfs_pod_identities(cluster)?; ensure_s3_access(&mut port_forward, cluster, &endpoint).await?; + let recovered_evidence = FaultEvidence { + scenario: scenario.name.clone(), + backend: plan.backend_summary(), + target: plan.target_summary(), + injected: true, + active_during_workload: true, + recovered: true, + client_disruptions: workload.summary.disrupted(), + workload_plan: workload_plan.clone(), + pods_before: pods_before.clone(), + pods_after: pods_after.clone(), + active_snapshots: active_snapshots.clone(), + workload_snapshots: workload_snapshots.clone(), + dm_recovery_snapshot: fault.recovery_dm_snapshot(), + }; + collector.write_text( + scenario.case_name, + "fault-evidence.json", + &serde_json::to_string_pretty(&recovered_evidence)?, + )?; workload.summary.recommitted_after_recovery = recommit_unconfirmed_objects( &s3, &history, From aad8f1650bb61960b5588b9d01d710b7416b5047 Mon Sep 17 00:00:00 2001 From: GatewayJ <18332154+GatewayJ@users.noreply.github.com> Date: Sat, 20 Jun 2026 22:25:03 +0800 Subject: [PATCH 5/9] test(e2e): improve fault failure artifacts --- e2e/FAULT_TESTING.md | 22 +- e2e/scripts/fault-test.sh | 111 +++++----- e2e/tests/faults.rs | 423 +++++++++++++++++++++++++++++++++++--- 3 files changed, 455 insertions(+), 101 deletions(-) diff --git a/e2e/FAULT_TESTING.md b/e2e/FAULT_TESTING.md index 3440bd1..9afdb76 100644 --- a/e2e/FAULT_TESTING.md +++ b/e2e/FAULT_TESTING.md @@ -40,12 +40,12 @@ annotation: rustfs.com/fault-test-tenant=fault-test-tenant - 当前 context 必须与 `RUSTFS_FAULT_TEST_EXPECTED_CONTEXT` 完全一致,并且不能是 `kind-*`。 - 四个 RustFS 测试 Pod 必须调度到至少四个 Ready 节点。 - 常规场景使用独立动态 StorageClass;`dm-flakey` 使用独立静态 Local PV StorageClass。 -- Make 编排器会监控所有节点和运行前已有的非 fault Tenant;任一异常会撤销 managed Chaos 并停止测试。 +- Make 编排器会监控 Ready 节点数量;常规 Chaos 场景还会监控 Chaos Mesh 健康。其他 Tenant 不参与 preflight、health guard 或通过判定。 - `fault-cleanup` 只删除带正确所有权标记的 namespace 和 Chaos,不删除外部 StorageClass、PV 或主机设备。 - The current context must exactly match `RUSTFS_FAULT_TEST_EXPECTED_CONTEXT` and must not be `kind-*`. - The four RustFS test Pods require at least four Ready schedulable nodes. - Regular scenarios use a dedicated dynamic StorageClass; `dm-flakey` uses a dedicated static Local PV StorageClass. -- The Make runner monitors every node and every pre-existing non-fault Tenant. It removes managed Chaos and stops on degradation. +- The Make runner monitors the Ready node count, and regular Chaos scenarios also monitor Chaos Mesh health. Other Tenants do not participate in preflight, health-guard, or pass/fail decisions. - `fault-cleanup` removes only the owned namespace and managed Chaos. It never removes external StorageClasses, PVs, or host devices. ## 2. Workload Profile / 工作负载 @@ -96,15 +96,15 @@ make -C e2e fault-cleanup | Target | Behavior / 行为 | | --- | --- | | `fault-check` | 单 job Rust fmt/test/clippy 和 Bash 语法检查;不访问集群。 / Single-job Rust fmt, tests, clippy, and Bash syntax; no cluster mutation. | -| `fault-preflight` | 校验 context、CRD、StorageClass、Chaos、节点、namespace 所有权和现有 Tenant。 / Validates context, CRDs, storage, Chaos, nodes, ownership, and existing Tenants. | +| `fault-preflight` | 校验 context、CRD、StorageClass、Chaos、节点和 namespace 所有权。 / Validates context, CRDs, storage, Chaos, nodes, and namespace ownership. | | `fault-run` | 运行一个场景,持续健康守护并验收 artifacts。 / Runs one guarded scenario and validates artifacts. | | `fault-run-regular` | 串行运行六个常规场景,首败停止。 / Runs six regular scenarios serially and stops on first failure. | | `fault-run-dm` | 使用预先准备的静态 PV 和 DM 设备运行 `dm-flakey`。 / Runs `dm-flakey` with pre-provisioned static PVs and DM storage. | | `fault-cleanup` | 安全删除 owned namespace 和 managed Chaos。 / Safely removes the owned namespace and managed Chaos. | -`fault-run*` 会先用单 job、最低主机优先级预编译精确的 `faults` 测试二进制,再等待 60 秒并确认原有 RustFS Pod 的 UID、重启数和 Ready 状态没有变化。故障窗口直接运行该二进制,不再次调用 Cargo。预编译不计入故障窗口;如果编译影响现有 Tenant,runner 会在创建故障 Tenant 前停止。 +`fault-run*` 会先用单 job、最低主机优先级预编译精确的 `faults` 测试二进制。故障窗口直接运行该二进制,不再次调用 Cargo。预编译不计入故障窗口;预编译前后都会重新执行 preflight。 -Before creating a fault Tenant, every `fault-run*` target prebuilds the exact `faults` binary with one job and the lowest host priority. It then verifies for 60 seconds that every pre-existing RustFS Pod keeps the same UID, restart count, and Ready state. The fault window executes that binary directly without invoking Cargo again. Compilation is outside the fault window, and the runner stops if the build disturbs an existing Tenant. +Before creating a fault Tenant, every `fault-run*` target prebuilds the exact `faults` binary with one job and the lowest host priority. The fault window executes that binary directly without invoking Cargo again. Compilation is outside the fault window, and the runner reruns preflight before and after prebuild. ### 3.1 Recommended Flow / 推荐执行顺序 @@ -357,11 +357,14 @@ Each scenario directory contains at least: ```text test.log health-watch.log +run-metadata.json workload-plan.json history.jsonl workload-summary.json +recommit-report.json checker-report.json fault-evidence.json +failure-summary.json / runner-failure-summary.json (failure only) nodes-before.txt / nodes-after.txt tenants-before.txt / tenants-after.txt pods-before.txt / pods-after.txt @@ -371,15 +374,19 @@ Chaos or DM snapshots 通过条件 / Pass criteria: - 测试退出码为 0。 +- `run-metadata.json` 记录 scenario、run id、context、StorageClass、RustFS image 和 workload 参数。 - `fault-evidence.json` 的 `injected`、`active_during_workload`、`recovered` 都为 `true`。 - `workload-plan.json` 精确记录 40,000 对象、80 并发和四档尺寸分布。 +- `recommit-report.json` 的 `attempted == committed` 且 `failed == 0`。 - `checker-report.json` 的 `committed_puts=40000`,并且 missing、hash mismatch、successful corrupted read、LIST warning 均为空。 -- fault Tenant 恢复 Ready;所有原有非 fault Tenant 和节点保持 Ready。 +- fault Tenant 恢复 Ready;Ready 节点数量不下降;常规 Chaos 场景中 Chaos Mesh 保持健康。 - The test exits with zero. +- `run-metadata.json` records the scenario, run id, context, StorageClass, RustFS image, and workload parameters. - `fault-evidence.json` reports `injected`, `active_during_workload`, and `recovered` as `true`. - `workload-plan.json` reports exactly 40,000 objects, concurrency 80, and the four size classes. +- `recommit-report.json` reports `attempted == committed` and `failed == 0`. - `checker-report.json` reports `committed_puts=40000` with no missing object, hash mismatch, successful corrupted read, or LIST warning. -- The fault Tenant recovers Ready while every pre-existing non-fault Tenant and node remains Ready. +- The fault Tenant recovers Ready, the Ready node count does not drop, and Chaos Mesh remains healthy during regular Chaos scenarios. 客户端没有看到错误并不表示故障无效。故障是否生效由 Chaos/DM 后端证据判断;客户端 disruption 单独记录。 @@ -430,7 +437,6 @@ kubectl get namespace rustfs-fault-test | `RUSTFS_FAULT_TEST_SEED` | generated | 固定后可重放相同对象。 / Replays the same objects when set. | | `RUSTFS_FAULT_TEST_USE_CLUSTER_IP` | `false` | 集群节点/Pod 内建议设为 `1`。 / Set to `1` on a node or in-cluster runner. | | `RUSTFS_FAULT_TEST_BUILD_JOBS` | `1` | 预编译并行度;小型控制面保持为 1。 / Prebuild parallelism; keep at 1 on small control planes. | -| `RUSTFS_FAULT_TEST_BUILD_SETTLE_SECONDS` | `60` | 预编译后原有 RustFS Pod 的稳定校验时间。 / Existing-Pod stability check after prebuild. | | `RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS` | `40000` | Make runner 强制验收该值。 / Required object count. | | `RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY` | `80` | Make runner 强制验收该值。 / Required concurrency. | | `RUSTFS_FAULT_TEST_DURATION_SECONDS` | `7200` | 最大故障 TTL。 / Maximum fault TTL. | diff --git a/e2e/scripts/fault-test.sh b/e2e/scripts/fault-test.sh index a8c18e9..45bd4bd 100644 --- a/e2e/scripts/fault-test.sh +++ b/e2e/scripts/fault-test.sh @@ -25,7 +25,6 @@ EXPECTED_OBJECTS=40000 EXPECTED_CONCURRENCY=80 EXPECTED_PAYLOAD_BYTES=20337459200 BUILD_JOBS="${RUSTFS_FAULT_TEST_BUILD_JOBS:-1}" -BUILD_SETTLE_SECONDS="${RUSTFS_FAULT_TEST_BUILD_SETTLE_SECONDS:-60}" FAULT_NAMESPACE="${RUSTFS_FAULT_TEST_NAMESPACE:-rustfs-fault-test}" FAULT_TENANT="${RUSTFS_FAULT_TEST_TENANT:-fault-test-tenant}" @@ -101,49 +100,16 @@ require_namespace_ownership() { [[ "$tenant" == "$FAULT_TENANT" ]] || die "namespace $FAULT_NAMESPACE is not owned by tenant $FAULT_TENANT" } -require_non_fault_tenants_ready() { - local unhealthy - unhealthy="$(kubectl_cluster get tenants -A -o json | jq -r --arg namespace "$FAULT_NAMESPACE" ' - .items[] - | select(.metadata.namespace != $namespace) - | select((.status.currentState // "") != "Ready") - | "\(.metadata.namespace)/\(.metadata.name)=\(.status.currentState // "missing")" - ')" - [[ -z "$unhealthy" ]] || die "non-fault Tenant is not Ready: $unhealthy" -} - -snapshot_non_fault_rustfs_pods() { - kubectl_cluster get pods -A -o json | jq -r --arg namespace "$FAULT_NAMESPACE" ' - .items[] - | select(.metadata.namespace != $namespace) - | select(.metadata.labels["rustfs.tenant"] != null) - | [ - .metadata.namespace, - .metadata.name, - .metadata.uid, - ([.status.containerStatuses[]?.restartCount] | add // 0), - ((.status.phase == "Running") and ([.status.containerStatuses[]?.ready] | all)) - ] - | @tsv - ' | sort -} - prepare_fault_binary() { local scenario="$1" run_root="$2" - local before="$run_root/build-pods-before.tsv" - local current="$run_root/build-pods-current.tsv" - local changes="$run_root/build-pod-changes.diff" local build_messages="$run_root/fault-build.jsonl" - local elapsed=0 interval=10 local -a build_command=( cargo test --manifest-path "$MANIFEST" --test faults --no-run --message-format=json-render-diagnostics ) [[ "$BUILD_JOBS" =~ ^[1-9][0-9]*$ ]] || die "RUSTFS_FAULT_TEST_BUILD_JOBS must be a positive integer" - [[ "$BUILD_SETTLE_SECONDS" =~ ^[0-9]+$ ]] || die "RUSTFS_FAULT_TEST_BUILD_SETTLE_SECONDS must be a non-negative integer" preflight "$scenario" - snapshot_non_fault_rustfs_pods >"$before" echo "preparing fault-test binary with jobs=$BUILD_JOBS and lowest host priority" if command -v ionice >/dev/null 2>&1; then CARGO_BUILD_JOBS="$BUILD_JOBS" nice -n 19 ionice -c3 "${build_command[@]}" \ @@ -163,20 +129,8 @@ prepare_fault_binary() { [[ -x "$FAULT_TEST_BINARY" ]] || die "faults test binary was not produced; see $run_root/fault-build.log" printf '%s\n' "$FAULT_TEST_BINARY" >"$run_root/fault-test-binary.path" - while (( elapsed <= BUILD_SETTLE_SECONDS )); do - snapshot_non_fault_rustfs_pods >"$current" - if ! cmp -s "$before" "$current"; then - diff -u "$before" "$current" >"$changes" || true - die "fault-test build changed a pre-existing RustFS Pod; see $changes" - fi - require_non_fault_tenants_ready - (( elapsed == BUILD_SETTLE_SECONDS )) && break - sleep "$interval" - elapsed=$((elapsed + interval)) - (( elapsed > BUILD_SETTLE_SECONDS )) && elapsed="$BUILD_SETTLE_SECONDS" - done preflight "$scenario" - echo "fault-test binary ready; pre-existing RustFS Pods remained unchanged for ${BUILD_SETTLE_SECONDS}s" + echo "fault-test binary ready" } chaos_deployment_ready() { @@ -254,7 +208,6 @@ preflight() { require_storage_class "$scenario" require_namespace_ownership - require_non_fault_tenants_ready if [[ "$scenario" != "dm-flakey" ]]; then crd="$(scenario_crd "$scenario")" @@ -335,17 +288,11 @@ capture_fault_logs() { } health_is_safe() { - local baseline_ready_nodes="$1" baseline_tenants="$2" require_chaos="$3" - local current_ready_nodes namespace tenant state + local baseline_ready_nodes="$1" require_chaos="$2" + local current_ready_nodes current_ready_nodes="$(kubectl_cluster get nodes -o json 2>/dev/null | jq -r '[.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))] | length' 2>/dev/null || echo 0)" [[ "$current_ready_nodes" -ge "$baseline_ready_nodes" ]] || return 1 [[ "$require_chaos" != "true" ]] || chaos_is_ready || return 1 - - while IFS=$'\t' read -r namespace tenant; do - [[ -n "$namespace" ]] || continue - state="$(kubectl_ns "$namespace" get tenant "$tenant" -o jsonpath='{.status.currentState}' 2>/dev/null || true)" - [[ "$state" == "Ready" ]] || return 1 - done <"$baseline_tenants" return 0 } @@ -355,16 +302,27 @@ find_artifact() { validate_scenario_artifacts() { local scenario="$1" artifacts="$2" run_root="$3" - local plan evidence checker summary seed disruptions recommitted committed + local metadata plan evidence checker summary recommit seed disruptions recommitted committed + metadata="$(find_artifact "$artifacts" run-metadata.json)" plan="$(find_artifact "$artifacts" workload-plan.json)" evidence="$(find_artifact "$artifacts" fault-evidence.json)" checker="$(find_artifact "$artifacts" checker-report.json)" summary="$(find_artifact "$artifacts" workload-summary.json)" + recommit="$(find_artifact "$artifacts" recommit-report.json)" + [[ -f "$metadata" ]] || die "$scenario did not produce run-metadata.json" [[ -f "$plan" ]] || die "$scenario did not produce workload-plan.json" [[ -f "$evidence" ]] || die "$scenario did not produce fault-evidence.json" [[ -f "$checker" ]] || die "$scenario did not produce checker-report.json" [[ -f "$summary" ]] || die "$scenario did not produce workload-summary.json" - + [[ -f "$recommit" ]] || die "$scenario did not produce recommit-report.json" + + jq -e --arg scenario "$scenario" ' + .scenario == $scenario + and (.run_id | length) > 0 + and (.rustfs_image | length) > 0 + and (.storage_class | length) > 0 + and (.context | length) > 0 + ' "$metadata" >/dev/null || die "$scenario run metadata is incomplete" jq -e --argjson objects "$EXPECTED_OBJECTS" --argjson concurrency "$EXPECTED_CONCURRENCY" --argjson payload "$EXPECTED_PAYLOAD_BYTES" ' .object_count == $objects and .concurrency == $concurrency @@ -387,31 +345,53 @@ validate_scenario_artifacts() { and .tenant_recovered == true and .passed == true ' "$checker" >/dev/null || die "$scenario checker verdict failed" + jq -e ' + .attempted == .committed + and .failed == 0 + and (.attempts | length) == .attempted + ' "$recommit" >/dev/null || die "$scenario recovery recommit report contains failed attempts" seed="$(jq -r '.seed' "$plan")" disruptions="$(jq -r '.client_disruptions' "$evidence")" - recommitted="$(jq -r '.recommitted_after_recovery' "$summary")" + recommitted="$(jq -r '.committed' "$recommit")" committed="$(jq -r '.committed_puts' "$checker")" printf '%s\t%s\t0\t%s\t%s\t%s\t0\t0\t0\t0\ttrue\n' \ "$scenario" "$seed" "$disruptions" "$recommitted" "$committed" >>"$run_root/validation-summary.tsv" } +write_runner_failure_summary() { + local scenario="$1" artifacts="$2" rc="$3" + local health_guard_failed=false rust_failure_summary=false + [[ ! -f "$artifacts/health-guard-failed" ]] || health_guard_failed=true + [[ ! -f "$artifacts/failure-summary.json" ]] || rust_failure_summary=true + jq -n \ + --arg scenario "$scenario" \ + --argjson exit_code "$rc" \ + --argjson health_guard_failed "$health_guard_failed" \ + --argjson rust_failure_summary "$rust_failure_summary" \ + --arg test_log "$artifacts/test.log" \ + '{ + scenario: $scenario, + stage: "runner", + exit_code: $exit_code, + health_guard_failed: $health_guard_failed, + rust_failure_summary_present: $rust_failure_summary, + test_log: $test_log + }' >"$artifacts/runner-failure-summary.json" +} + run_scenario() { local scenario="$1" run_root="$2" local artifacts="$run_root/$scenario" - local baseline_ready_nodes baseline_tenants test_pid rc current_time health_checks require_chaos + local baseline_ready_nodes test_pid rc current_time health_checks require_chaos preflight "$scenario" mkdir -p "$artifacts" baseline_ready_nodes="$(kubectl_cluster get nodes -o json | jq -r '[.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))] | length')" - baseline_tenants="$artifacts/baseline-tenants.tsv" if [[ "$scenario" == "dm-flakey" ]]; then require_chaos=false else require_chaos=true fi - kubectl_cluster get tenants -A -o json | jq -r --arg namespace "$FAULT_NAMESPACE" ' - .items[] | select(.metadata.namespace != $namespace) | [.metadata.namespace,.metadata.name] | @tsv - ' >"$baseline_tenants" capture_cluster_snapshot "$artifacts" before echo "starting scenario=$scenario artifacts=$artifacts" @@ -435,7 +415,7 @@ run_scenario() { while kill -0 "$test_pid" 2>/dev/null; do current_time="$(date -u +%FT%TZ)" health_checks=$((health_checks + 1)) - if health_is_safe "$baseline_ready_nodes" "$baseline_tenants" "$require_chaos"; then + if health_is_safe "$baseline_ready_nodes" "$require_chaos"; then echo "$current_time safe=true" >>"$artifacts/health-watch.log" if (( health_checks % 6 == 0 )); then echo "scenario=$scenario running safe=true time=$current_time" @@ -461,6 +441,7 @@ run_scenario() { capture_fault_logs "$artifacts" if [[ "$rc" -ne 0 ]]; then + write_runner_failure_summary "$scenario" "$artifacts" "$rc" cleanup_managed_chaos echo "scenario failed: $scenario rc=$rc log=$artifacts/test.log" >&2 return "$rc" diff --git a/e2e/tests/faults.rs b/e2e/tests/faults.rs index dcf28fe..7641721 100644 --- a/e2e/tests/faults.rs +++ b/e2e/tests/faults.rs @@ -64,6 +64,12 @@ async fn fault_selected_scenario() -> Result<()> { let result = run_fault_case(&config, &collector, &scenario, &plan).await; if let Err(error) = &result { + write_failure_summary_if_absent( + &collector, + scenario.case_name, + FailureSummary::new(&scenario.name, "scenario", "unknown", error.to_string()), + ) + .ok(); match collector.collect_kubernetes_snapshot(scenario.case_name, &config.cluster) { Ok(report) => { eprintln!( @@ -105,6 +111,13 @@ async fn run_fault_case( let bucket = bucket_name(&run_id); let history_path = collector.case_dir(scenario.case_name).join("history.jsonl"); let history = Recorder::create(history_path, &scenario.name, &run_id)?; + collector.write_text( + scenario.case_name, + "run-metadata.json", + &serde_json::to_string_pretty(&RunMetadata::from_case( + config, scenario, plan, &run_id, &bucket, + ))?, + )?; collector.write_text( scenario.case_name, "workload-plan.json", @@ -150,12 +163,32 @@ async fn run_fault_case( if let Err(error) = fault.wait_active(cluster.timeout) { collect_fault_artifacts(collector, scenario.case_name, &fault, "wait-active-failed")?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "wait-active", + "environment_or_fault_backend", + error.to_string(), + ), + )?; return Err(error); } let active_snapshots = fault.snapshots("active")?; if let Err(error) = ensure_s3_access(&mut port_forward, cluster, &endpoint).await { collect_fault_artifacts(collector, scenario.case_name, &fault, "port-forward-failed")?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "s3-access-under-fault", + "environment_or_workload", + error.to_string(), + ), + )?; return Err(error); } @@ -171,6 +204,16 @@ async fn run_fault_case( secret_key, ) { collect_fault_artifacts(collector, scenario.case_name, &fault, "warp-failed")?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "warp-workload", + "workload_or_product", + error.to_string(), + ), + )?; return Err(error); } @@ -181,6 +224,16 @@ async fn run_fault_case( &fault, "post-warp-port-forward-failed", )?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "post-warp-s3-access", + "environment_or_workload", + error.to_string(), + ), + )?; return Err(error); } } @@ -199,6 +252,16 @@ async fn run_fault_case( Ok(workload) => workload, Err(error) => { collect_fault_artifacts(collector, scenario.case_name, &fault, "workload-failed")?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "mixed-workload", + "workload_or_product", + error.to_string(), + ), + )?; return Err(error); } }; @@ -217,6 +280,16 @@ async fn run_fault_case( &fault, "workload-no-fault-evidence", )?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "fault-evidence", + "test_or_environment", + error.to_string(), + ), + )?; return Err(error); } if let Err(error) = fault.ensure_active("after fault workload") { @@ -226,19 +299,75 @@ async fn run_fault_case( &fault, "workload-outlived-fault", )?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "fault-still-active", + "test_or_environment", + error.to_string(), + ), + )?; return Err(error); } let workload_snapshots = fault.snapshots("after-workload")?; if let Err(error) = fault.delete(cluster.timeout) { collect_fault_artifacts(collector, scenario.case_name, &fault, "delete-failed")?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "fault-delete", + "environment_or_fault_backend", + error.to_string(), + ), + )?; return Err(error); } - wait_for_ready_tenant(cluster).await?; - wait_for_stable_rustfs_pods(cluster, RUSTFS_POD_STABLE_WINDOW).await?; + if let Err(error) = wait_for_ready_tenant(cluster).await { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "tenant-recovery", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + if let Err(error) = wait_for_stable_rustfs_pods(cluster, RUSTFS_POD_STABLE_WINDOW).await { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "pod-stability-after-recovery", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } let pods_after = rustfs_pod_identities(cluster)?; - ensure_s3_access(&mut port_forward, cluster, &endpoint).await?; + if let Err(error) = ensure_s3_access(&mut port_forward, cluster, &endpoint).await { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "s3-access-after-recovery", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } let recovered_evidence = FaultEvidence { scenario: scenario.name.clone(), backend: plan.backend_summary(), @@ -259,18 +388,38 @@ async fn run_fault_case( "fault-evidence.json", &serde_json::to_string_pretty(&recovered_evidence)?, )?; - workload.summary.recommitted_after_recovery = recommit_unconfirmed_objects( + let recommit_report = recommit_unconfirmed_objects( &s3, &history, &workload.unconfirmed_puts, workload_plan.concurrency, ) - .await?; + .await; + collector.write_text( + scenario.case_name, + "recommit-report.json", + &serde_json::to_string_pretty(&recommit_report)?, + )?; + workload.summary.recommitted_after_recovery = recommit_report.committed; collector.write_text( scenario.case_name, "workload-summary.json", &serde_json::to_string_pretty(&workload.summary)?, )?; + if recommit_report.has_failures() { + let message = recommit_report.failure_message(); + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "recommit-unconfirmed", + "product_or_environment", + message.clone(), + ), + )?; + bail!("{message}"); + } let report = checker::check_s3_history(&s3, &history, true, workload_plan.concurrency).await?; collector.write_text( scenario.case_name, @@ -297,14 +446,36 @@ async fn run_fault_case( "fault-evidence.json", &serde_json::to_string_pretty(&evidence)?, )?; - ensure!( - report.committed_puts == scenario.object_count, - "fault scenario {} expected {} committed objects after recovery reconciliation, got {}", - scenario.name, - scenario.object_count, - report.committed_puts - ); - report.require_success()?; + if report.committed_puts != scenario.object_count { + let message = format!( + "fault scenario {} expected {} committed objects after recovery reconciliation, got {}", + scenario.name, scenario.object_count, report.committed_puts + ); + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "checker-committed-count", + "product_or_environment", + message.clone(), + ), + )?; + bail!("{message}"); + } + if let Err(error) = report.require_success() { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "checker-verdict", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } Ok(()) } @@ -752,6 +923,112 @@ struct FaultEvidence { dm_recovery_snapshot: Option, } +#[derive(Debug, Clone, Serialize)] +struct RunMetadata { + scenario: String, + case_name: String, + run_id: String, + bucket: String, + backend: String, + target: String, + context: String, + namespace: String, + tenant: String, + storage_class: String, + rustfs_image: String, + artifacts_dir: String, + duration_seconds: u64, + percent: u8, + workload_objects: usize, + workload_concurrency: usize, + request_timeout_seconds: u64, + use_cluster_ip: bool, + require_client_disruption: bool, + chaos_namespace: String, +} + +impl RunMetadata { + fn from_case( + config: &FaultTestConfig, + scenario: &FaultScenario, + plan: &FaultPlan, + run_id: &str, + bucket: &str, + ) -> Self { + Self { + scenario: scenario.name.clone(), + case_name: scenario.case_name.to_string(), + run_id: run_id.to_string(), + bucket: bucket.to_string(), + backend: plan.backend_summary(), + target: plan.target_summary(), + context: config.cluster.context.clone(), + namespace: config.cluster.test_namespace.clone(), + tenant: config.cluster.tenant_name.clone(), + storage_class: config.cluster.storage_class.clone(), + rustfs_image: config.cluster.rustfs_image.clone(), + artifacts_dir: config.cluster.artifacts_dir.display().to_string(), + duration_seconds: scenario.duration.as_secs(), + percent: scenario.percent, + workload_objects: scenario.object_count, + workload_concurrency: config.workload_concurrency, + request_timeout_seconds: config.request_timeout.as_secs(), + use_cluster_ip: config.use_cluster_ip, + require_client_disruption: config.require_client_disruption, + chaos_namespace: config.chaos_namespace.clone(), + } + } +} + +#[derive(Debug, Clone, Serialize)] +struct FailureSummary { + scenario: String, + stage: String, + classification: String, + message: String, +} + +impl FailureSummary { + fn new( + scenario: impl Into, + stage: impl Into, + classification: impl Into, + message: impl Into, + ) -> Self { + Self { + scenario: scenario.into(), + stage: stage.into(), + classification: classification.into(), + message: message.into(), + } + } +} + +fn write_failure_summary( + collector: &ArtifactCollector, + case_name: &str, + summary: FailureSummary, +) -> Result<()> { + collector.write_text( + case_name, + "failure-summary.json", + &serde_json::to_string_pretty(&summary)?, + )?; + Ok(()) +} + +fn write_failure_summary_if_absent( + collector: &ArtifactCollector, + case_name: &str, + summary: FailureSummary, +) -> Result<()> { + let path = collector.case_dir(case_name).join("failure-summary.json"); + if path.exists() { + return Ok(()); + } + write_failure_summary(collector, case_name, summary) +} + fn collect_fault_artifacts( collector: &ArtifactCollector, case_name: &str, @@ -1248,29 +1525,93 @@ async fn recommit_unconfirmed_objects( history: &Recorder, objects: &[ObjectSpec], concurrency: usize, -) -> Result { +) -> RecommitReport { let tasks = objects.iter().cloned().map(|object| { let s3 = s3.clone(); let history = history.clone(); async move { let prepared = object.prepare(); - let outcome = s3.put_object(&prepared, &history).await?; - Ok::<_, anyhow::Error>((object.key, outcome)) + match s3.put_object(&prepared, &history).await { + Ok(outcome) => RecommitAttempt { + key: object.key, + size_bytes: object.size_bytes, + sha256: object.sha256, + outcome, + error: None, + }, + Err(error) => RecommitAttempt { + key: object.key, + size_bytes: object.size_bytes, + sha256: object.sha256, + outcome: OperationOutcome::Unknown, + error: Some(error.to_string()), + }, + } } }); - let results = stream::iter(tasks) + let mut attempts = stream::iter(tasks) .buffer_unordered(concurrency) .collect::>() .await; - for result in results { - let (key, outcome) = result?; - ensure!( - outcome == OperationOutcome::Ok, - "PUT for previously unconfirmed object {} did not commit after recovery: {outcome:?}", - key - ); + attempts.sort_by(|left, right| left.key.cmp(&right.key)); + RecommitReport::from_attempts(attempts) +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +struct RecommitReport { + attempted: usize, + committed: usize, + failed: usize, + attempts: Vec, +} + +impl RecommitReport { + fn from_attempts(attempts: Vec) -> Self { + let committed = attempts + .iter() + .filter(|attempt| attempt.outcome == OperationOutcome::Ok) + .count(); + Self { + attempted: attempts.len(), + committed, + failed: attempts.len() - committed, + attempts, + } + } + + fn has_failures(&self) -> bool { + self.failed > 0 + } + + fn failure_message(&self) -> String { + let sample = self + .attempts + .iter() + .filter(|attempt| attempt.outcome != OperationOutcome::Ok) + .take(5) + .map(|attempt| format!("{}={:?}", attempt.key, attempt.outcome)) + .collect::>() + .join(", "); + format!( + "{} of {} previously unconfirmed PUTs did not commit after recovery{}", + self.failed, + self.attempted, + if sample.is_empty() { + String::new() + } else { + format!("; sample: {sample}") + } + ) } - Ok(objects.len()) +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +struct RecommitAttempt { + key: String, + size_bytes: usize, + sha256: String, + outcome: OperationOutcome, + error: Option, } #[derive(Debug)] @@ -1389,9 +1730,9 @@ fn warp_bucket_name(run_id: &str) -> String { #[cfg(test)] mod tests { use super::{ - OutcomeCounts, PodIdentity, PodRuntimeState, WorkloadSummary, bucket_name, - chaos_manifest_artifact_name, chaos_resource_name_suffix, pod_deletion_observed, - pod_replacement_observed, stable_pod_fingerprint, warp_bucket_name, + OutcomeCounts, PodIdentity, PodRuntimeState, RecommitAttempt, RecommitReport, + WorkloadSummary, bucket_name, chaos_manifest_artifact_name, chaos_resource_name_suffix, + pod_deletion_observed, pod_replacement_observed, stable_pod_fingerprint, warp_bucket_name, }; use rustfs_operator_e2e::framework::fault_plan::{ DEFAULT_RUSTFS_DATA_VOLUME, FaultInjection, FaultKind, FaultTarget, @@ -1472,6 +1813,32 @@ mod tests { assert!(summary.require_fault_evidence(true).is_err()); } + #[test] + fn recommit_report_counts_and_summarizes_failed_attempts() { + let report = RecommitReport::from_attempts(vec![ + RecommitAttempt { + key: "object-a".to_string(), + size_bytes: 4096, + sha256: "sha-a".to_string(), + outcome: OperationOutcome::Ok, + error: None, + }, + RecommitAttempt { + key: "object-b".to_string(), + size_bytes: 4096, + sha256: "sha-b".to_string(), + outcome: OperationOutcome::Failed, + error: Some("service unavailable".to_string()), + }, + ]); + + assert_eq!(report.attempted, 2); + assert_eq!(report.committed, 1); + assert_eq!(report.failed, 1); + assert!(report.has_failures()); + assert!(report.failure_message().contains("object-b=Failed")); + } + #[test] fn pod_replacement_requires_old_uid_removed_and_new_uid_added() { let before = vec![ From c636c94fec8cc729792ab7e7a70056296b8a8937 Mon Sep 17 00:00:00 2001 From: GatewayJ <18332154+GatewayJ@users.noreply.github.com> Date: Sat, 20 Jun 2026 22:37:55 +0800 Subject: [PATCH 6/9] test(e2e): harden fault artifact reporting --- e2e/FAULT_TESTING.md | 44 +++- e2e/scripts/fault-test.sh | 1 + e2e/src/framework/history.rs | 6 +- e2e/src/framework/s3_workload.rs | 33 +-- e2e/tests/faults.rs | 368 +++++++++++++++++++++++++++---- 5 files changed, 377 insertions(+), 75 deletions(-) diff --git a/e2e/FAULT_TESTING.md b/e2e/FAULT_TESTING.md index 9afdb76..2fea43f 100644 --- a/e2e/FAULT_TESTING.md +++ b/e2e/FAULT_TESTING.md @@ -350,25 +350,49 @@ make -C e2e fault-run-dm ## 7. Evidence And Acceptance / 证据与验收 -每个场景目录至少包含: +每个场景目录由 runner 固定创建: -Each scenario directory contains at least: +Each scenario directory is created by the runner with: ```text test.log health-watch.log +nodes-before.txt / nodes-after.txt +tenants-before.txt / tenants-after.txt +pods-before.txt / pods-after.txt +Chaos or DM snapshots +``` + +Rust 测试初始化后会写入: + +After Rust test initialization, it contains: + +```text run-metadata.json workload-plan.json history.jsonl +``` + +workload 执行后会写入: + +After workload execution, it contains: + +```text workload-summary.json +``` + +Successful recovery and reconciliation also contain: + +```text +fault-evidence.json recommit-report.json checker-report.json -fault-evidence.json -failure-summary.json / runner-failure-summary.json (failure only) -nodes-before.txt / nodes-after.txt -tenants-before.txt / tenants-after.txt -pods-before.txt / pods-after.txt -Chaos or DM snapshots +``` + +Failure paths contain the artifacts that were reachable before the failure plus: + +```text +failure-summary.json / runner-failure-summary.json ``` 通过条件 / Pass criteria: @@ -377,14 +401,14 @@ Chaos or DM snapshots - `run-metadata.json` 记录 scenario、run id、context、StorageClass、RustFS image 和 workload 参数。 - `fault-evidence.json` 的 `injected`、`active_during_workload`、`recovered` 都为 `true`。 - `workload-plan.json` 精确记录 40,000 对象、80 并发和四档尺寸分布。 -- `recommit-report.json` 的 `attempted == committed` 且 `failed == 0`。 +- `recommit-report.json` 的 `attempted == committed`、`failed == 0` 且 `harness_errors == 0`。 - `checker-report.json` 的 `committed_puts=40000`,并且 missing、hash mismatch、successful corrupted read、LIST warning 均为空。 - fault Tenant 恢复 Ready;Ready 节点数量不下降;常规 Chaos 场景中 Chaos Mesh 保持健康。 - The test exits with zero. - `run-metadata.json` records the scenario, run id, context, StorageClass, RustFS image, and workload parameters. - `fault-evidence.json` reports `injected`, `active_during_workload`, and `recovered` as `true`. - `workload-plan.json` reports exactly 40,000 objects, concurrency 80, and the four size classes. -- `recommit-report.json` reports `attempted == committed` and `failed == 0`. +- `recommit-report.json` reports `attempted == committed`, `failed == 0`, and `harness_errors == 0`. - `checker-report.json` reports `committed_puts=40000` with no missing object, hash mismatch, successful corrupted read, or LIST warning. - The fault Tenant recovers Ready, the Ready node count does not drop, and Chaos Mesh remains healthy during regular Chaos scenarios. diff --git a/e2e/scripts/fault-test.sh b/e2e/scripts/fault-test.sh index 45bd4bd..bb3cdb3 100644 --- a/e2e/scripts/fault-test.sh +++ b/e2e/scripts/fault-test.sh @@ -348,6 +348,7 @@ validate_scenario_artifacts() { jq -e ' .attempted == .committed and .failed == 0 + and .harness_errors == 0 and (.attempts | length) == .attempted ' "$recommit" >/dev/null || die "$scenario recovery recommit report contains failed attempts" diff --git a/e2e/src/framework/history.rs b/e2e/src/framework/history.rs index 99dc105..bb6da73 100644 --- a/e2e/src/framework/history.rs +++ b/e2e/src/framework/history.rs @@ -129,7 +129,7 @@ impl Recorder { outcome: OperationOutcome, http_status: Option, error: Option, - ) -> Result<()> { + ) -> Result { record.ended_at_ms = now_ms(); record.outcome = outcome; record.http_status = http_status; @@ -139,8 +139,8 @@ impl Recorder { serde_json::to_writer(&mut state.writer, &record)?; state.writer.write_all(b"\n")?; state.writer.flush()?; - state.records.push(record); - Ok(()) + state.records.push(record.clone()); + Ok(record) } pub fn records(&self) -> Vec { diff --git a/e2e/src/framework/s3_workload.rs b/e2e/src/framework/s3_workload.rs index 3e8d0a8..fa1e3fa 100644 --- a/e2e/src/framework/s3_workload.rs +++ b/e2e/src/framework/s3_workload.rs @@ -21,7 +21,7 @@ use sha2::{Digest, Sha256}; use std::time::Duration; use tokio::time::timeout; -use crate::framework::history::{OperationKind, OperationOutcome, Recorder}; +use crate::framework::history::{OperationKind, OperationOutcome, OperationRecord, Recorder}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct ObjectSpec { @@ -228,6 +228,14 @@ impl S3WorkloadClient { object: &PreparedObject, recorder: &Recorder, ) -> Result { + Ok(self.put_object_record(object, recorder).await?.outcome) + } + + pub async fn put_object_record( + &self, + object: &PreparedObject, + recorder: &Recorder, + ) -> Result { let spec = &object.spec; let record = recorder.begin( OperationKind::Put, @@ -248,10 +256,7 @@ impl S3WorkloadClient { .await; match result { - Ok(Ok(_)) => { - recorder.finish(record, OperationOutcome::Ok, Some(200), None)?; - Ok(OperationOutcome::Ok) - } + Ok(Ok(_)) => recorder.finish(record, OperationOutcome::Ok, Some(200), None), Ok(Err(error)) => { let outcome = classify_sdk_error(&error); recorder.finish( @@ -259,18 +264,14 @@ impl S3WorkloadClient { outcome, sdk_error_status(&error), Some(format!("put object failed: {error}")), - )?; - Ok(outcome) - } - Err(_) => { - recorder.finish( - record, - OperationOutcome::Timeout, - None, - Some("put object timed out".to_string()), - )?; - Ok(OperationOutcome::Timeout) + ) } + Err(_) => recorder.finish( + record, + OperationOutcome::Timeout, + None, + Some("put object timed out".to_string()), + ), } } diff --git a/e2e/tests/faults.rs b/e2e/tests/faults.rs index 7641721..525a60b 100644 --- a/e2e/tests/faults.rs +++ b/e2e/tests/faults.rs @@ -25,8 +25,7 @@ use rustfs_operator_e2e::framework::{ fault_config::FaultTestConfig, fault_plan::{FaultInjection, FaultKind, FaultPlan}, fault_scenarios::{self, FaultBackend, FaultIsolation, FaultScenario}, - history::OperationOutcome, - history::Recorder, + history::{OperationOutcome, OperationRecord, Recorder}, host_faults::{self, DmFlakeyGuard, DmFlakeySpec, DmStatusSnapshot}, kube_client, kubectl::Kubectl, @@ -94,12 +93,73 @@ async fn run_fault_case( plan: &FaultPlan, ) -> Result<()> { let spec = fault_scenarios::scenario_spec(&scenario.name)?; - require_fault_backends(config, plan)?; - cleanup_fault_backends(config, plan)?; + if let Err(error) = require_fault_backends(config, plan) { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "fault-backend-preflight", + "environment_or_fault_backend", + error.to_string(), + ), + )?; + return Err(error); + } + if let Err(error) = cleanup_fault_backends(config, plan) { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "fault-backend-pre-cleanup", + "environment_or_fault_backend", + error.to_string(), + ), + )?; + return Err(error); + } - prepare_fault_fixture(&config.cluster, spec.isolation)?; - wait_for_ready_tenant(&config.cluster).await?; - wait_for_stable_rustfs_pods(&config.cluster, RUSTFS_POD_STABLE_WINDOW).await?; + if let Err(error) = prepare_fault_fixture(&config.cluster, spec.isolation) { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "fixture-prepare", + "test_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + if let Err(error) = wait_for_ready_tenant(&config.cluster).await { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "tenant-ready-before-fault", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + if let Err(error) = wait_for_stable_rustfs_pods(&config.cluster, RUSTFS_POD_STABLE_WINDOW).await + { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "pod-stability-before-fault", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } let run_id = format!("run-{}", Uuid::new_v4()); let workload_seed = config.workload_seed.unwrap_or_else(generated_seed); @@ -132,34 +192,148 @@ async fn run_fault_case( ); let cluster = &config.cluster; - let (endpoint, mut port_forward) = s3_access(config)?; - ensure_s3_access(&mut port_forward, cluster, &endpoint).await?; + let (endpoint, mut port_forward) = match s3_access(config) { + Ok(access) => access, + Err(error) => { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "s3-endpoint", + "test_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + }; + if let Err(error) = ensure_s3_access(&mut port_forward, cluster, &endpoint).await { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "initial-s3-access", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } let (access_key, secret_key) = resources::test_credentials(); - let s3 = S3WorkloadClient::new( + let s3 = match S3WorkloadClient::new( &endpoint, &bucket, access_key, secret_key, config.request_timeout, ) - .await?; - let bucket_outcome = s3.create_bucket(&history).await?; - ensure!( - bucket_outcome == OperationOutcome::Ok, - "fault workload bucket creation did not succeed: {bucket_outcome:?}" - ); + .await + { + Ok(client) => client, + Err(error) => { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "s3-client", + "test_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + }; + let bucket_outcome = match s3.create_bucket(&history).await { + Ok(outcome) => outcome, + Err(error) => { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "bucket-create", + "test_harness", + error.to_string(), + ), + )?; + return Err(error); + } + }; + if bucket_outcome != OperationOutcome::Ok { + let message = format!("fault workload bucket creation did not succeed: {bucket_outcome:?}"); + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "bucket-create", + "product_or_environment", + message.clone(), + ), + )?; + bail!("{message}"); + } - let prefilled = prefill_objects( + let prefilled = match prefill_objects( &s3, &history, &run_id, &workload_plan, scenario.prefill_count(), ) - .await?; - let pods_before = rustfs_pod_identities(cluster)?; - let mut fault = AppliedFaults::apply(config, collector, scenario, plan, &run_id)?; + .await + { + Ok(prefilled) => prefilled, + Err(error) => { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "prefill", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + }; + let pods_before = match rustfs_pod_identities(cluster) { + Ok(pods) => pods, + Err(error) => { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "pod-identity-before-fault", + "test_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + }; + let mut fault = match AppliedFaults::apply(config, collector, scenario, plan, &run_id) { + Ok(fault) => fault, + Err(error) => { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "fault-apply", + "environment_or_fault_backend", + error.to_string(), + ), + )?; + return Err(error); + } + }; if let Err(error) = fault.wait_active(cluster.timeout) { collect_fault_artifacts(collector, scenario.case_name, &fault, "wait-active-failed")?; @@ -414,7 +588,7 @@ async fn run_fault_case( FailureSummary::new( &scenario.name, "recommit-unconfirmed", - "product_or_environment", + recommit_report.failure_classification(), message.clone(), ), )?; @@ -1531,21 +1705,11 @@ async fn recommit_unconfirmed_objects( let history = history.clone(); async move { let prepared = object.prepare(); - match s3.put_object(&prepared, &history).await { - Ok(outcome) => RecommitAttempt { - key: object.key, - size_bytes: object.size_bytes, - sha256: object.sha256, - outcome, - error: None, - }, - Err(error) => RecommitAttempt { - key: object.key, - size_bytes: object.size_bytes, - sha256: object.sha256, - outcome: OperationOutcome::Unknown, - error: Some(error.to_string()), - }, + match s3.put_object_record(&prepared, &history).await { + Ok(record) => RecommitAttempt::from_record(object, record), + Err(error) => { + RecommitAttempt::from_harness_error(object, format!("record PUT: {error}")) + } } } }); @@ -1562,6 +1726,7 @@ struct RecommitReport { attempted: usize, committed: usize, failed: usize, + harness_errors: usize, attempts: Vec, } @@ -1569,33 +1734,50 @@ impl RecommitReport { fn from_attempts(attempts: Vec) -> Self { let committed = attempts .iter() - .filter(|attempt| attempt.outcome == OperationOutcome::Ok) + .filter(|attempt| attempt.outcome == Some(OperationOutcome::Ok)) + .count(); + let failed = attempts + .iter() + .filter(|attempt| attempt.is_s3_failure()) + .count(); + let harness_errors = attempts + .iter() + .filter(|attempt| attempt.is_harness_error()) .count(); Self { attempted: attempts.len(), committed, - failed: attempts.len() - committed, + failed, + harness_errors, attempts, } } fn has_failures(&self) -> bool { - self.failed > 0 + self.failed > 0 || self.harness_errors > 0 + } + + fn failure_classification(&self) -> &'static str { + if self.harness_errors > 0 { + "test_harness" + } else { + "product_or_environment" + } } fn failure_message(&self) -> String { let sample = self .attempts .iter() - .filter(|attempt| attempt.outcome != OperationOutcome::Ok) + .filter_map(RecommitAttempt::failure_sample) .take(5) - .map(|attempt| format!("{}={:?}", attempt.key, attempt.outcome)) .collect::>() .join(", "); format!( - "{} of {} previously unconfirmed PUTs did not commit after recovery{}", + "{} of {} previously unconfirmed PUTs did not commit after recovery; harness_errors={}{}", self.failed, self.attempted, + self.harness_errors, if sample.is_empty() { String::new() } else { @@ -1610,8 +1792,67 @@ struct RecommitAttempt { key: String, size_bytes: usize, sha256: String, - outcome: OperationOutcome, + outcome: Option, + http_status: Option, error: Option, + harness_error: Option, +} + +impl RecommitAttempt { + fn from_record(object: ObjectSpec, record: OperationRecord) -> Self { + Self { + key: object.key, + size_bytes: object.size_bytes, + sha256: object.sha256, + outcome: Some(record.outcome), + http_status: record.http_status, + error: record.error, + harness_error: None, + } + } + + fn from_harness_error(object: ObjectSpec, error: String) -> Self { + Self { + key: object.key, + size_bytes: object.size_bytes, + sha256: object.sha256, + outcome: None, + http_status: None, + error: None, + harness_error: Some(error), + } + } + + fn is_s3_failure(&self) -> bool { + matches!( + self.outcome, + Some(OperationOutcome::Failed | OperationOutcome::Timeout | OperationOutcome::Unknown) + ) + } + + fn is_harness_error(&self) -> bool { + self.harness_error.is_some() + } + + fn failure_sample(&self) -> Option { + if let Some(error) = &self.harness_error { + return Some(format!("{}=harness_error({error})", self.key)); + } + let outcome = self.outcome?; + if outcome == OperationOutcome::Ok { + return None; + } + let status = self + .http_status + .map(|status| format!(" status={status}")) + .unwrap_or_default(); + let error = self + .error + .as_ref() + .map(|error| format!(" error={error}")) + .unwrap_or_default(); + Some(format!("{}={outcome:?}{status}{error}", self.key)) + } } #[derive(Debug)] @@ -1820,23 +2061,58 @@ mod tests { key: "object-a".to_string(), size_bytes: 4096, sha256: "sha-a".to_string(), - outcome: OperationOutcome::Ok, + outcome: Some(OperationOutcome::Ok), + http_status: Some(200), error: None, + harness_error: None, }, RecommitAttempt { key: "object-b".to_string(), size_bytes: 4096, sha256: "sha-b".to_string(), - outcome: OperationOutcome::Failed, + outcome: Some(OperationOutcome::Failed), + http_status: Some(503), error: Some("service unavailable".to_string()), + harness_error: None, }, ]); assert_eq!(report.attempted, 2); assert_eq!(report.committed, 1); assert_eq!(report.failed, 1); + assert_eq!(report.harness_errors, 0); assert!(report.has_failures()); - assert!(report.failure_message().contains("object-b=Failed")); + assert!( + report + .failure_message() + .contains("object-b=Failed status=503") + ); + assert_eq!(report.failure_classification(), "product_or_environment"); + } + + #[test] + fn recommit_report_separates_harness_errors_from_s3_failures() { + let report = RecommitReport::from_attempts(vec![RecommitAttempt { + key: "object-a".to_string(), + size_bytes: 4096, + sha256: "sha-a".to_string(), + outcome: None, + http_status: None, + error: None, + harness_error: Some("record PUT: disk full".to_string()), + }]); + + assert_eq!(report.attempted, 1); + assert_eq!(report.committed, 0); + assert_eq!(report.failed, 0); + assert_eq!(report.harness_errors, 1); + assert!(report.has_failures()); + assert_eq!(report.failure_classification(), "test_harness"); + assert!( + report + .failure_message() + .contains("object-a=harness_error(record PUT: disk full)") + ); } #[test] From 3fe11e261d83123c3e436115cb14229ea2b71ae1 Mon Sep 17 00:00:00 2001 From: GatewayJ <18332154+GatewayJ@users.noreply.github.com> Date: Sun, 21 Jun 2026 00:03:01 +0800 Subject: [PATCH 7/9] fix(e2e): harden fault test runtime config --- e2e/FAULT_TESTING.md | 549 ++++++++++++++------------- e2e/Makefile | 3 +- e2e/scripts/fault-test.sh | 212 +++++++++-- e2e/src/framework/chaos_mesh.rs | 7 + e2e/src/framework/fault_config.rs | 248 +++++++++--- e2e/src/framework/fault_plan.rs | 98 ++++- e2e/src/framework/fault_scenarios.rs | 43 ++- e2e/tests/faults.rs | 32 +- 8 files changed, 796 insertions(+), 396 deletions(-) diff --git a/e2e/FAULT_TESTING.md b/e2e/FAULT_TESTING.md index 2fea43f..3e4e4f0 100644 --- a/e2e/FAULT_TESTING.md +++ b/e2e/FAULT_TESTING.md @@ -14,77 +14,37 @@ See the License for the specific language governing permissions and limitations under the License. --> -# RustFS Fault-Test Operations / RustFS 故障测试操作手册 +# RustFS Fault-Test Operations -本手册是 Agent 和开发人员使用 `e2e` package 故障测试工具的唯一操作入口。它说明执行步骤、步骤原因、安全边界、验收证据和清理方式。 +This guide is the operational entry point for running RustFS workload fault +tests from the `e2e` package. -This manual is the single operational entry point for agents and developers using the fault-test tooling in the `e2e` package. Fault-test commands, prerequisites, safety limits, evidence, and cleanup are intentionally kept here instead of duplicated in README files. +## Scope -## 1. Purpose And Safety / 目的与安全边界 +Fault tests run only on a dedicated real Kubernetes or K3s cluster. They are +not Kind tests and are not designed for shared application clusters. The suite +creates and deletes its own namespace, Tenant, PVCs, Pods, Services, StatefulSet, +and Chaos Mesh resources. -故障测试只允许在专用真实 Kubernetes 测试集群执行。测试会创建并删除专用 Tenant、PVC、Pod、Service、StatefulSet 和 Chaos resources。禁止把测试 namespace、Tenant、StorageClass 或 DM 路径指向现有业务资源。 +The fault-test cluster is dedicated to this work, so other Tenants are outside +the test contract. The runner does not use other Tenants as health input, does +not assert their readiness, and does not try to protect unrelated workloads. Do +not point the fault namespace, Tenant, StorageClass, or device-mapper path at +shared or production resources. -Run fault tests only in a dedicated real Kubernetes test cluster. The suite creates and removes a dedicated Tenant, PVCs, Pods, Services, StatefulSets, and Chaos resources. Never point its namespace, Tenant, StorageClass, or DM path at application resources. - -固定测试所有权: +Default owned resources: ```text -namespace: rustfs-fault-test -tenant: fault-test-tenant -manager: app.kubernetes.io/managed-by=rustfs-operator-fault-test -annotation: rustfs.com/fault-test-tenant=fault-test-tenant +namespace: rustfs-fault-test +tenant: fault-test-tenant +manager: app.kubernetes.io/managed-by=rustfs-operator-fault-test ``` -安全规则 / Safety rules: - -- 当前 context 必须与 `RUSTFS_FAULT_TEST_EXPECTED_CONTEXT` 完全一致,并且不能是 `kind-*`。 -- 四个 RustFS 测试 Pod 必须调度到至少四个 Ready 节点。 -- 常规场景使用独立动态 StorageClass;`dm-flakey` 使用独立静态 Local PV StorageClass。 -- Make 编排器会监控 Ready 节点数量;常规 Chaos 场景还会监控 Chaos Mesh 健康。其他 Tenant 不参与 preflight、health guard 或通过判定。 -- `fault-cleanup` 只删除带正确所有权标记的 namespace 和 Chaos,不删除外部 StorageClass、PV 或主机设备。 -- The current context must exactly match `RUSTFS_FAULT_TEST_EXPECTED_CONTEXT` and must not be `kind-*`. -- The four RustFS test Pods require at least four Ready schedulable nodes. -- Regular scenarios use a dedicated dynamic StorageClass; `dm-flakey` uses a dedicated static Local PV StorageClass. -- The Make runner monitors the Ready node count, and regular Chaos scenarios also monitor Chaos Mesh health. Other Tenants do not participate in preflight, health-guard, or pass/fail decisions. -- `fault-cleanup` removes only the owned namespace and managed Chaos. It never removes external StorageClasses, PVs, or host devices. - -## 2. Workload Profile / 工作负载 - -每个场景使用 seed 确定性生成对象内容和尺寸顺序。未设置 `RUSTFS_FAULT_TEST_SEED` 时自动生成 seed;所有重放信息写入 `workload-plan.json` 和 `history.jsonl`。 - -Each scenario deterministically generates object content and size order from a seed. A seed is generated when `RUSTFS_FAULT_TEST_SEED` is unset. Replay information is recorded in `workload-plan.json` and `history.jsonl`. - -| Size | Weight | Objects | -| --- | ---: | ---: | -| 4KiB | 85% | 34,000 | -| 16KiB | 10% | 4,000 | -| 8MiB | 4% | 1,600 | -| 16MiB | 1% | 400 | - -```text -objects: 40,000 -concurrency: 80 -payload/scenario: 20,337,459,200 bytes (~18.94GiB) -PVCs: 4 × 100Gi -maximum fault TTL: 7,200 seconds -``` +## Commands -7,200 秒是故障资源的最大保护时间,不是固定等待时间。正常测试在 workload 完成后立即恢复故障。较长 TTL 防止 40,000 对象 workload 在完成前超过 Chaos duration。 - -The 7,200-second duration is a maximum fault-resource safety window, not a fixed wait. Successful runs recover immediately after the workload. The larger window prevents the 40,000-object workload from outliving Chaos. - -Tenant `Ready` 之后、注入故障之前,以及故障恢复之后,测试都会等待四个 RustFS Pod 连续 60 秒保持 `Running/Ready`,且 Pod UID 和容器重启数不变。这个稳定窗口避免把启动期 DNS 或 Pod 重启抖动误判为故障注入结果。 - -After Tenant `Ready`, both before injection and after recovery, the test requires all four RustFS Pods to remain `Running/Ready` for 60 seconds with unchanged Pod UIDs and container restart counts. This stability window prevents startup DNS or restart churn from being misclassified as a fault-injection result. - -## 3. Package Commands / Package 命令 - -所有公共入口都位于 `e2e/Makefile`。从仓库根目录执行: - -All public entry points are in `e2e/Makefile`. Run them from the repository root: +Run all commands from the repository root. ```bash -make -C e2e help make -C e2e fault-check make -C e2e fault-preflight SCENARIO=io-eio make -C e2e fault-run SCENARIO=io-eio @@ -93,74 +53,101 @@ make -C e2e fault-run-dm make -C e2e fault-cleanup ``` -| Target | Behavior / 行为 | -| --- | --- | -| `fault-check` | 单 job Rust fmt/test/clippy 和 Bash 语法检查;不访问集群。 / Single-job Rust fmt, tests, clippy, and Bash syntax; no cluster mutation. | -| `fault-preflight` | 校验 context、CRD、StorageClass、Chaos、节点和 namespace 所有权。 / Validates context, CRDs, storage, Chaos, nodes, and namespace ownership. | -| `fault-run` | 运行一个场景,持续健康守护并验收 artifacts。 / Runs one guarded scenario and validates artifacts. | -| `fault-run-regular` | 串行运行六个常规场景,首败停止。 / Runs six regular scenarios serially and stops on first failure. | -| `fault-run-dm` | 使用预先准备的静态 PV 和 DM 设备运行 `dm-flakey`。 / Runs `dm-flakey` with pre-provisioned static PVs and DM storage. | -| `fault-cleanup` | 安全删除 owned namespace 和 managed Chaos。 / Safely removes the owned namespace and managed Chaos. | +`fault-check` is local only. It runs Bash syntax, Rust fmt, tests, and clippy. -`fault-run*` 会先用单 job、最低主机优先级预编译精确的 `faults` 测试二进制。故障窗口直接运行该二进制,不再次调用 Cargo。预编译不计入故障窗口;预编译前后都会重新执行 preflight。 +`fault-run*` prebuilds the ignored `faults` test binary before the fault window, +then runs that binary directly. The runner reruns preflight before and after the +build. -Before creating a fault Tenant, every `fault-run*` target prebuilds the exact `faults` binary with one job and the lowest host priority. The fault window executes that binary directly without invoking Cargo again. Compilation is outside the fault window, and the runner reruns preflight before and after prebuild. +## Required Environment -### 3.1 Recommended Flow / 推荐执行顺序 +Only these variables are required for regular fault scenarios: -1. 运行 `make -C e2e fault-check`,先确认本地代码、脚本和普通测试可用。 / Run `make -C e2e fault-check` first to validate code, scripts, and non-live tests. -2. 准备真实测试集群、专用 StorageClass、Chaos Mesh 和固定 digest 的 RustFS image。 / Prepare the real test cluster, dedicated StorageClass, Chaos Mesh, and a pinned RustFS image digest. -3. 导出 `RUSTFS_FAULT_TEST_EXPECTED_CONTEXT`、`RUSTFS_FAULT_TEST_STORAGE_CLASS` 和 `RUSTFS_FAULT_TEST_SERVER_IMAGE`。 / Export the required context, StorageClass, and image variables. -4. 先执行 `make -C e2e fault-preflight SCENARIO=io-eio`,再单独跑 `io-eio`。 / Run `io-eio` preflight first, then run `io-eio` alone. -5. `io-eio` 通过后再执行 `make -C e2e fault-run-regular`。 / After `io-eio` passes, run the remaining regular scenarios with `fault-run-regular`. -6. 只有准备好静态 Local PV 和 Device Mapper 后,才执行 `make -C e2e fault-run-dm`。 / Run `fault-run-dm` only after static Local PVs and Device Mapper are ready. -7. 结束后先收集 artifacts,再执行 `make -C e2e fault-cleanup`。 / Collect artifacts before running `fault-cleanup`. +```bash +export RUSTFS_FAULT_TEST_STORAGE_CLASS= +export RUSTFS_FAULT_TEST_SERVER_IMAGE='docker.io/rustfs/rustfs@sha256:' +``` -## 4. Cluster Preparation / 集群准备 +`RUSTFS_FAULT_TEST_SERVER_IMAGE` must be explicit. Prefer a pinned digest so a +failed run can be reproduced. -### 4.1 Required Tools / 必需工具 +`RUSTFS_FAULT_TEST_EXPECTED_CONTEXT` is optional. When set, both the shell +runner and Rust test entrypoint require the current context to match it exactly. +When unset, the current non-Kind context is used and pinned for the run. ```bash -rustc --version -cargo --version -kubectl version --client -jq --version -make -C e2e fault-check +export RUSTFS_FAULT_TEST_EXPECTED_CONTEXT= +``` + +## Common Overrides + +Defaults are centralized in `e2e/src/framework/fault_config.rs`. The shell +runner passes the same values into the Rust test and validates artifacts against +the selected values. Shell preflight mirrors the Rust entrypoint for numeric +ranges, booleans, and scenario-specific percent overrides. + +| Variable | Default | Use | +| --- | --- | --- | +| `RUSTFS_FAULT_TEST_NAMESPACE` | `rustfs-fault-test` | Fault namespace. | +| `RUSTFS_FAULT_TEST_TENANT` | `fault-test-tenant` | Tenant name. | +| `RUSTFS_FAULT_TEST_CHAOS_NAMESPACE` | `chaos-mesh` | Chaos Mesh namespace. | +| `RUSTFS_FAULT_TEST_USE_CLUSTER_IP` | `false` | Set to `1` when the runner can reach Service ClusterIPs. | +| `RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS` | `40000` | Total object count; must be at least 4. | +| `RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY` | `80` | S3 workload concurrency; must be 1 through object count. | +| `RUSTFS_FAULT_TEST_DURATION_SECONDS` | `7200` | Maximum fault TTL. Successful runs recover earlier. | +| `RUSTFS_FAULT_TEST_REQUEST_TIMEOUT_SECONDS` | `30` | Per S3 request timeout. | +| `RUSTFS_FAULT_TEST_TIMEOUT_SECONDS` | `300` | Kubernetes wait timeout. | +| `RUSTFS_FAULT_TEST_SEED` | generated | Reuse a workload plan. | +| `RUSTFS_FAULT_TEST_REQUIRE_CLIENT_DISRUPTION` | `false` | Require at least one client-visible failed/timeout/unknown S3 operation. | +| `RUSTFS_FAULT_TEST_BUILD_JOBS` | `1` | Cargo prebuild job count. | +| `RUSTFS_FAULT_TEST_RUN_ROOT` | timestamped target dir | Artifact root. | +| `RUSTFS_FAULT_TEST_SCENARIOS` | regular scenario list | Space-separated list for `fault-run-regular`. | + +For a small rehearsal run: + +```bash +export RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS=64 +export RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY=8 +make -C e2e fault-run SCENARIO=io-eio ``` -`warp` v1.3.1 仅用于 `warp-under-chaos`。运行机必须能访问 Kubernetes API;如果设置 ClusterIP 直连,还必须能访问 Service ClusterIP。 +`RUSTFS_FAULT_TEST_PERCENT` applies only to percent-based IOChaos scenarios: +`io-eio`, `io-read-mistake`, `disk-full`, and `warp-under-chaos`. Fixed-target +scenarios such as `pod-kill-one`, `network-partition-one`, and `dm-flakey` +reject a percent override. Their run metadata records `percent: null` and +`fault_selection` such as `1 target(s)`. -`warp` v1.3.1 is required only for `warp-under-chaos`. The runner must reach the Kubernetes API and, when ClusterIP mode is enabled, Service ClusterIPs. +## Cluster Preparation -### 4.2 Kubernetes And Storage / Kubernetes 与存储 +Check the current context first: ```bash kubectl config current-context kubectl get nodes kubectl get crd tenants.rustfs.com kubectl get storageclass -kubectl get tenant -A ``` -常规场景要求动态 StorageClass。每个承载测试 PVC 的节点应在实际 provisioner 路径上至少有 120Gi 可用空间。hostPath/local-path 的 PVC capacity 通常不执行真实配额,必须检查后端文件系统,而不能只看 `kubectl get pvc`。 +Requirements: + +- The current context must be a real Kubernetes or K3s cluster, not `kind-*`. +- At least four schedulable Ready nodes are required for the current regular + Tenant shape. +- Regular scenarios need a dedicated dynamic StorageClass. +- `dm-flakey` needs a dedicated static Local PV StorageClass and explicit + device-mapper variables. +- Other Tenants in the cluster are intentionally ignored by preflight, health + guard, and pass/fail checks. -Regular scenarios require a dynamic StorageClass. Every node that can host a test PVC should have at least 120Gi available on the actual provisioner filesystem. hostPath/local-path capacity is commonly not enforced, so inspect the backing filesystem instead of trusting only `kubectl get pvc`. +For K3s with local-path storage, verify the actual backing filesystem has +enough free space. PVC capacity alone may not enforce real disk quota. ```bash kubectl -n kube-system get configmap local-path-config -o yaml -kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.hostPath.path}{"\n"}{end}' df -h ``` -如果 K3s 默认 `/var/lib/rancher/k3s/storage` 位于小系统盘,应创建独立 provisioner/StorageClass,把 fault-test PVC 放到 `/data/rustfs/rustfs-fault-local-path` 等专用数据盘目录。不得修改现有业务 PVC 或默认 provisioner。 - -If K3s stores its default local-path data on a small system disk, create an independent provisioner and StorageClass backed by a dedicated data-disk path such as `/data/rustfs/rustfs-fault-local-path`. Do not modify existing application PVCs or the default provisioner. - -### 4.3 Chaos Mesh / Chaos Mesh - -已验证版本为 Chaos Mesh v2.8.3: - -The validated version is Chaos Mesh v2.8.3: +Chaos Mesh is required for regular scenarios. The validated version is v2.8.3: ```bash helm repo add chaos-mesh https://charts.chaos-mesh.org @@ -176,28 +163,46 @@ kubectl -n chaos-mesh get deployment,daemonset kubectl get crd iochaos.chaos-mesh.org podchaos.chaos-mesh.org networkchaos.chaos-mesh.org ``` -非 K3s 集群必须使用实际 container runtime socket。 +Use the actual runtime socket for non-K3s clusters. -Non-K3s clusters must use their actual container runtime socket. +## Recommended Run Flow -## 5. Regular Scenarios / 常规场景 +1. Run the local gate: -先固定 context、动态 StorageClass 和 RustFS image digest。测试机位于集群节点或 Pod 内时使用 ClusterIP,避免 80 并发经过 `kubectl port-forward`。 + ```bash + make -C e2e fault-check + ``` -Pin the context, dynamic StorageClass, and RustFS image digest. Use ClusterIP when the runner is on a cluster node or in a Pod so 80 concurrent requests do not traverse `kubectl port-forward`. +2. Export the required runtime values: -```bash -export RUSTFS_FAULT_TEST_EXPECTED_CONTEXT=default -export RUSTFS_FAULT_TEST_STORAGE_CLASS= -export RUSTFS_FAULT_TEST_SERVER_IMAGE='docker.io/rustfs/rustfs@sha256:' -export RUSTFS_FAULT_TEST_USE_CLUSTER_IP=1 -export RUSTFS_FAULT_TEST_RUN_ROOT="$PWD/e2e/target/fault-tests/$(date -u +%Y%m%dT%H%M%SZ)" + ```bash + export RUSTFS_FAULT_TEST_STORAGE_CLASS= + export RUSTFS_FAULT_TEST_SERVER_IMAGE='docker.io/rustfs/rustfs@sha256:' + export RUSTFS_FAULT_TEST_USE_CLUSTER_IP=1 + ``` -make -C e2e fault-preflight SCENARIO=io-eio -make -C e2e fault-run SCENARIO=io-eio -``` +3. Run preflight and the first P0 scenario: + + ```bash + make -C e2e fault-preflight SCENARIO=io-eio + make -C e2e fault-run SCENARIO=io-eio + ``` + +4. Run the regular suite: + + ```bash + make -C e2e fault-run-regular + ``` -场景顺序 / Scenario order: +5. Collect artifacts, then clean owned resources: + + ```bash + make -C e2e fault-cleanup + ``` + +## Regular Scenarios + +Current regular scenarios run serially and stop on the first failure: ```text io-eio @@ -208,263 +213,275 @@ disk-full warp-under-chaos ``` -完整运行: +The current catalog maps each scenario to one fault. The internal plan model is +not limited to that: a future scenario can contain multiple `FaultInjection` +entries and can use fixed target counts for multiple Pods or percent selection +for IO faults. -Run all regular scenarios: +To run a subset: ```bash +export RUSTFS_FAULT_TEST_SCENARIOS='pod-kill-one network-partition-one' make -C e2e fault-run-regular +unset RUSTFS_FAULT_TEST_SCENARIOS ``` -分阶段验证时,可以先运行 `io-eio`,再通过 `RUSTFS_FAULT_TEST_SCENARIOS` 指定剩余场景: +`warp-under-chaos` also requires `warp` in `PATH`. + +## Artifacts And Pass Criteria + +Artifacts are written under: + +```text +${RUSTFS_FAULT_TEST_RUN_ROOT:-e2e/target/fault-tests/}// +``` + +Key files: + +```text +run-metadata.json +workload-plan.json +history.jsonl +workload-summary.json +recommit-report.json +checker-report.json +fault-evidence.json +chaos-manifest.yaml +fault-status-*.json +nodes-*.txt +pods-*.txt +pvcs-*.txt +pvs-*.txt +events-*.txt +*.log +``` + +A successful run must show: + +- `fault-evidence.json`: `injected`, `active_during_workload`, and `recovered` + are `true`. +- `checker-report.json`: `passed` is `true`, committed object count equals the + selected workload object count, and missing objects, hash mismatches, + successful corrupted reads, and LIST warnings are empty. +- `recommit-report.json`: every previously unconfirmed PUT was recommitted + after recovery. +- `workload-plan.json`: object count, concurrency, and payload distribution are + internally consistent with the selected environment values. + +If a scenario fails, inspect `failure-summary.json`, +`runner-failure-summary.json`, `test.log`, `fault-status-*.json`, and the +RustFS Pod logs first. + +## Cleanup -For staged validation, run `io-eio` first and then select the remaining scenarios with `RUSTFS_FAULT_TEST_SCENARIOS`: +`fault-cleanup` removes managed Chaos resources and the owned fault namespace. +It does not remove external StorageClasses, PVs, provisioners, host paths, loop +devices, or device-mapper devices. ```bash -export RUSTFS_FAULT_TEST_SCENARIOS='pod-kill-one network-partition-one io-read-mistake disk-full warp-under-chaos' -make -C e2e fault-run-regular -unset RUSTFS_FAULT_TEST_SCENARIOS +make -C e2e fault-cleanup ``` -测试可能持续数小时。不要并行运行场景。每个场景完成后编排脚本会校验 seed、尺寸分布、故障状态、40,000 committed PUT 和 checker verdict。 +Manual checks: -The suite can run for several hours. Do not run scenarios in parallel. After every scenario, the runner validates the seed, size distribution, fault state, 40,000 committed PUTs, and checker verdict. +```bash +kubectl -n "${RUSTFS_FAULT_TEST_CHAOS_NAMESPACE:-chaos-mesh}" \ + get iochaos,podchaos,networkchaos \ + -l app.kubernetes.io/managed-by=rustfs-operator-fault-test -## 6. dm-flakey / dm-flakey +kubectl get namespace "${RUSTFS_FAULT_TEST_NAMESPACE:-rustfs-fault-test}" +``` -`dm-flakey` 不需要重装 Kubernetes、Operator、Chaos Mesh 或 Rust。它只需要把 fault Tenant 的存储切换为四个专用静态 Local PV,其中一个 PV 由 Device Mapper 提供。 +## dm-flakey -`dm-flakey` does not require reinstalling Kubernetes, the Operator, Chaos Mesh, or Rust. It only switches the fault Tenant to four dedicated static Local PVs, one backed by Device Mapper. +`dm-flakey` is separate from the regular suite. It needs a dedicated static +Local PV setup and privileged helper access on the fault namespace. -### 6.1 Host Storage / 主机存储 +There is no Make target that installs this environment. Prepare the host storage +and Kubernetes Local PVs first, then use `fault-preflight` to verify them. -真实专用块设备优先。loop 文件仅适用于实验室。每个 backing 至少 120Gi,并且路径必须只服务 fault-test。 +### dm-flakey Host Storage -Prefer dedicated block devices. Loop files are for lab use only. Each backing device must be at least 120Gi and serve only fault-test. +Prefer real dedicated block devices. The loop-file commands below are for lab +clusters only. Run them on the Kubernetes nodes that will host the four static +Local PVs. -DM 节点示例 / DM-node example: +On the node that will receive the device-mapper fault: ```bash export LAB=/data/rustfs/rustfs-fault-lab export DM_NAME=rustfs-fault-dm + sudo mkdir -p "$LAB/volume" sudo truncate -s 120G "$LAB/disk.img" -export BACKING=$(sudo losetup --find --show "$LAB/disk.img") -export SECTORS=$(sudo blockdev --getsz "$BACKING") +export BACKING="$(sudo losetup --find --show "$LAB/disk.img")" +export SECTORS="$(sudo blockdev --getsz "$BACKING")" sudo dmsetup create "$DM_NAME" --table "0 $SECTORS linear $BACKING 0" sudo mkfs.ext4 -F "/dev/mapper/$DM_NAME" sudo mount "/dev/mapper/$DM_NAME" "$LAB/volume" sudo chmod 0777 "$LAB/volume" + +sudo dmsetup table "$DM_NAME" +findmnt -n -o SOURCE --target "$LAB/volume" ``` -其他三个节点 / Other three nodes: +On each of the other three nodes: ```bash export LAB=/data/rustfs/rustfs-fault-lab + sudo mkdir -p "$LAB/volume" sudo truncate -s 120G "$LAB/disk.img" -export BACKING=$(sudo losetup --find --show "$LAB/disk.img") +export BACKING="$(sudo losetup --find --show "$LAB/disk.img")" sudo mkfs.ext4 -F "$BACKING" sudo mount "$BACKING" "$LAB/volume" sudo chmod 0777 "$LAB/volume" +findmnt -n -o SOURCE --target "$LAB/volume" ``` -### 6.2 Static StorageClass And PVs / 静态 StorageClass 与 PV +### dm-flakey Kubernetes Storage -创建 `kubernetes.io/no-provisioner` StorageClass,并为四个节点各创建一个 `100Gi` Local PV。每个 PV 的 node affinity 必须匹配实际节点;`local.path` 必须是 `/data/rustfs/rustfs-fault-lab/volume`。 +Create one `kubernetes.io/no-provisioner` StorageClass and exactly four `100Gi` +Local PVs for the fault StorageClass. Each PV must point at the host path created +above and must use node affinity for its real node name. -Create a `kubernetes.io/no-provisioner` StorageClass and one `100Gi` Local PV per node. Each PV must use the matching node affinity and `/data/rustfs/rustfs-fault-lab/volume` as `local.path`. +```bash +export DM_STORAGE_CLASS=rustfs-fault-dm +export DM_MOUNT_PATH=/data/rustfs/rustfs-fault-lab/volume -```yaml +kubectl apply -f - <` and `` each time: + +```bash +kubectl apply -f - < + name: labels: app.kubernetes.io/managed-by: rustfs-operator-fault-test spec: capacity: storage: 100Gi volumeMode: Filesystem - accessModes: [ReadWriteOnce] + accessModes: + - ReadWriteOnce persistentVolumeReclaimPolicy: Retain - storageClassName: rustfs-fault-dm + storageClassName: ${DM_STORAGE_CLASS} local: - path: /data/rustfs/rustfs-fault-lab/volume + path: ${DM_MOUNT_PATH} nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In - values: [] + values: + - +EOF ``` -验证四个 PV 为 `Available`: - -Verify all four PVs are `Available`: +Pre-create or update the fault namespace so the helper pod can run privileged +and the runner can prove ownership: ```bash -kubectl get storageclass rustfs-fault-dm -kubectl get pv -l app.kubernetes.io/managed-by=rustfs-operator-fault-test -o wide +export RUSTFS_FAULT_TEST_NAMESPACE="${RUSTFS_FAULT_TEST_NAMESPACE:-rustfs-fault-test}" +export RUSTFS_FAULT_TEST_TENANT="${RUSTFS_FAULT_TEST_TENANT:-fault-test-tenant}" + +kubectl create namespace "$RUSTFS_FAULT_TEST_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - +kubectl label namespace "$RUSTFS_FAULT_TEST_NAMESPACE" \ + app.kubernetes.io/managed-by=rustfs-operator-fault-test \ + pod-security.kubernetes.io/enforce=privileged \ + --overwrite +kubectl annotate namespace "$RUSTFS_FAULT_TEST_NAMESPACE" \ + "rustfs.com/fault-test-tenant=$RUSTFS_FAULT_TEST_TENANT" \ + --overwrite ``` -helper Pod 需要 privileged Pod Security。复用常规场景创建的 namespace 时补充 label;如果 namespace 不存在,则预创建带完整所有权的 namespace: - -The helper Pod requires privileged Pod Security. Label the namespace left by regular scenarios, or pre-create an owned namespace when it does not exist: +Verify the storage setup before running the scenario: ```bash -if kubectl get namespace rustfs-fault-test >/dev/null 2>&1; then - kubectl label namespace rustfs-fault-test \ - pod-security.kubernetes.io/enforce=privileged --overwrite -else - kubectl create namespace rustfs-fault-test - kubectl label namespace rustfs-fault-test \ - app.kubernetes.io/managed-by=rustfs-operator-fault-test \ - pod-security.kubernetes.io/enforce=privileged - kubectl annotate namespace rustfs-fault-test \ - rustfs.com/fault-test-tenant=fault-test-tenant -fi +kubectl get storageclass "$DM_STORAGE_CLASS" +kubectl get pv -o wide | grep "$DM_STORAGE_CLASS" +kubectl get namespace "$RUSTFS_FAULT_TEST_NAMESPACE" --show-labels ``` -### 6.3 Run / 执行 +The `dm-flakey` preflight requires exactly four `Available` or `Bound` `100Gi` +PVs in the selected static StorageClass. + +### dm-flakey Run + +Required variables on the machine that runs the e2e command: ```bash +export RUSTFS_FAULT_TEST_SERVER_IMAGE='docker.io/rustfs/rustfs@sha256:' export RUSTFS_FAULT_TEST_STORAGE_CLASS=rustfs-fault-dm export RUSTFS_FAULT_TEST_DM_NAME=rustfs-fault-dm export RUSTFS_FAULT_TEST_DM_NODE= export RUSTFS_FAULT_TEST_DM_MOUNT_PATH=/data/rustfs/rustfs-fault-lab/volume -export RUSTFS_FAULT_TEST_DM_FAULT_TABLE="0 $SECTORS flakey $BACKING 0 1 15" - -make -C e2e fault-preflight SCENARIO=dm-flakey -make -C e2e fault-run-dm -``` - -## 7. Evidence And Acceptance / 证据与验收 - -每个场景目录由 runner 固定创建: - -Each scenario directory is created by the runner with: - -```text -test.log -health-watch.log -nodes-before.txt / nodes-after.txt -tenants-before.txt / tenants-after.txt -pods-before.txt / pods-after.txt -Chaos or DM snapshots -``` - -Rust 测试初始化后会写入: - -After Rust test initialization, it contains: - -```text -run-metadata.json -workload-plan.json -history.jsonl +export RUSTFS_FAULT_TEST_DM_FAULT_TABLE='0 flakey 0 1 15' ``` -workload 执行后会写入: +Use the `SECTORS` and `BACKING` values from the DM node host-storage setup for +`` and ``. -After workload execution, it contains: +Optional: -```text -workload-summary.json -``` - -Successful recovery and reconciliation also contain: - -```text -fault-evidence.json -recommit-report.json -checker-report.json +```bash +export RUSTFS_FAULT_TEST_DM_RECOVERY_TABLE='' +export RUSTFS_FAULT_TEST_DM_HELPER_IMAGE='rancher/mirrored-library-busybox:1.37.0' ``` -Failure paths contain the artifacts that were reachable before the failure plus: +Run: -```text -failure-summary.json / runner-failure-summary.json +```bash +make -C e2e fault-preflight SCENARIO=dm-flakey +make -C e2e fault-run-dm ``` -通过条件 / Pass criteria: - -- 测试退出码为 0。 -- `run-metadata.json` 记录 scenario、run id、context、StorageClass、RustFS image 和 workload 参数。 -- `fault-evidence.json` 的 `injected`、`active_during_workload`、`recovered` 都为 `true`。 -- `workload-plan.json` 精确记录 40,000 对象、80 并发和四档尺寸分布。 -- `recommit-report.json` 的 `attempted == committed`、`failed == 0` 且 `harness_errors == 0`。 -- `checker-report.json` 的 `committed_puts=40000`,并且 missing、hash mismatch、successful corrupted read、LIST warning 均为空。 -- fault Tenant 恢复 Ready;Ready 节点数量不下降;常规 Chaos 场景中 Chaos Mesh 保持健康。 -- The test exits with zero. -- `run-metadata.json` records the scenario, run id, context, StorageClass, RustFS image, and workload parameters. -- `fault-evidence.json` reports `injected`, `active_during_workload`, and `recovered` as `true`. -- `workload-plan.json` reports exactly 40,000 objects, concurrency 80, and the four size classes. -- `recommit-report.json` reports `attempted == committed`, `failed == 0`, and `harness_errors == 0`. -- `checker-report.json` reports `committed_puts=40000` with no missing object, hash mismatch, successful corrupted read, or LIST warning. -- The fault Tenant recovers Ready, the Ready node count does not drop, and Chaos Mesh remains healthy during regular Chaos scenarios. +The Rust test reads the original `dmsetup table` as the recovery table when +`RUSTFS_FAULT_TEST_DM_RECOVERY_TABLE` is unset. On normal failure paths it +restores that table, but operators must still verify host storage manually after +the run. -客户端没有看到错误并不表示故障无效。故障是否生效由 Chaos/DM 后端证据判断;客户端 disruption 单独记录。 +### dm-flakey Cleanup -No client-visible error does not mean the fault was inactive. Chaos/DM backend evidence proves injection; client disruption is reported separately. - -## 8. Cleanup And Recovery / 清理与恢复 - -先运行安全清理: - -Start with owned-resource cleanup: +`fault-cleanup` removes the owned Kubernetes namespace and managed Chaos +resources only. It does not remove the static StorageClass, PVs, loop devices, +mounts, or device-mapper device. ```bash make -C e2e fault-cleanup +kubectl delete pv -l app.kubernetes.io/managed-by=rustfs-operator-fault-test +kubectl delete storageclass rustfs-fault-dm ``` -然后由运维删除本次创建的外部 StorageClass、静态 PV、独立 provisioner 和主机设备。DM 实验室清理示例: - -Operators must then remove the external StorageClass, static PVs, independent provisioner, and host devices created for the run. Lab DM cleanup example: +On the DM node: ```bash sudo umount /data/rustfs/rustfs-fault-lab/volume -sudo dmsetup remove rustfs-fault-dm # DM node only +sudo dmsetup remove rustfs-fault-dm +sudo losetup -j /data/rustfs/rustfs-fault-lab/disk.img sudo losetup -d sudo rm -rf /data/rustfs/rustfs-fault-lab -kubectl delete pv -l app.kubernetes.io/managed-by=rustfs-operator-fault-test -kubectl delete storageclass rustfs-fault-dm ``` -最终确认 / Final checks: +On the other three nodes: ```bash -kubectl get nodes -kubectl get tenant -A -kubectl -n chaos-mesh get deployment,daemonset -kubectl get iochaos,podchaos,networkchaos -A -kubectl get namespace rustfs-fault-test +sudo umount /data/rustfs/rustfs-fault-lab/volume +sudo losetup -j /data/rustfs/rustfs-fault-lab/disk.img +sudo losetup -d +sudo rm -rf /data/rustfs/rustfs-fault-lab ``` - -## 9. Runtime Variables / 运行参数 - -| Variable | Default | Purpose / 用途 | -| --- | --- | --- | -| `RUSTFS_FAULT_TEST_EXPECTED_CONTEXT` | required | 防止在错误 context 执行。 / Prevents execution against the wrong context. | -| `RUSTFS_FAULT_TEST_STORAGE_CLASS` | required | 常规动态 SC 或 DM 静态 SC。 / Dynamic regular SC or static DM SC. | -| `RUSTFS_FAULT_TEST_SERVER_IMAGE` | required by Make | 建议固定 digest。 / Pin an image digest. | -| `RUSTFS_FAULT_TEST_RUN_ROOT` | timestamp directory | 整次运行的 artifacts 根目录。 / Artifact root for the run. | -| `RUSTFS_FAULT_TEST_SCENARIOS` | six regular scenarios | `fault-run-regular` 的空格分隔场景列表。 / Space-separated regular scenario list. | -| `RUSTFS_FAULT_TEST_SEED` | generated | 固定后可重放相同对象。 / Replays the same objects when set. | -| `RUSTFS_FAULT_TEST_USE_CLUSTER_IP` | `false` | 集群节点/Pod 内建议设为 `1`。 / Set to `1` on a node or in-cluster runner. | -| `RUSTFS_FAULT_TEST_BUILD_JOBS` | `1` | 预编译并行度;小型控制面保持为 1。 / Prebuild parallelism; keep at 1 on small control planes. | -| `RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS` | `40000` | Make runner 强制验收该值。 / Required object count. | -| `RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY` | `80` | Make runner 强制验收该值。 / Required concurrency. | -| `RUSTFS_FAULT_TEST_DURATION_SECONDS` | `7200` | 最大故障 TTL。 / Maximum fault TTL. | -| `RUSTFS_FAULT_TEST_REQUEST_TIMEOUT_SECONDS` | `30` | 单次 S3 请求超时。 / Per-request S3 timeout. | -| `RUSTFS_FAULT_TEST_REQUIRE_CLIENT_DISRUPTION` | `false` | 是否要求客户端可见错误。 / Whether client-visible disruption is mandatory. | -| `RUSTFS_FAULT_TEST_CHAOS_NAMESPACE` | `chaos-mesh` | Chaos resource namespace。 | -| `RUSTFS_FAULT_TEST_DM_*` | unset | `dm-flakey` 专用映射参数。 / DM mapping parameters. | diff --git a/e2e/Makefile b/e2e/Makefile index cff152e..5752d34 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -32,9 +32,10 @@ help: @echo " make -C e2e fault-cleanup" @echo "" @echo "Required runtime environment:" - @echo " RUSTFS_FAULT_TEST_EXPECTED_CONTEXT" @echo " RUSTFS_FAULT_TEST_STORAGE_CLASS" @echo " RUSTFS_FAULT_TEST_SERVER_IMAGE" + @echo "Optional safety guard:" + @echo " RUSTFS_FAULT_TEST_EXPECTED_CONTEXT" @echo "" @echo "See e2e/FAULT_TESTING.md for cluster preparation and safety requirements." diff --git a/e2e/scripts/fault-test.sh b/e2e/scripts/fault-test.sh index bb3cdb3..35eeed7 100644 --- a/e2e/scripts/fault-test.sh +++ b/e2e/scripts/fault-test.sh @@ -21,11 +21,11 @@ MANIFEST="$PACKAGE_DIR/Cargo.toml" MANAGER="rustfs-operator-fault-test" MANAGER_SELECTOR="app.kubernetes.io/managed-by=$MANAGER" DEFAULT_SCENARIOS="io-eio pod-kill-one network-partition-one io-read-mistake disk-full warp-under-chaos" -EXPECTED_OBJECTS=40000 -EXPECTED_CONCURRENCY=80 -EXPECTED_PAYLOAD_BYTES=20337459200 +WORKLOAD_OBJECTS="${RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS:-40000}" +WORKLOAD_CONCURRENCY="${RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY:-80}" BUILD_JOBS="${RUSTFS_FAULT_TEST_BUILD_JOBS:-1}" +FAULT_CONTEXT="${RUSTFS_FAULT_TEST_EXPECTED_CONTEXT:-}" FAULT_NAMESPACE="${RUSTFS_FAULT_TEST_NAMESPACE:-rustfs-fault-test}" FAULT_TENANT="${RUSTFS_FAULT_TEST_TENANT:-fault-test-tenant}" CHAOS_NAMESPACE="${RUSTFS_FAULT_TEST_CHAOS_NAMESPACE:-chaos-mesh}" @@ -43,7 +43,8 @@ Commands: run-regular Run the six regular scenarios serially. cleanup Remove managed Chaos and the owned fault namespace. -Run through the package Make targets documented in e2e/FAULT_TESTING.md. +RUSTFS_FAULT_TEST_EXPECTED_CONTEXT is optional. When unset, the current +non-Kind kubectl context is used and pinned for the run. EOF } @@ -56,16 +57,106 @@ require_command() { command -v "$1" >/dev/null 2>&1 || die "required command not found: $1" } +trim_value() { + local value="$1" + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + printf '%s' "$value" +} + +require_nonempty_env() { + local name="$1" value + value="$(trim_value "${!name:-}")" + [[ -n "$value" ]] || die "$name is required" + export "$name=$value" +} + +require_positive_integer() { + local name="$1" value="$2" + [[ "$value" =~ ^[1-9][0-9]*$ ]] || die "$name must be a positive integer" +} + +require_unsigned_integer() { + local name="$1" value="$2" + [[ "$value" =~ ^[0-9]+$ ]] || die "$name must be an unsigned integer" +} + +require_optional_unsigned_integer() { + local name="$1" value + value="$(trim_value "${!name:-}")" + [[ -z "$value" ]] && return 0 + require_unsigned_integer "$name" "$value" + export "$name=$value" +} + +require_optional_positive_integer() { + local name="$1" value + value="$(trim_value "${!name:-}")" + [[ -z "$value" ]] && return 0 + require_positive_integer "$name" "$value" + export "$name=$value" +} + +require_optional_bool() { + local name="$1" value + value="$(trim_value "${!name:-}")" + [[ -z "$value" ]] && return 0 + case "$value" in + 1|0|[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]|[Yy][Ee][Ss]|[Nn][Oo]) + export "$name=$value" + ;; + *) + die "$name must be a boolean: 1/0, true/false, or yes/no" + ;; + esac +} + +require_safe_node_name() { + local name="$1" value="$2" + [[ "$value" =~ ^[A-Za-z0-9.-]+$ ]] || die "$name must be a valid node name" +} + +require_safe_dm_name() { + local name="$1" value="$2" + [[ "$value" =~ ^[A-Za-z0-9._+-]+$ ]] || die "$name contains unsupported characters" +} + +require_absolute_non_root_path() { + local name="$1" value="$2" + [[ "$value" == /* && "$value" != "/" ]] || die "$name must be an absolute non-root path" + [[ "$value" != *$'\n'* && "$value" != *$'\r'* ]] || die "$name must not contain newlines" +} + +require_safe_image_ref() { + local name="$1" value="$2" + [[ -n "$value" ]] || die "$name must be a non-empty image reference" + [[ "$value" != *[[:space:]]* ]] || die "$name must not contain whitespace" +} + kubectl_context() { kubectl config current-context } +resolve_fault_context() { + local current_context + FAULT_CONTEXT="$(trim_value "$FAULT_CONTEXT")" + current_context="$(kubectl_context)" + if [[ -n "$FAULT_CONTEXT" ]]; then + [[ "$current_context" == "$FAULT_CONTEXT" ]] || die "current context $current_context does not match RUSTFS_FAULT_TEST_EXPECTED_CONTEXT $FAULT_CONTEXT" + export RUSTFS_FAULT_TEST_EXPECTED_CONTEXT="$FAULT_CONTEXT" + else + FAULT_CONTEXT="$current_context" + export RUSTFS_FAULT_TEST_EXPECTED_CONTEXT="$FAULT_CONTEXT" + fi + [[ "$FAULT_CONTEXT" != kind-* ]] || die "fault tests require a real Kubernetes or K3s cluster, got $FAULT_CONTEXT" +} + kubectl_ns() { - kubectl --context "$RUSTFS_FAULT_TEST_EXPECTED_CONTEXT" -n "$1" "${@:2}" + kubectl --context "$FAULT_CONTEXT" -n "$1" "${@:2}" } kubectl_cluster() { - kubectl --context "$RUSTFS_FAULT_TEST_EXPECTED_CONTEXT" "$@" + kubectl --context "$FAULT_CONTEXT" "$@" } is_supported_scenario() { @@ -79,6 +170,17 @@ is_supported_scenario() { esac } +is_percent_scenario() { + case "$1" in + io-eio|io-read-mistake|disk-full|warp-under-chaos) + return 0 + ;; + *) + return 1 + ;; + esac +} + scenario_crd() { case "$1" in pod-kill-one) echo "podchaos.chaos-mesh.org" ;; @@ -88,6 +190,46 @@ scenario_crd() { esac } +validate_runtime_env_contract() { + local scenario="$1" percent + + WORKLOAD_OBJECTS="$(trim_value "$WORKLOAD_OBJECTS")" + WORKLOAD_CONCURRENCY="$(trim_value "$WORKLOAD_CONCURRENCY")" + BUILD_JOBS="$(trim_value "$BUILD_JOBS")" + + require_positive_integer RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS "$WORKLOAD_OBJECTS" + (( 10#$WORKLOAD_OBJECTS >= 4 )) || die "RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS must be at least 4" + require_positive_integer RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY "$WORKLOAD_CONCURRENCY" + (( 10#$WORKLOAD_CONCURRENCY <= 10#$WORKLOAD_OBJECTS )) || die "RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY must be <= RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS" + require_optional_positive_integer RUSTFS_FAULT_TEST_DURATION_SECONDS + require_optional_unsigned_integer RUSTFS_FAULT_TEST_REQUEST_TIMEOUT_SECONDS + require_optional_unsigned_integer RUSTFS_FAULT_TEST_TIMEOUT_SECONDS + require_optional_unsigned_integer RUSTFS_FAULT_TEST_WARP_DURATION_SECONDS + require_optional_unsigned_integer RUSTFS_FAULT_TEST_SEED + require_optional_bool RUSTFS_FAULT_TEST_USE_CLUSTER_IP + require_optional_bool RUSTFS_FAULT_TEST_REQUIRE_CLIENT_DISRUPTION + + percent="$(trim_value "${RUSTFS_FAULT_TEST_PERCENT:-}")" + if [[ -n "$percent" ]]; then + require_positive_integer RUSTFS_FAULT_TEST_PERCENT "$percent" + (( 10#$percent <= 100 )) || die "RUSTFS_FAULT_TEST_PERCENT must be in 1..=100" + is_percent_scenario "$scenario" || die "RUSTFS_FAULT_TEST_PERCENT only applies to percent-based IOChaos scenarios" + export RUSTFS_FAULT_TEST_PERCENT="$percent" + fi +} + +validate_dm_env_contract() { + require_nonempty_env RUSTFS_FAULT_TEST_DM_NAME + require_nonempty_env RUSTFS_FAULT_TEST_DM_NODE + require_nonempty_env RUSTFS_FAULT_TEST_DM_MOUNT_PATH + require_nonempty_env RUSTFS_FAULT_TEST_DM_FAULT_TABLE + + require_safe_dm_name RUSTFS_FAULT_TEST_DM_NAME "$RUSTFS_FAULT_TEST_DM_NAME" + require_safe_node_name RUSTFS_FAULT_TEST_DM_NODE "$RUSTFS_FAULT_TEST_DM_NODE" + require_absolute_non_root_path RUSTFS_FAULT_TEST_DM_MOUNT_PATH "$RUSTFS_FAULT_TEST_DM_MOUNT_PATH" + require_safe_image_ref RUSTFS_FAULT_TEST_DM_HELPER_IMAGE "${RUSTFS_FAULT_TEST_DM_HELPER_IMAGE:-rancher/mirrored-library-busybox:1.37.0}" +} + require_namespace_ownership() { if ! kubectl_cluster get namespace "$FAULT_NAMESPACE" >/dev/null 2>&1; then return 0 @@ -108,7 +250,8 @@ prepare_fault_binary() { --message-format=json-render-diagnostics ) - [[ "$BUILD_JOBS" =~ ^[1-9][0-9]*$ ]] || die "RUSTFS_FAULT_TEST_BUILD_JOBS must be a positive integer" + BUILD_JOBS="$(trim_value "$BUILD_JOBS")" + require_positive_integer RUSTFS_FAULT_TEST_BUILD_JOBS "$BUILD_JOBS" preflight "$scenario" echo "preparing fault-test binary with jobs=$BUILD_JOBS and lowest host priority" if command -v ionice >/dev/null 2>&1; then @@ -163,8 +306,8 @@ require_chaos_ready() { require_storage_class() { local scenario="$1" local storage_class provisioner pv_count - storage_class="${RUSTFS_FAULT_TEST_STORAGE_CLASS:-}" - [[ -n "$storage_class" ]] || die "RUSTFS_FAULT_TEST_STORAGE_CLASS is required" + require_nonempty_env RUSTFS_FAULT_TEST_STORAGE_CLASS + storage_class="$RUSTFS_FAULT_TEST_STORAGE_CLASS" provisioner="$(kubectl_cluster get storageclass "$storage_class" -o json | jq -r '.provisioner // ""')" [[ -n "$provisioner" ]] || die "StorageClass $storage_class has no provisioner" @@ -185,7 +328,7 @@ require_storage_class() { preflight() { local scenario="${1:-io-eio}" - local current_context ready_nodes crd + local ready_nodes crd is_supported_scenario "$scenario" || die "unsupported scenario: $scenario" require_command cargo @@ -193,12 +336,10 @@ preflight() { require_command kubectl require_command nice require_command pgrep - [[ -n "${RUSTFS_FAULT_TEST_EXPECTED_CONTEXT:-}" ]] || die "RUSTFS_FAULT_TEST_EXPECTED_CONTEXT is required" - [[ -n "${RUSTFS_FAULT_TEST_SERVER_IMAGE:-}" ]] || die "RUSTFS_FAULT_TEST_SERVER_IMAGE is required" + validate_runtime_env_contract "$scenario" + require_nonempty_env RUSTFS_FAULT_TEST_SERVER_IMAGE - current_context="$(kubectl_context)" - [[ "$current_context" == "$RUSTFS_FAULT_TEST_EXPECTED_CONTEXT" ]] || die "current context $current_context does not match expected context $RUSTFS_FAULT_TEST_EXPECTED_CONTEXT" - [[ "$current_context" != kind-* ]] || die "fault tests require a real Kubernetes cluster, got $current_context" + resolve_fault_context kubectl_cluster get crd tenants.rustfs.com >/dev/null ready_nodes="$(kubectl_cluster get nodes -o json | jq -r '[.items[] @@ -218,25 +359,18 @@ preflight() { require_command warp fi if [[ "$scenario" == "dm-flakey" ]]; then - [[ -n "${RUSTFS_FAULT_TEST_DM_NAME:-}" ]] || die "RUSTFS_FAULT_TEST_DM_NAME is required" - [[ -n "${RUSTFS_FAULT_TEST_DM_NODE:-}" ]] || die "RUSTFS_FAULT_TEST_DM_NODE is required" - [[ -n "${RUSTFS_FAULT_TEST_DM_MOUNT_PATH:-}" ]] || die "RUSTFS_FAULT_TEST_DM_MOUNT_PATH is required" - [[ -n "${RUSTFS_FAULT_TEST_DM_FAULT_TABLE:-}" ]] || die "RUSTFS_FAULT_TEST_DM_FAULT_TABLE is required" + validate_dm_env_contract kubectl_cluster get namespace "$FAULT_NAMESPACE" >/dev/null 2>&1 || die "dm-flakey requires a pre-created owned fault namespace with privileged Pod Security" [[ "$(kubectl_cluster get namespace "$FAULT_NAMESPACE" -o jsonpath='{.metadata.labels.pod-security\.kubernetes\.io/enforce}')" == "privileged" ]] || die "dm-flakey requires pod-security.kubernetes.io/enforce=privileged on $FAULT_NAMESPACE" fi - echo "preflight passed: context=$current_context scenario=$scenario nodes=$ready_nodes storageClass=${RUSTFS_FAULT_TEST_STORAGE_CLASS}" + echo "preflight passed: context=$FAULT_CONTEXT scenario=$scenario nodes=$ready_nodes storageClass=${RUSTFS_FAULT_TEST_STORAGE_CLASS} objects=$WORKLOAD_OBJECTS concurrency=$WORKLOAD_CONCURRENCY" } preflight_cleanup() { - local current_context require_command jq require_command kubectl - [[ -n "${RUSTFS_FAULT_TEST_EXPECTED_CONTEXT:-}" ]] || die "RUSTFS_FAULT_TEST_EXPECTED_CONTEXT is required" - current_context="$(kubectl_context)" - [[ "$current_context" == "$RUSTFS_FAULT_TEST_EXPECTED_CONTEXT" ]] || die "current context $current_context does not match expected context $RUSTFS_FAULT_TEST_EXPECTED_CONTEXT" - [[ "$current_context" != kind-* ]] || die "fault cleanup requires a real Kubernetes cluster, got $current_context" + resolve_fault_context require_namespace_ownership } @@ -271,9 +405,10 @@ handle_signal() { capture_cluster_snapshot() { local artifacts="$1" stage="$2" kubectl_cluster get nodes -o wide >"$artifacts/nodes-$stage.txt" 2>&1 || true - kubectl_cluster get tenants -A -o wide >"$artifacts/tenants-$stage.txt" 2>&1 || true - kubectl_cluster get pods -A -o wide >"$artifacts/pods-$stage.txt" 2>&1 || true - kubectl_cluster get pv,pvc -A -o wide >"$artifacts/volumes-$stage.txt" 2>&1 || true + kubectl_ns "$FAULT_NAMESPACE" get tenants -o wide >"$artifacts/tenants-$stage.txt" 2>&1 || true + kubectl_ns "$FAULT_NAMESPACE" get pods -o wide >"$artifacts/pods-$stage.txt" 2>&1 || true + kubectl_ns "$FAULT_NAMESPACE" get pvc -o wide >"$artifacts/pvcs-$stage.txt" 2>&1 || true + kubectl_cluster get pv -o wide >"$artifacts/pvs-$stage.txt" 2>&1 || true kubectl_ns "$CHAOS_NAMESPACE" get iochaos,podchaos,networkchaos -o yaml >"$artifacts/chaos-$stage.yaml" 2>&1 || true kubectl_ns "$FAULT_NAMESPACE" get events --sort-by=.lastTimestamp >"$artifacts/events-$stage.txt" 2>&1 || true } @@ -316,27 +451,24 @@ validate_scenario_artifacts() { [[ -f "$summary" ]] || die "$scenario did not produce workload-summary.json" [[ -f "$recommit" ]] || die "$scenario did not produce recommit-report.json" - jq -e --arg scenario "$scenario" ' + jq -e --arg scenario "$scenario" --argjson objects "$WORKLOAD_OBJECTS" --argjson concurrency "$WORKLOAD_CONCURRENCY" ' .scenario == $scenario and (.run_id | length) > 0 and (.rustfs_image | length) > 0 and (.storage_class | length) > 0 and (.context | length) > 0 + and .workload_objects == $objects + and .workload_concurrency == $concurrency ' "$metadata" >/dev/null || die "$scenario run metadata is incomplete" - jq -e --argjson objects "$EXPECTED_OBJECTS" --argjson concurrency "$EXPECTED_CONCURRENCY" --argjson payload "$EXPECTED_PAYLOAD_BYTES" ' + jq -e --argjson objects "$WORKLOAD_OBJECTS" --argjson concurrency "$WORKLOAD_CONCURRENCY" ' .object_count == $objects and .concurrency == $concurrency - and .total_payload_bytes == $payload - and .size_distribution == [ - {"size_bytes":4096,"object_count":34000}, - {"size_bytes":16384,"object_count":4000}, - {"size_bytes":8388608,"object_count":1600}, - {"size_bytes":16777216,"object_count":400} - ] + and ([.size_distribution[].object_count] | add) == $objects + and ([.size_distribution[] | (.size_bytes * .object_count)] | add) == .total_payload_bytes ' "$plan" >/dev/null || die "$scenario workload plan does not match the required profile" jq -e '.injected == true and .active_during_workload == true and .recovered == true' "$evidence" >/dev/null || die "$scenario fault evidence is incomplete" jq -e '(.active_snapshots | length) > 0 and (.workload_snapshots | length) > 0' "$evidence" >/dev/null || die "$scenario fault evidence snapshots are missing" - jq -e --argjson objects "$EXPECTED_OBJECTS" ' + jq -e --argjson objects "$WORKLOAD_OBJECTS" ' .committed_puts == $objects and (.missing_committed_objects | length) == 0 and (.hash_mismatches | length) == 0 @@ -400,8 +532,8 @@ run_scenario() { set +e RUSTFS_FAULT_TEST_DESTRUCTIVE=1 \ RUSTFS_FAULT_TEST_SCENARIO="$scenario" \ - RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS="$EXPECTED_OBJECTS" \ - RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY="$EXPECTED_CONCURRENCY" \ + RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS="$WORKLOAD_OBJECTS" \ + RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY="$WORKLOAD_CONCURRENCY" \ RUSTFS_FAULT_TEST_DURATION_SECONDS="${RUSTFS_FAULT_TEST_DURATION_SECONDS:-7200}" \ RUSTFS_FAULT_TEST_ARTIFACTS="$artifacts" \ "$FAULT_TEST_BINARY" --ignored --test-threads=1 --nocapture \ diff --git a/e2e/src/framework/chaos_mesh.rs b/e2e/src/framework/chaos_mesh.rs index 6c9dbd7..cf2012c 100644 --- a/e2e/src/framework/chaos_mesh.rs +++ b/e2e/src/framework/chaos_mesh.rs @@ -465,6 +465,13 @@ pub fn cleanup_run_kind( Ok(()) } +pub fn cleanup_managed_chaos(config: &ClusterTestConfig, namespace: &str) -> Result<()> { + for kind in ["iochaos", "podchaos", "networkchaos"] { + cleanup_managed_kind(config, namespace, kind)?; + } + Ok(()) +} + pub fn cleanup_managed_iochaos(config: &ClusterTestConfig, namespace: &str) -> Result<()> { cleanup_managed_kind(config, namespace, "iochaos") } diff --git a/e2e/src/framework/fault_config.rs b/e2e/src/framework/fault_config.rs index ab018d6..77d4aac 100644 --- a/e2e/src/framework/fault_config.rs +++ b/e2e/src/framework/fault_config.rs @@ -12,22 +12,65 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::{Context, Result, ensure}; +use anyhow::{Context, Result, bail, ensure}; use serde_json::Value; use std::path::PathBuf; use std::time::Duration; use crate::framework::{command::CommandSpec, config::ClusterTestConfig, kubectl::Kubectl}; +pub const DEFAULT_FAULT_NAMESPACE: &str = "rustfs-fault-test"; +pub const DEFAULT_FAULT_TENANT: &str = "fault-test-tenant"; +pub const DEFAULT_CHAOS_NAMESPACE: &str = "chaos-mesh"; +pub const DEFAULT_OPERATOR_NAMESPACE: &str = "rustfs-system"; +pub const DEFAULT_WORKLOAD_OBJECTS: usize = 40_000; +pub const DEFAULT_WORKLOAD_CONCURRENCY: usize = 80; +pub const DEFAULT_FAULT_DURATION_SECONDS: u64 = 7_200; +pub const DEFAULT_REQUEST_TIMEOUT_SECONDS: u64 = 30; +pub const DEFAULT_CLUSTER_TIMEOUT_SECONDS: u64 = 300; +pub const DEFAULT_WARP_DURATION_SECONDS: u64 = 60; +pub const DEFAULT_DM_HELPER_IMAGE: &str = "rancher/mirrored-library-busybox:1.37.0"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FaultWorkloadProfile { + pub object_count: usize, + pub concurrency: usize, +} + +impl FaultWorkloadProfile { + pub fn new(object_count: usize, concurrency: usize) -> Result { + let profile = Self { + object_count, + concurrency, + }; + profile.validate()?; + Ok(profile) + } + + pub fn validate(self) -> Result<()> { + ensure!( + self.object_count >= 4, + "RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS must be at least 4" + ); + ensure!( + (1..=self.object_count).contains(&self.concurrency), + "RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY must be between 1 and RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS ({})", + self.object_count + ); + Ok(()) + } +} + #[derive(Debug, Clone)] pub struct FaultTestConfig { pub cluster: ClusterTestConfig, + pub expected_context: Option, pub destructive_enabled: bool, pub scenario: String, pub duration: Duration, pub percent: u8, - pub workload_objects: usize, - pub workload_concurrency: usize, + pub percent_overridden: bool, + pub workload: FaultWorkloadProfile, pub workload_seed: Option, pub request_timeout: Duration, pub use_cluster_ip: bool, @@ -52,31 +95,51 @@ impl FaultTestConfig { where F: Fn(&str) -> Option, { + let expected_context = env_optional(&get_env, "RUSTFS_FAULT_TEST_EXPECTED_CONTEXT"); + if let Some(expected) = expected_context.as_deref() { + ensure!( + context == expected, + "current context {context:?} does not match RUSTFS_FAULT_TEST_EXPECTED_CONTEXT {expected:?}" + ); + } ensure!( !context.starts_with("kind-"), - "fault tests require a real Kubernetes cluster; current context {context:?} is a Kind context" + "fault tests require a real Kubernetes or K3s cluster; current context {context:?} is a Kind context" ); let storage_class = required_env(&get_env, "RUSTFS_FAULT_TEST_STORAGE_CLASS")?; - let namespace = env_or(&get_env, "RUSTFS_FAULT_TEST_NAMESPACE", "rustfs-fault-test"); + let rustfs_image = required_env(&get_env, "RUSTFS_FAULT_TEST_SERVER_IMAGE")?; + let namespace = env_or( + &get_env, + "RUSTFS_FAULT_TEST_NAMESPACE", + DEFAULT_FAULT_NAMESPACE, + ); let scenario = env_or(&get_env, "RUSTFS_FAULT_TEST_SCENARIO", "io-eio"); let default_percent = if scenario == "disk-full" { 100 } else { 20 }; + let workload = FaultWorkloadProfile::new( + env_usize( + &get_env, + "RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS", + DEFAULT_WORKLOAD_OBJECTS, + )?, + env_usize( + &get_env, + "RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY", + DEFAULT_WORKLOAD_CONCURRENCY, + )?, + )?; let cluster = ClusterTestConfig { context, operator_namespace: env_or( &get_env, "RUSTFS_FAULT_TEST_OPERATOR_NAMESPACE", - "rustfs-system", + DEFAULT_OPERATOR_NAMESPACE, ), test_namespace_prefix: namespace.clone(), test_namespace: namespace, - tenant_name: env_or(&get_env, "RUSTFS_FAULT_TEST_TENANT", "fault-test-tenant"), + tenant_name: env_or(&get_env, "RUSTFS_FAULT_TEST_TENANT", DEFAULT_FAULT_TENANT), storage_class, - rustfs_image: env_or( - &get_env, - "RUSTFS_FAULT_TEST_SERVER_IMAGE", - "rustfs/rustfs:latest", - ), + rustfs_image, artifacts_dir: PathBuf::from(env_or( &get_env, "RUSTFS_FAULT_TEST_ARTIFACTS", @@ -86,33 +149,34 @@ impl FaultTestConfig { timeout: Duration::from_secs(env_u64( &get_env, "RUSTFS_FAULT_TEST_TIMEOUT_SECONDS", - 300, - )), + DEFAULT_CLUSTER_TIMEOUT_SECONDS, + )?), }; Ok(Self { cluster, - destructive_enabled: env_bool(&get_env, "RUSTFS_FAULT_TEST_DESTRUCTIVE"), + expected_context, + destructive_enabled: env_bool(&get_env, "RUSTFS_FAULT_TEST_DESTRUCTIVE")?, scenario, duration: Duration::from_secs(env_u64( &get_env, "RUSTFS_FAULT_TEST_DURATION_SECONDS", - 7200, - )), - percent: env_u8(&get_env, "RUSTFS_FAULT_TEST_PERCENT", default_percent), - workload_objects: env_usize(&get_env, "RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS", 40000), - workload_concurrency: env_usize(&get_env, "RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY", 80), + DEFAULT_FAULT_DURATION_SECONDS, + )?), + percent: env_u8(&get_env, "RUSTFS_FAULT_TEST_PERCENT", default_percent)?, + percent_overridden: env_optional(&get_env, "RUSTFS_FAULT_TEST_PERCENT").is_some(), + workload, workload_seed: env_optional_u64(&get_env, "RUSTFS_FAULT_TEST_SEED")?, request_timeout: Duration::from_secs(env_u64( &get_env, "RUSTFS_FAULT_TEST_REQUEST_TIMEOUT_SECONDS", - 30, - )), - use_cluster_ip: env_bool(&get_env, "RUSTFS_FAULT_TEST_USE_CLUSTER_IP"), + DEFAULT_REQUEST_TIMEOUT_SECONDS, + )?), + use_cluster_ip: env_bool(&get_env, "RUSTFS_FAULT_TEST_USE_CLUSTER_IP")?, require_client_disruption: env_bool( &get_env, "RUSTFS_FAULT_TEST_REQUIRE_CLIENT_DISRUPTION", - ), + )?, dm_name: env_optional(&get_env, "RUSTFS_FAULT_TEST_DM_NAME"), dm_node: env_optional(&get_env, "RUSTFS_FAULT_TEST_DM_NODE"), dm_mount_path: env_optional(&get_env, "RUSTFS_FAULT_TEST_DM_MOUNT_PATH"), @@ -121,14 +185,18 @@ impl FaultTestConfig { dm_helper_image: env_or( &get_env, "RUSTFS_FAULT_TEST_DM_HELPER_IMAGE", - "rancher/mirrored-library-busybox:1.37.0", + DEFAULT_DM_HELPER_IMAGE, ), warp_duration: Duration::from_secs(env_u64( &get_env, "RUSTFS_FAULT_TEST_WARP_DURATION_SECONDS", - 60, - )), - chaos_namespace: env_or(&get_env, "RUSTFS_FAULT_TEST_CHAOS_NAMESPACE", "chaos-mesh"), + DEFAULT_WARP_DURATION_SECONDS, + )?), + chaos_namespace: env_or( + &get_env, + "RUSTFS_FAULT_TEST_CHAOS_NAMESPACE", + DEFAULT_CHAOS_NAMESPACE, + ), }) } @@ -169,6 +237,7 @@ impl FaultTestConfig { Self::from_env_with( |name| match name { "RUSTFS_FAULT_TEST_STORAGE_CLASS" => Some(storage_class.to_string()), + "RUSTFS_FAULT_TEST_SERVER_IMAGE" => Some("rustfs/rustfs:test".to_string()), _ => None, }, context.to_string(), @@ -221,32 +290,44 @@ fn env_optional(get_env: &F, name: &str) -> Option where F: Fn(&str) -> Option, { - get_env(name).filter(|value| !value.trim().is_empty()) + get_env(name) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) } -fn env_bool(get_env: &F, name: &str) -> bool +fn env_bool(get_env: &F, name: &str) -> Result where F: Fn(&str) -> Option, { - get_env(name) - .map(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES")) - .unwrap_or(false) + let Some(value) = env_optional(get_env, name) else { + return Ok(false); + }; + match value.to_ascii_lowercase().as_str() { + "1" | "true" | "yes" => Ok(true), + "0" | "false" | "no" => Ok(false), + _ => bail!("{name} must be a boolean: 1/0, true/false, or yes/no"), + } } -fn env_u64(get_env: &F, name: &str, default: u64) -> u64 +fn env_u64(get_env: &F, name: &str, default: u64) -> Result where F: Fn(&str) -> Option, { - get_env(name) - .and_then(|value| value.parse::().ok()) - .unwrap_or(default) + env_optional(get_env, name) + .map(|value| { + value + .parse::() + .with_context(|| format!("{name} must be an unsigned 64-bit integer")) + }) + .transpose() + .map(|value| value.unwrap_or(default)) } fn env_optional_u64(get_env: &F, name: &str) -> Result> where F: Fn(&str) -> Option, { - get_env(name) + env_optional(get_env, name) .map(|value| { value .parse::() @@ -255,22 +336,32 @@ where .transpose() } -fn env_usize(get_env: &F, name: &str, default: usize) -> usize +fn env_usize(get_env: &F, name: &str, default: usize) -> Result where F: Fn(&str) -> Option, { - get_env(name) - .and_then(|value| value.parse::().ok()) - .unwrap_or(default) + env_optional(get_env, name) + .map(|value| { + value + .parse::() + .with_context(|| format!("{name} must be an unsigned integer")) + }) + .transpose() + .map(|value| value.unwrap_or(default)) } -fn env_u8(get_env: &F, name: &str, default: u8) -> u8 +fn env_u8(get_env: &F, name: &str, default: u8) -> Result where F: Fn(&str) -> Option, { - get_env(name) - .and_then(|value| value.parse::().ok()) - .unwrap_or(default) + env_optional(get_env, name) + .map(|value| { + value + .parse::() + .with_context(|| format!("{name} must be an unsigned 8-bit integer")) + }) + .transpose() + .map(|value| value.unwrap_or(default)) } #[cfg(test)] @@ -282,6 +373,7 @@ mod tests { let config = FaultTestConfig::from_env_with( |name| match name { "RUSTFS_FAULT_TEST_STORAGE_CLASS" => Some("fast-csi".to_string()), + "RUSTFS_FAULT_TEST_SERVER_IMAGE" => Some("rustfs/rustfs:test".to_string()), _ => None, }, "production-test-cluster".to_string(), @@ -289,9 +381,11 @@ mod tests { .expect("fault config"); assert_eq!(config.cluster.context, "production-test-cluster"); + assert_eq!(config.expected_context, None); assert_eq!(config.cluster.test_namespace, "rustfs-fault-test"); assert_eq!(config.cluster.tenant_name, "fault-test-tenant"); assert_eq!(config.cluster.storage_class, "fast-csi"); + assert_eq!(config.cluster.rustfs_image, "rustfs/rustfs:test"); assert_eq!( config.cluster.artifacts_dir, std::path::PathBuf::from("target/fault-tests/artifacts") @@ -299,8 +393,9 @@ mod tests { assert_eq!(config.scenario, "io-eio"); assert_eq!(config.duration, std::time::Duration::from_secs(7200)); assert_eq!(config.percent, 20); - assert_eq!(config.workload_objects, 40000); - assert_eq!(config.workload_concurrency, 80); + assert!(!config.percent_overridden); + assert_eq!(config.workload.object_count, 40000); + assert_eq!(config.workload.concurrency, 80); assert_eq!(config.workload_seed, None); assert_eq!(config.request_timeout, std::time::Duration::from_secs(30)); assert!(!config.use_cluster_ip); @@ -323,6 +418,8 @@ mod tests { let config = FaultTestConfig::from_env_with( |name| match name { "RUSTFS_FAULT_TEST_STORAGE_CLASS" => Some("fast-csi".to_string()), + "RUSTFS_FAULT_TEST_SERVER_IMAGE" => Some("rustfs/rustfs:test".to_string()), + "RUSTFS_FAULT_TEST_EXPECTED_CONTEXT" => Some("production-test-cluster".to_string()), "RUSTFS_FAULT_TEST_SCENARIO" => Some("dm-flakey".to_string()), "RUSTFS_FAULT_TEST_DURATION_SECONDS" => Some("45".to_string()), "RUSTFS_FAULT_TEST_PERCENT" => Some("35".to_string()), @@ -349,11 +446,16 @@ mod tests { ) .expect("fault config"); + assert_eq!( + config.expected_context.as_deref(), + Some("production-test-cluster") + ); assert_eq!(config.scenario, "dm-flakey"); assert_eq!(config.duration, std::time::Duration::from_secs(45)); assert_eq!(config.percent, 35); - assert_eq!(config.workload_objects, 64); - assert_eq!(config.workload_concurrency, 8); + assert!(config.percent_overridden); + assert_eq!(config.workload.object_count, 64); + assert_eq!(config.workload.concurrency, 8); assert_eq!(config.workload_seed, Some(4242)); assert_eq!(config.request_timeout, std::time::Duration::from_secs(7)); assert!(config.use_cluster_ip); @@ -378,6 +480,7 @@ mod tests { let result = FaultTestConfig::from_env_with( |name| match name { "RUSTFS_FAULT_TEST_STORAGE_CLASS" => Some("local-storage".to_string()), + "RUSTFS_FAULT_TEST_SERVER_IMAGE" => Some("rustfs/rustfs:test".to_string()), _ => None, }, "kind-rustfs-e2e".to_string(), @@ -391,6 +494,7 @@ mod tests { let result = FaultTestConfig::from_env_with( |name| match name { "RUSTFS_FAULT_TEST_STORAGE_CLASS" => Some("fast-csi".to_string()), + "RUSTFS_FAULT_TEST_SERVER_IMAGE" => Some("rustfs/rustfs:test".to_string()), "RUSTFS_FAULT_TEST_SEED" => Some("not-a-number".to_string()), _ => None, }, @@ -400,6 +504,49 @@ mod tests { assert!(result.is_err()); } + #[test] + fn expected_context_is_optional_but_checked_when_set() { + let result = FaultTestConfig::from_env_with( + |name| match name { + "RUSTFS_FAULT_TEST_STORAGE_CLASS" => Some("fast-csi".to_string()), + "RUSTFS_FAULT_TEST_SERVER_IMAGE" => Some("rustfs/rustfs:test".to_string()), + "RUSTFS_FAULT_TEST_EXPECTED_CONTEXT" => Some("other-cluster".to_string()), + _ => None, + }, + "production-test-cluster".to_string(), + ); + + assert!(result.is_err()); + } + + #[test] + fn explicit_server_image_is_required() { + let result = FaultTestConfig::from_env_with( + |name| match name { + "RUSTFS_FAULT_TEST_STORAGE_CLASS" => Some("fast-csi".to_string()), + _ => None, + }, + "production-test-cluster".to_string(), + ); + + assert!(result.is_err()); + } + + #[test] + fn invalid_workload_numbers_are_rejected() { + let result = FaultTestConfig::from_env_with( + |name| match name { + "RUSTFS_FAULT_TEST_STORAGE_CLASS" => Some("fast-csi".to_string()), + "RUSTFS_FAULT_TEST_SERVER_IMAGE" => Some("rustfs/rustfs:test".to_string()), + "RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS" => Some("not-a-number".to_string()), + _ => None, + }, + "production-test-cluster".to_string(), + ); + + assert!(result.is_err()); + } + #[test] fn dynamic_storage_class_is_required() { assert!(validate_storage_class(r#"{"provisioner":"ebs.csi.aws.com"}"#, false).is_ok()); @@ -418,6 +565,7 @@ mod tests { let config = FaultTestConfig::from_env_with( |name| match name { "RUSTFS_FAULT_TEST_STORAGE_CLASS" => Some("fast-csi".to_string()), + "RUSTFS_FAULT_TEST_SERVER_IMAGE" => Some("rustfs/rustfs:test".to_string()), "RUSTFS_FAULT_TEST_SCENARIO" => Some("disk-full".to_string()), _ => None, }, diff --git a/e2e/src/framework/fault_plan.rs b/e2e/src/framework/fault_plan.rs index 4dbef31..76086c8 100644 --- a/e2e/src/framework/fault_plan.rs +++ b/e2e/src/framework/fault_plan.rs @@ -79,12 +79,27 @@ impl FaultTarget { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FaultSelection { + Percent(u8), + FixedTargets(u32), +} + +impl FaultSelection { + pub fn summary(self) -> String { + match self { + Self::Percent(percent) => format!("{percent}%"), + Self::FixedTargets(count) => format!("{count} target(s)"), + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct FaultInjection { kind: FaultKind, backend: FaultBackend, target: FaultTarget, - percent: u8, + selection: FaultSelection, duration: Duration, } @@ -93,7 +108,7 @@ impl FaultInjection { kind: FaultKind, backend: FaultBackend, target: FaultTarget, - percent: u8, + selection: FaultSelection, duration: Duration, ) -> Result { ensure!( @@ -103,8 +118,10 @@ impl FaultInjection { backend ); ensure!( - (1..=100).contains(&percent), - "fault percent must be in 1..=100, got {percent}" + fault_kind_accepts_selection(kind, selection), + "fault kind {} cannot run with selection {:?}", + kind.as_str(), + selection ); ensure!(duration > Duration::ZERO, "fault duration must be positive"); @@ -112,7 +129,7 @@ impl FaultInjection { kind, backend, target, - percent, + selection, duration, }) } @@ -129,8 +146,19 @@ impl FaultInjection { self.target } - pub fn percent(&self) -> u8 { - self.percent + pub fn selection(&self) -> FaultSelection { + self.selection + } + + pub fn percent(&self) -> Result { + match self.selection { + FaultSelection::Percent(percent) => Ok(percent), + other => bail!( + "fault kind {} requires a percent selection, got {:?}", + self.kind.as_str(), + other + ), + } } pub fn duration(&self) -> Duration { @@ -171,6 +199,23 @@ fn fault_kind_accepts_backend(kind: FaultKind, backend: FaultBackend) -> bool { ) } +fn fault_kind_accepts_selection(kind: FaultKind, selection: FaultSelection) -> bool { + match kind { + FaultKind::RustfsVolumeIoError + | FaultKind::RustfsVolumeReadMistake + | FaultKind::RustfsVolumeEnospc => match selection { + FaultSelection::Percent(percent) => (1..=100).contains(&percent), + FaultSelection::FixedTargets(_) => false, + }, + FaultKind::RustfsServerPodKill + | FaultKind::RustfsServerNetworkPartition + | FaultKind::RustfsBlockDeviceFlakey => match selection { + FaultSelection::FixedTargets(count) => count > 0, + FaultSelection::Percent(_) => false, + }, + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct FaultPlan { pub scenario: String, @@ -218,14 +263,14 @@ impl FaultPlan { FaultKind::RustfsServerPodKill, spec.backend, FaultTarget::RustfsServerPod, - scenario.percent, + FaultSelection::FixedTargets(1), scenario.duration, )?, NETWORK_PARTITION_ONE_SCENARIO => FaultInjection::new( FaultKind::RustfsServerNetworkPartition, spec.backend, FaultTarget::RustfsServerPeerNetwork, - scenario.percent, + FaultSelection::FixedTargets(1), scenario.duration, )?, IO_READ_MISTAKE_SCENARIO => { @@ -236,7 +281,7 @@ impl FaultPlan { FaultKind::RustfsBlockDeviceFlakey, spec.backend, FaultTarget::DedicatedBlockDevice, - scenario.percent, + FaultSelection::FixedTargets(1), scenario.duration, )?, WARP_UNDER_CHAOS_SCENARIO => { @@ -285,7 +330,13 @@ impl FaultPlan { pub fn target_summary(&self) -> String { self.faults .iter() - .map(|fault| fault.target().summary()) + .map(|fault| { + format!( + "{} via {}", + fault.target().summary(), + fault.selection().summary() + ) + }) .collect::>() .join(" + ") } @@ -302,7 +353,7 @@ fn volume_fault( FaultTarget::RustfsVolume { path: DEFAULT_RUSTFS_DATA_VOLUME, }, - scenario.percent, + FaultSelection::Percent(scenario.percent), scenario.duration, ) } @@ -310,8 +361,8 @@ fn volume_fault( #[cfg(test)] mod tests { use super::{ - DEFAULT_RUSTFS_DATA_VOLUME, FaultInjection, FaultKind, FaultPlan, FaultTarget, - FaultWorkloadMode, + DEFAULT_RUSTFS_DATA_VOLUME, FaultInjection, FaultKind, FaultPlan, FaultSelection, + FaultTarget, FaultWorkloadMode, }; use crate::framework::{ fault_config::FaultTestConfig, @@ -387,7 +438,7 @@ mod tests { FaultTarget::RustfsVolume { path: DEFAULT_RUSTFS_DATA_VOLUME, }, - 20, + FaultSelection::Percent(20), Duration::from_secs(60), ) .expect("first fault"); @@ -395,7 +446,7 @@ mod tests { FaultKind::RustfsServerNetworkPartition, FaultBackend::ChaosMeshNetworkChaos, FaultTarget::RustfsServerPeerNetwork, - 100, + FaultSelection::FixedTargets(1), Duration::from_secs(60), ) .expect("second fault"); @@ -427,7 +478,20 @@ mod tests { FaultTarget::RustfsVolume { path: DEFAULT_RUSTFS_DATA_VOLUME, }, - 20, + FaultSelection::Percent(20), + Duration::from_secs(60), + ); + + assert!(result.is_err()); + } + + #[test] + fn fixed_target_faults_reject_percent_selection() { + let result = FaultInjection::new( + FaultKind::RustfsServerPodKill, + FaultBackend::ChaosMeshPodChaos, + FaultTarget::RustfsServerPod, + FaultSelection::Percent(20), Duration::from_secs(60), ); diff --git a/e2e/src/framework/fault_scenarios.rs b/e2e/src/framework/fault_scenarios.rs index 7f83c03..d756971 100644 --- a/e2e/src/framework/fault_scenarios.rs +++ b/e2e/src/framework/fault_scenarios.rs @@ -47,6 +47,12 @@ pub enum FaultBackend { MinioWarpWithChaos, } +impl FaultBackend { + pub fn accepts_percent(self) -> bool { + matches!(self, Self::ChaosMeshIoChaos | Self::MinioWarpWithChaos) + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FaultIsolation { FreshTenant, @@ -209,14 +215,12 @@ impl FaultScenario { config.duration > Duration::ZERO, "RUSTFS_FAULT_TEST_DURATION_SECONDS must be greater than zero" ); + config.workload.validate()?; ensure!( - config.workload_objects >= 4, - "RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS must be at least 4" - ); - ensure!( - (1..=config.workload_objects).contains(&config.workload_concurrency), - "RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY must be between 1 and RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS ({})", - config.workload_objects + !config.percent_overridden || spec.backend.accepts_percent(), + "RUSTFS_FAULT_TEST_PERCENT only applies to percent-based IOChaos scenarios; scenario {:?} targets {:?} with a fixed target count", + spec.scenario, + spec.backend ); Ok(Self { @@ -224,7 +228,7 @@ impl FaultScenario { case_name: spec.case_name, duration: config.duration, percent: config.percent, - object_count: config.workload_objects, + object_count: config.workload.object_count, }) } @@ -257,8 +261,11 @@ pub fn scenario_spec(name: &str) -> Result<&'static FaultScenarioSpec> { #[cfg(test)] mod tests { - use super::{FaultScenario, FaultScenarioStatus, IO_EIO_SCENARIO, scenario_catalog}; - use crate::framework::fault_config::FaultTestConfig; + use super::{ + FaultScenario, FaultScenarioStatus, IO_EIO_SCENARIO, POD_KILL_ONE_SCENARIO, + scenario_catalog, + }; + use crate::framework::fault_config::{FaultTestConfig, FaultWorkloadProfile}; use std::time::Duration; #[test] @@ -288,8 +295,20 @@ mod tests { #[test] fn workload_concurrency_must_fit_the_object_count() { let mut config = FaultTestConfig::for_test("real-cluster", "fast-csi"); - config.workload_objects = 4; - config.workload_concurrency = 5; + config.workload = FaultWorkloadProfile { + object_count: 4, + concurrency: 5, + }; + + assert!(FaultScenario::from_config(&config).is_err()); + } + + #[test] + fn fixed_target_scenarios_reject_percent_override() { + let mut config = FaultTestConfig::for_test("real-cluster", "fast-csi"); + config.scenario = POD_KILL_ONE_SCENARIO.to_string(); + config.percent = 50; + config.percent_overridden = true; assert!(FaultScenario::from_config(&config).is_err()); } diff --git a/e2e/tests/faults.rs b/e2e/tests/faults.rs index 525a60b..2401a57 100644 --- a/e2e/tests/faults.rs +++ b/e2e/tests/faults.rs @@ -23,7 +23,7 @@ use rustfs_operator_e2e::framework::{ command::CommandSpec, config::ClusterTestConfig, fault_config::FaultTestConfig, - fault_plan::{FaultInjection, FaultKind, FaultPlan}, + fault_plan::{FaultInjection, FaultKind, FaultPlan, FaultSelection}, fault_scenarios::{self, FaultBackend, FaultIsolation, FaultScenario}, history::{OperationOutcome, OperationRecord, Recorder}, host_faults::{self, DmFlakeyGuard, DmFlakeySpec, DmStatusSnapshot}, @@ -166,7 +166,7 @@ async fn run_fault_case( let workload_plan = WorkloadPlan::seeded( workload_seed, scenario.object_count, - config.workload_concurrency, + config.workload.concurrency, ); let bucket = bucket_name(&run_id); let history_path = collector.case_dir(scenario.case_name).join("history.jsonl"); @@ -864,7 +864,7 @@ impl AppliedFault { run_id, &scenario.name, injection.rustfs_volume_path()?, - injection.percent(), + injection.percent()?, injection.duration(), )? .with_name_suffix(resource_name_suffix); @@ -881,7 +881,7 @@ impl AppliedFault { run_id, &scenario.name, injection.rustfs_volume_path()?, - injection.percent(), + injection.percent()?, injection.duration(), )? .with_name_suffix(resource_name_suffix); @@ -898,7 +898,7 @@ impl AppliedFault { run_id, &scenario.name, injection.rustfs_volume_path()?, - injection.percent(), + injection.percent()?, injection.duration(), )? .with_name_suffix(resource_name_suffix); @@ -1112,7 +1112,8 @@ struct RunMetadata { rustfs_image: String, artifacts_dir: String, duration_seconds: u64, - percent: u8, + percent: Option, + fault_selection: Vec, workload_objects: usize, workload_concurrency: usize, request_timeout_seconds: u64, @@ -1143,9 +1144,20 @@ impl RunMetadata { rustfs_image: config.cluster.rustfs_image.clone(), artifacts_dir: config.cluster.artifacts_dir.display().to_string(), duration_seconds: scenario.duration.as_secs(), - percent: scenario.percent, + percent: plan + .faults() + .iter() + .find_map(|fault| match fault.selection() { + FaultSelection::Percent(percent) => Some(percent), + FaultSelection::FixedTargets(_) => None, + }), + fault_selection: plan + .faults() + .iter() + .map(|fault| fault.selection().summary()) + .collect(), workload_objects: scenario.object_count, - workload_concurrency: config.workload_concurrency, + workload_concurrency: config.workload.concurrency, request_timeout_seconds: config.request_timeout.as_secs(), use_cluster_ip: config.use_cluster_ip, require_client_disruption: config.require_client_disruption, @@ -1976,7 +1988,7 @@ mod tests { pod_deletion_observed, pod_replacement_observed, stable_pod_fingerprint, warp_bucket_name, }; use rustfs_operator_e2e::framework::fault_plan::{ - DEFAULT_RUSTFS_DATA_VOLUME, FaultInjection, FaultKind, FaultTarget, + DEFAULT_RUSTFS_DATA_VOLUME, FaultInjection, FaultKind, FaultSelection, FaultTarget, }; use rustfs_operator_e2e::framework::fault_scenarios::FaultBackend; use rustfs_operator_e2e::framework::history::OperationOutcome; @@ -2002,7 +2014,7 @@ mod tests { FaultTarget::RustfsVolume { path: DEFAULT_RUSTFS_DATA_VOLUME, }, - 20, + FaultSelection::Percent(20), std::time::Duration::from_secs(60), ) .expect("valid fault"); From 4a2217b67580eb5f6eb6e229003522862ff8b95f Mon Sep 17 00:00:00 2001 From: GatewayJ <18332154+GatewayJ@users.noreply.github.com> Date: Sun, 21 Jun 2026 17:17:28 +0800 Subject: [PATCH 8/9] refactor(e2e): isolate fault test modules --- e2e/FAULT_TESTING.md | 8 +- e2e/README.md | 14 +- .../backends}/chaos_mesh.rs | 2 +- .../host_faults.rs => fault/backends/host.rs} | 2 +- e2e/src/fault/backends/mod.rs | 16 + e2e/src/{framework => fault}/checker.rs | 6 +- .../fault_config.rs => fault/config.rs} | 0 e2e/src/fault/fixture.rs | 198 ++ e2e/src/{framework => fault}/history.rs | 0 e2e/src/fault/mod.rs | 23 + .../fault_plan.rs => fault/plan.rs} | 8 +- e2e/src/fault/runner.rs | 2191 +++++++++++++++++ .../fault_scenarios.rs => fault/scenarios.rs} | 4 +- .../s3_workload.rs => fault/workload.rs} | 2 +- e2e/src/framework/mod.rs | 8 - e2e/src/framework/resources.rs | 168 +- e2e/src/lib.rs | 1 + e2e/tests/faults.rs | 2170 +--------------- 18 files changed, 2464 insertions(+), 2357 deletions(-) rename e2e/src/{framework => fault/backends}/chaos_mesh.rs (99%) rename e2e/src/{framework/host_faults.rs => fault/backends/host.rs} (99%) create mode 100644 e2e/src/fault/backends/mod.rs rename e2e/src/{framework => fault}/checker.rs (97%) rename e2e/src/{framework/fault_config.rs => fault/config.rs} (100%) create mode 100644 e2e/src/fault/fixture.rs rename e2e/src/{framework => fault}/history.rs (100%) create mode 100644 e2e/src/fault/mod.rs rename e2e/src/{framework/fault_plan.rs => fault/plan.rs} (99%) create mode 100644 e2e/src/fault/runner.rs rename e2e/src/{framework/fault_scenarios.rs => fault/scenarios.rs} (99%) rename e2e/src/{framework/s3_workload.rs => fault/workload.rs} (99%) diff --git a/e2e/FAULT_TESTING.md b/e2e/FAULT_TESTING.md index 3e4e4f0..2240c22 100644 --- a/e2e/FAULT_TESTING.md +++ b/e2e/FAULT_TESTING.md @@ -81,11 +81,17 @@ export RUSTFS_FAULT_TEST_EXPECTED_CONTEXT= ## Common Overrides -Defaults are centralized in `e2e/src/framework/fault_config.rs`. The shell +Defaults are centralized in `e2e/src/fault/config.rs`. The shell runner passes the same values into the Rust test and validates artifacts against the selected values. Shell preflight mirrors the Rust entrypoint for numeric ranges, booleans, and scenario-specific percent overrides. +Fault-test orchestration lives under `e2e/src/fault/`: runtime configuration, +scenario catalog, plan expansion, fault backends, fixture ownership checks, S3 +workload history, and the Rust runner. Shared Kubernetes wrappers, kubectl +command helpers, artifact collection, port-forwarding, and generic Tenant +resource cleanup remain under `e2e/src/framework/`. + | Variable | Default | Use | | --- | --- | --- | | `RUSTFS_FAULT_TEST_NAMESPACE` | `rustfs-fault-test` | Fault namespace. | diff --git a/e2e/README.md b/e2e/README.md index 837a1b7..d9198e8 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -6,10 +6,11 @@ The harness is intentionally separated from the main operator crate so e2e-only ## Architecture -The harness is split into four top-level domains: +The harness is split into five top-level domains: - `manifests/`: e2e-owned static manifests such as the dedicated Kind config. - `framework/`: reusable infrastructure primitives. +- `fault/`: real-cluster fault-test orchestration and fault-specific helpers. - `cases/`: release test-case inventory grouped by product boundary. - `tests/`: executable suite entrypoints; live tests are ignored by default and run only through explicit Make targets. @@ -24,9 +25,18 @@ e2e/ src/ lib.rs bin/rustfs-e2e.rs Makefile-internal helper for live workflow steps + fault/ + config.rs real-cluster fault-test configuration and safety checks + scenarios.rs fault scenario catalog + plan.rs fault plan expansion for one or more fault injections + runner.rs destructive fault-test orchestration + fixture.rs fault namespace ownership and real-cluster Tenant fixture + backends/ Chaos Mesh and host-side fault backends + workload.rs S3 workload generation and execution + history.rs workload operation recorder + checker.rs committed-object correctness checker framework/ config.rs dedicated Kind e2e configuration - fault_config.rs real-cluster fault-test configuration and safety checks command.rs safe subprocess wrapper for kind/docker/kubectl kind.rs Kind cluster lifecycle and host mount preparation kubectl.rs kubectl command construction boundary diff --git a/e2e/src/framework/chaos_mesh.rs b/e2e/src/fault/backends/chaos_mesh.rs similarity index 99% rename from e2e/src/framework/chaos_mesh.rs rename to e2e/src/fault/backends/chaos_mesh.rs index cf2012c..0cf0700 100644 --- a/e2e/src/framework/chaos_mesh.rs +++ b/e2e/src/fault/backends/chaos_mesh.rs @@ -668,7 +668,7 @@ impl Drop for ChaosGuard { #[cfg(test)] mod tests { use super::{IoChaosSpec, chaos_experiment_is_active}; - use crate::framework::fault_config::FaultTestConfig; + use crate::fault::config::FaultTestConfig; use std::time::Duration; #[test] diff --git a/e2e/src/framework/host_faults.rs b/e2e/src/fault/backends/host.rs similarity index 99% rename from e2e/src/framework/host_faults.rs rename to e2e/src/fault/backends/host.rs index 641420d..1cc25f2 100644 --- a/e2e/src/framework/host_faults.rs +++ b/e2e/src/fault/backends/host.rs @@ -516,7 +516,7 @@ mod tests { DmFlakeySpec, dm_helper_manifest, dm_resume_args, dm_suspend_args, helper_pod_name, pv_targets_node, validate_dm_spec, }; - use crate::framework::fault_config::FaultTestConfig; + use crate::fault::config::FaultTestConfig; #[test] fn dm_helper_is_pinned_to_one_node_and_host_root() { diff --git a/e2e/src/fault/backends/mod.rs b/e2e/src/fault/backends/mod.rs new file mode 100644 index 0000000..65d496b --- /dev/null +++ b/e2e/src/fault/backends/mod.rs @@ -0,0 +1,16 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod chaos_mesh; +pub mod host; diff --git a/e2e/src/framework/checker.rs b/e2e/src/fault/checker.rs similarity index 97% rename from e2e/src/framework/checker.rs rename to e2e/src/fault/checker.rs index 7f72b5f..111e7b9 100644 --- a/e2e/src/framework/checker.rs +++ b/e2e/src/fault/checker.rs @@ -17,9 +17,9 @@ use futures::{StreamExt, stream}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; -use crate::framework::{ +use crate::fault::{ history::{OperationKind, OperationOutcome, OperationRecord, Recorder}, - s3_workload::{ObjectSpec, S3WorkloadClient, sha256_hex}, + workload::{ObjectSpec, S3WorkloadClient, sha256_hex}, }; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -193,7 +193,7 @@ fn successful_corrupted_reads( #[cfg(test)] mod tests { use super::{CheckerReport, successful_corrupted_reads}; - use crate::framework::history::{OperationKind, OperationOutcome, OperationRecord}; + use crate::fault::history::{OperationKind, OperationOutcome, OperationRecord}; use std::collections::BTreeMap; fn record( diff --git a/e2e/src/framework/fault_config.rs b/e2e/src/fault/config.rs similarity index 100% rename from e2e/src/framework/fault_config.rs rename to e2e/src/fault/config.rs diff --git a/e2e/src/fault/fixture.rs b/e2e/src/fault/fixture.rs new file mode 100644 index 0000000..29fcfd0 --- /dev/null +++ b/e2e/src/fault/fixture.rs @@ -0,0 +1,198 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Context, Result, bail, ensure}; +use serde_json::Value; + +use crate::framework::{ + command::CommandOutput, + config::ClusterTestConfig, + kubectl::Kubectl, + resources::{ + credential_secret_manifest, credential_secret_name, + reset_tenant_resources as reset_generic_tenant_resources, + }, + tenant_factory::TenantTemplate, +}; + +const MANAGED_BY_LABEL: &str = "app.kubernetes.io/managed-by"; +const FAULT_TEST_MANAGER: &str = "rustfs-operator-fault-test"; +const FAULT_TEST_TENANT_ANNOTATION: &str = "rustfs.com/fault-test-tenant"; + +pub fn namespace_manifest(config: &ClusterTestConfig) -> String { + format!( + r#"apiVersion: v1 +kind: Namespace +metadata: + name: {namespace} + labels: + {managed_by_label}: {manager} + annotations: + {tenant_annotation}: {tenant_name} +"#, + namespace = config.test_namespace, + managed_by_label = MANAGED_BY_LABEL, + manager = FAULT_TEST_MANAGER, + tenant_annotation = FAULT_TEST_TENANT_ANNOTATION, + tenant_name = config.tenant_name, + ) +} + +pub fn tenant_manifest(config: &ClusterTestConfig) -> Result { + let template = TenantTemplate::real_cluster( + &config.test_namespace, + &config.tenant_name, + &config.rustfs_image, + &config.storage_class, + credential_secret_name(config), + ); + Ok(serde_yaml_ng::to_string(&template.build())?) +} + +pub fn apply_tenant_resources(config: &ClusterTestConfig) -> Result<()> { + let kubectl = Kubectl::new(config); + if !ensure_namespace_owned_or_absent(config)? { + kubectl + .create_yaml_command(namespace_manifest(config)) + .run_checked() + .with_context(|| { + format!( + "create dedicated fault-test namespace {:?}", + config.test_namespace + ) + })?; + } + kubectl + .apply_yaml_command(credential_secret_manifest(config)) + .run_checked()?; + kubectl + .apply_yaml_command(tenant_manifest(config)?) + .run_checked()?; + Ok(()) +} + +pub fn reset_tenant_resources(config: &ClusterTestConfig) -> Result<()> { + if !ensure_namespace_owned_or_absent(config)? { + return Ok(()); + } + reset_generic_tenant_resources(config) +} + +fn ensure_namespace_owned_or_absent(config: &ClusterTestConfig) -> Result { + let output = Kubectl::new(config) + .command(["get", "namespace", &config.test_namespace, "-o", "json"]) + .run()?; + + match output.code { + Some(0) => { + validate_namespace_ownership( + &output.stdout, + &config.test_namespace, + &config.tenant_name, + )?; + Ok(true) + } + _ if is_not_found(&output) => Ok(false), + _ => bail!( + "failed to inspect fault-test namespace {:?} before destructive operation\nexit: {:?}\nstdout:\n{}\nstderr:\n{}", + config.test_namespace, + output.code, + output.stdout, + output.stderr + ), + } +} + +fn validate_namespace_ownership(raw: &str, namespace: &str, tenant_name: &str) -> Result<()> { + let value = serde_json::from_str::(raw) + .with_context(|| format!("parse namespace {namespace:?} json"))?; + let manager = value + .pointer("/metadata/labels/app.kubernetes.io~1managed-by") + .and_then(Value::as_str); + let owned_tenant = value + .pointer("/metadata/annotations/rustfs.com~1fault-test-tenant") + .and_then(Value::as_str); + + ensure!( + manager == Some(FAULT_TEST_MANAGER) && owned_tenant == Some(tenant_name), + "refusing destructive fault-test operation in namespace {namespace:?}: expected label \ + {MANAGED_BY_LABEL}={FAULT_TEST_MANAGER:?} and annotation \ + {FAULT_TEST_TENANT_ANNOTATION}={tenant_name:?}, got manager={manager:?}, \ + tenant={owned_tenant:?}; use a dedicated namespace or explicitly label and annotate it \ + only after verifying that it contains no non-test workloads" + ); + Ok(()) +} + +fn is_not_found(output: &CommandOutput) -> bool { + output.stderr.contains("NotFound") + || output.stderr.contains("not found") + || output.stdout.contains("NotFound") + || output.stdout.contains("not found") +} + +#[cfg(test)] +mod tests { + use super::{namespace_manifest, tenant_manifest, validate_namespace_ownership}; + use crate::fault::config::FaultTestConfig; + + #[test] + fn fault_tenant_manifest_uses_real_cluster_defaults() { + let config = FaultTestConfig::for_test("real-cluster", "fast-csi"); + let manifest = tenant_manifest(&config.cluster).expect("fault tenant manifest"); + + assert!(manifest.contains("namespace: rustfs-fault-test")); + assert!(manifest.contains("storageClassName: fast-csi")); + assert!(manifest.contains("storage: 100Gi")); + assert!(!manifest.contains("rustfs-storage")); + assert!(!manifest.contains("RUSTFS_UNSAFE_BYPASS_DISK_CHECK")); + } + + #[test] + fn fault_namespace_manifest_records_destructive_test_ownership() { + let config = FaultTestConfig::for_test("real-cluster", "fast-csi"); + let manifest = namespace_manifest(&config.cluster); + + assert!(manifest.contains("name: rustfs-fault-test")); + assert!(manifest.contains("app.kubernetes.io/managed-by: rustfs-operator-fault-test")); + assert!(manifest.contains("rustfs.com/fault-test-tenant: fault-test-tenant")); + } + + #[test] + fn fault_namespace_ownership_requires_matching_manager_and_tenant() { + let owned = r#"{ + "metadata": { + "labels": { + "app.kubernetes.io/managed-by": "rustfs-operator-fault-test" + }, + "annotations": { + "rustfs.com/fault-test-tenant": "fault-test-tenant" + } + } + }"#; + assert!( + validate_namespace_ownership(owned, "rustfs-fault-test", "fault-test-tenant").is_ok() + ); + + let unowned = r#"{"metadata":{"labels":{},"annotations":{}}}"#; + assert!( + validate_namespace_ownership(unowned, "rustfs-fault-test", "fault-test-tenant") + .is_err() + ); + + assert!( + validate_namespace_ownership(owned, "rustfs-fault-test", "another-tenant").is_err() + ); + } +} diff --git a/e2e/src/framework/history.rs b/e2e/src/fault/history.rs similarity index 100% rename from e2e/src/framework/history.rs rename to e2e/src/fault/history.rs diff --git a/e2e/src/fault/mod.rs b/e2e/src/fault/mod.rs new file mode 100644 index 0000000..f559e04 --- /dev/null +++ b/e2e/src/fault/mod.rs @@ -0,0 +1,23 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod backends; +pub mod checker; +pub mod config; +pub mod fixture; +pub mod history; +pub mod plan; +pub mod runner; +pub mod scenarios; +pub mod workload; diff --git a/e2e/src/framework/fault_plan.rs b/e2e/src/fault/plan.rs similarity index 99% rename from e2e/src/framework/fault_plan.rs rename to e2e/src/fault/plan.rs index 76086c8..edf3355 100644 --- a/e2e/src/framework/fault_plan.rs +++ b/e2e/src/fault/plan.rs @@ -15,7 +15,7 @@ use anyhow::{Result, bail, ensure}; use std::time::Duration; -use crate::framework::fault_scenarios::{ +use crate::fault::scenarios::{ DISK_FULL_SCENARIO, DM_FLAKEY_SCENARIO, FaultBackend, FaultScenario, FaultScenarioSpec, IO_EIO_SCENARIO, IO_READ_MISTAKE_SCENARIO, NETWORK_PARTITION_ONE_SCENARIO, POD_KILL_ONE_SCENARIO, WARP_UNDER_CHAOS_SCENARIO, @@ -364,9 +364,9 @@ mod tests { DEFAULT_RUSTFS_DATA_VOLUME, FaultInjection, FaultKind, FaultPlan, FaultSelection, FaultTarget, FaultWorkloadMode, }; - use crate::framework::{ - fault_config::FaultTestConfig, - fault_scenarios::{ + use crate::fault::{ + config::FaultTestConfig, + scenarios::{ FaultBackend, FaultScenario, WARP_UNDER_CHAOS_SCENARIO, scenario_catalog, scenario_spec, }, }; diff --git a/e2e/src/fault/runner.rs b/e2e/src/fault/runner.rs new file mode 100644 index 0000000..04511c1 --- /dev/null +++ b/e2e/src/fault/runner.rs @@ -0,0 +1,2191 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + fault::{ + backends::{ + chaos_mesh::{self, ChaosGuard, IoChaosSpec, NetworkChaosSpec, PodChaosSpec}, + host::{self, DmFlakeyGuard, DmFlakeySpec, DmStatusSnapshot}, + }, + checker, + config::FaultTestConfig, + fixture, + history::{OperationOutcome, OperationRecord, Recorder}, + plan::{FaultInjection, FaultKind, FaultPlan, FaultSelection}, + scenarios::{self, FaultBackend, FaultIsolation, FaultScenario}, + workload::{ObjectSpec, S3WorkloadClient, WorkloadPlan, wait_for_s3_endpoint}, + }, + framework::{ + artifacts::ArtifactCollector, + command::CommandSpec, + config::ClusterTestConfig, + kube_client, + kubectl::Kubectl, + port_forward::{PortForwardGuard, PortForwardSpec}, + resources, wait, + }, +}; +use anyhow::{Context, Result, bail, ensure}; +use futures::{StreamExt, TryStreamExt, stream}; +use kube::Api; +use operator::types::v1alpha1::tenant::Tenant; +use serde::Serialize; +use std::collections::BTreeSet; +use std::thread::sleep; +use std::time::{Duration, Instant}; +use tokio::time::sleep as async_sleep; +use uuid::Uuid; + +const FAULT_TENANT_POD_COUNT: usize = 4; +const RUSTFS_POD_STABLE_WINDOW: Duration = Duration::from_secs(60); + +pub async fn run_selected_scenario_from_env() -> Result<()> { + let config = FaultTestConfig::from_env()?; + let scenario = FaultScenario::from_config(&config)?; + let spec = scenarios::scenario_spec(&scenario.name)?; + let plan = FaultPlan::from_scenario(&scenario, spec)?; + + config.require_destructive_enabled()?; + config.validate_cluster(plan.requires_static_storage())?; + eprintln!( + "running destructive RustFS fault scenario {} against real Kubernetes context: {}", + scenario.name, config.cluster.context + ); + + let collector = ArtifactCollector::new(&config.cluster.artifacts_dir); + let result = run_fault_case(&config, &collector, &scenario, &plan).await; + + if let Err(error) = &result { + write_failure_summary_if_absent( + &collector, + scenario.case_name, + FailureSummary::new(&scenario.name, "scenario", "unknown", error.to_string()), + ) + .ok(); + match collector.collect_kubernetes_snapshot(scenario.case_name, &config.cluster) { + Ok(report) => { + eprintln!( + "collected fault-test artifacts under {}", + report.dir.display() + ); + eprintln!("{}", report.diagnosis); + } + Err(artifact_error) => { + eprintln!("failed to collect fault-test artifacts after {error}: {artifact_error}"); + } + } + } + + result +} + +async fn run_fault_case( + config: &FaultTestConfig, + collector: &ArtifactCollector, + scenario: &FaultScenario, + plan: &FaultPlan, +) -> Result<()> { + let spec = scenarios::scenario_spec(&scenario.name)?; + if let Err(error) = require_fault_backends(config, plan) { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "fault-backend-preflight", + "environment_or_fault_backend", + error.to_string(), + ), + )?; + return Err(error); + } + if let Err(error) = cleanup_fault_backends(config, plan) { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "fault-backend-pre-cleanup", + "environment_or_fault_backend", + error.to_string(), + ), + )?; + return Err(error); + } + + if let Err(error) = prepare_fault_fixture(&config.cluster, spec.isolation) { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "fixture-prepare", + "test_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + if let Err(error) = wait_for_ready_tenant(&config.cluster).await { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "tenant-ready-before-fault", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + if let Err(error) = wait_for_stable_rustfs_pods(&config.cluster, RUSTFS_POD_STABLE_WINDOW).await + { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "pod-stability-before-fault", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + + let run_id = format!("run-{}", Uuid::new_v4()); + let workload_seed = config.workload_seed.unwrap_or_else(generated_seed); + let workload_plan = WorkloadPlan::seeded( + workload_seed, + scenario.object_count, + config.workload.concurrency, + ); + let bucket = bucket_name(&run_id); + let history_path = collector.case_dir(scenario.case_name).join("history.jsonl"); + let history = Recorder::create(history_path, &scenario.name, &run_id)?; + collector.write_text( + scenario.case_name, + "run-metadata.json", + &serde_json::to_string_pretty(&RunMetadata::from_case( + config, scenario, plan, &run_id, &bucket, + ))?, + )?; + collector.write_text( + scenario.case_name, + "workload-plan.json", + &serde_json::to_string_pretty(&workload_plan)?, + )?; + eprintln!( + "fault workload seed={} objects={} concurrency={} payload_bytes={}", + workload_plan.seed, + workload_plan.object_count, + workload_plan.concurrency, + workload_plan.total_payload_bytes + ); + + let cluster = &config.cluster; + let (endpoint, mut port_forward) = match s3_access(config) { + Ok(access) => access, + Err(error) => { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "s3-endpoint", + "test_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + }; + if let Err(error) = ensure_s3_access(&mut port_forward, cluster, &endpoint).await { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "initial-s3-access", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + + let (access_key, secret_key) = resources::test_credentials(); + let s3 = match S3WorkloadClient::new( + &endpoint, + &bucket, + access_key, + secret_key, + config.request_timeout, + ) + .await + { + Ok(client) => client, + Err(error) => { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "s3-client", + "test_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + }; + let bucket_outcome = match s3.create_bucket(&history).await { + Ok(outcome) => outcome, + Err(error) => { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "bucket-create", + "test_harness", + error.to_string(), + ), + )?; + return Err(error); + } + }; + if bucket_outcome != OperationOutcome::Ok { + let message = format!("fault workload bucket creation did not succeed: {bucket_outcome:?}"); + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "bucket-create", + "product_or_environment", + message.clone(), + ), + )?; + bail!("{message}"); + } + + let prefilled = match prefill_objects( + &s3, + &history, + &run_id, + &workload_plan, + scenario.prefill_count(), + ) + .await + { + Ok(prefilled) => prefilled, + Err(error) => { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "prefill", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + }; + let pods_before = match rustfs_pod_identities(cluster) { + Ok(pods) => pods, + Err(error) => { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "pod-identity-before-fault", + "test_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + }; + let mut fault = match AppliedFaults::apply(config, collector, scenario, plan, &run_id) { + Ok(fault) => fault, + Err(error) => { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "fault-apply", + "environment_or_fault_backend", + error.to_string(), + ), + )?; + return Err(error); + } + }; + + if let Err(error) = fault.wait_active(cluster.timeout) { + collect_fault_artifacts(collector, scenario.case_name, &fault, "wait-active-failed")?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "wait-active", + "environment_or_fault_backend", + error.to_string(), + ), + )?; + return Err(error); + } + let active_snapshots = fault.snapshots("active")?; + + if let Err(error) = ensure_s3_access(&mut port_forward, cluster, &endpoint).await { + collect_fault_artifacts(collector, scenario.case_name, &fault, "port-forward-failed")?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "s3-access-under-fault", + "environment_or_workload", + error.to_string(), + ), + )?; + return Err(error); + } + + if plan.workload_mode.runs_warp() { + let warp_bucket = warp_bucket_name(&run_id); + if let Err(error) = host::run_warp_mixed( + config.warp_duration, + collector, + scenario.case_name, + &endpoint, + &warp_bucket, + access_key, + secret_key, + ) { + collect_fault_artifacts(collector, scenario.case_name, &fault, "warp-failed")?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "warp-workload", + "workload_or_product", + error.to_string(), + ), + )?; + return Err(error); + } + + if let Err(error) = ensure_s3_access(&mut port_forward, cluster, &endpoint).await { + collect_fault_artifacts( + collector, + scenario.case_name, + &fault, + "post-warp-port-forward-failed", + )?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "post-warp-s3-access", + "environment_or_workload", + error.to_string(), + ), + )?; + return Err(error); + } + } + + let mut workload = match run_mixed_workload( + &s3, + &history, + &run_id, + &workload_plan, + &prefilled, + scenario.prefill_count(), + scenario.mixed_workload_count(), + ) + .await + { + Ok(workload) => workload, + Err(error) => { + collect_fault_artifacts(collector, scenario.case_name, &fault, "workload-failed")?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "mixed-workload", + "workload_or_product", + error.to_string(), + ), + )?; + return Err(error); + } + }; + collector.write_text( + scenario.case_name, + "workload-summary.json", + &serde_json::to_string_pretty(&workload.summary)?, + )?; + if let Err(error) = workload + .summary + .require_fault_evidence(config.require_client_disruption) + { + collect_fault_artifacts( + collector, + scenario.case_name, + &fault, + "workload-no-fault-evidence", + )?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "fault-evidence", + "test_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + if let Err(error) = fault.ensure_active("after fault workload") { + collect_fault_artifacts( + collector, + scenario.case_name, + &fault, + "workload-outlived-fault", + )?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "fault-still-active", + "test_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + let workload_snapshots = fault.snapshots("after-workload")?; + + if let Err(error) = fault.delete(cluster.timeout) { + collect_fault_artifacts(collector, scenario.case_name, &fault, "delete-failed")?; + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "fault-delete", + "environment_or_fault_backend", + error.to_string(), + ), + )?; + return Err(error); + } + + if let Err(error) = wait_for_ready_tenant(cluster).await { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "tenant-recovery", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + if let Err(error) = wait_for_stable_rustfs_pods(cluster, RUSTFS_POD_STABLE_WINDOW).await { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "pod-stability-after-recovery", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + let pods_after = rustfs_pod_identities(cluster)?; + if let Err(error) = ensure_s3_access(&mut port_forward, cluster, &endpoint).await { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "s3-access-after-recovery", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + let recovered_evidence = FaultEvidence { + scenario: scenario.name.clone(), + backend: plan.backend_summary(), + target: plan.target_summary(), + injected: true, + active_during_workload: true, + recovered: true, + client_disruptions: workload.summary.disrupted(), + workload_plan: workload_plan.clone(), + pods_before: pods_before.clone(), + pods_after: pods_after.clone(), + active_snapshots: active_snapshots.clone(), + workload_snapshots: workload_snapshots.clone(), + dm_recovery_snapshot: fault.recovery_dm_snapshot(), + }; + collector.write_text( + scenario.case_name, + "fault-evidence.json", + &serde_json::to_string_pretty(&recovered_evidence)?, + )?; + let recommit_report = recommit_unconfirmed_objects( + &s3, + &history, + &workload.unconfirmed_puts, + workload_plan.concurrency, + ) + .await; + collector.write_text( + scenario.case_name, + "recommit-report.json", + &serde_json::to_string_pretty(&recommit_report)?, + )?; + workload.summary.recommitted_after_recovery = recommit_report.committed; + collector.write_text( + scenario.case_name, + "workload-summary.json", + &serde_json::to_string_pretty(&workload.summary)?, + )?; + if recommit_report.has_failures() { + let message = recommit_report.failure_message(); + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "recommit-unconfirmed", + recommit_report.failure_classification(), + message.clone(), + ), + )?; + bail!("{message}"); + } + let report = checker::check_s3_history(&s3, &history, true, workload_plan.concurrency).await?; + collector.write_text( + scenario.case_name, + "checker-report.json", + &serde_json::to_string_pretty(&report)?, + )?; + let evidence = FaultEvidence { + scenario: scenario.name.clone(), + backend: plan.backend_summary(), + target: plan.target_summary(), + injected: true, + active_during_workload: true, + recovered: report.tenant_recovered, + client_disruptions: workload.summary.disrupted(), + workload_plan, + pods_before, + pods_after, + active_snapshots, + workload_snapshots, + dm_recovery_snapshot: fault.recovery_dm_snapshot(), + }; + collector.write_text( + scenario.case_name, + "fault-evidence.json", + &serde_json::to_string_pretty(&evidence)?, + )?; + if report.committed_puts != scenario.object_count { + let message = format!( + "fault scenario {} expected {} committed objects after recovery reconciliation, got {}", + scenario.name, scenario.object_count, report.committed_puts + ); + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "checker-committed-count", + "product_or_environment", + message.clone(), + ), + )?; + bail!("{message}"); + } + if let Err(error) = report.require_success() { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "checker-verdict", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } + + Ok(()) +} + +fn require_fault_backends(config: &FaultTestConfig, plan: &FaultPlan) -> Result<()> { + for backend in plan.required_backends() { + require_fault_backend(config, backend)?; + } + Ok(()) +} + +fn require_fault_backend(config: &FaultTestConfig, backend: FaultBackend) -> Result<()> { + let cluster = &config.cluster; + match backend { + FaultBackend::ChaosMeshIoChaos => chaos_mesh::require_iochaos_crd(cluster), + FaultBackend::MinioWarpWithChaos => { + chaos_mesh::require_iochaos_crd(cluster)?; + require_tool("warp", ["--help"]) + } + FaultBackend::ChaosMeshPodChaos => chaos_mesh::require_podchaos_crd(cluster), + FaultBackend::ChaosMeshNetworkChaos => chaos_mesh::require_networkchaos_crd(cluster), + FaultBackend::DeviceMapper => require_dm_flakey_preflight(config), + } +} + +fn require_tool(program: &'static str, args: I) -> Result<()> +where + I: IntoIterator, + S: Into, +{ + CommandSpec::new(program) + .args(args) + .run_checked() + .with_context(|| format!("{program} is required for the selected fault scenario"))?; + Ok(()) +} + +fn require_dm_flakey_preflight(config: &FaultTestConfig) -> Result<()> { + config + .dm_name + .as_deref() + .context("RUSTFS_FAULT_TEST_DM_NAME is required for dm-flakey")?; + config + .dm_node + .as_deref() + .context("RUSTFS_FAULT_TEST_DM_NODE is required for dm-flakey")?; + config + .dm_mount_path + .as_deref() + .context("RUSTFS_FAULT_TEST_DM_MOUNT_PATH is required for dm-flakey")?; + config + .dm_fault_table + .as_deref() + .context("RUSTFS_FAULT_TEST_DM_FAULT_TABLE is required for dm-flakey")?; + Ok(()) +} + +fn cleanup_fault_backends(config: &FaultTestConfig, plan: &FaultPlan) -> Result<()> { + for backend in plan.required_backends() { + cleanup_fault_backend(config, backend)?; + } + Ok(()) +} + +fn cleanup_fault_backend(config: &FaultTestConfig, backend: FaultBackend) -> Result<()> { + match backend { + FaultBackend::ChaosMeshIoChaos | FaultBackend::MinioWarpWithChaos => { + chaos_mesh::cleanup_managed_iochaos(&config.cluster, &config.chaos_namespace) + } + FaultBackend::ChaosMeshPodChaos => { + chaos_mesh::cleanup_managed_podchaos(&config.cluster, &config.chaos_namespace) + } + FaultBackend::ChaosMeshNetworkChaos => { + chaos_mesh::cleanup_managed_networkchaos(&config.cluster, &config.chaos_namespace) + } + FaultBackend::DeviceMapper => Ok(()), + } +} + +fn prepare_fault_fixture(config: &ClusterTestConfig, isolation: FaultIsolation) -> Result<()> { + match isolation { + FaultIsolation::ReusableTenant => fixture::apply_tenant_resources(config)?, + FaultIsolation::FreshTenant | FaultIsolation::DedicatedLinuxBlockDevice => { + fixture::reset_tenant_resources(config)?; + fixture::apply_tenant_resources(config)?; + } + } + Ok(()) +} + +enum AppliedFault { + Chaos { + guard: Box, + active_required: bool, + }, + PodKill { + guard: Box, + before_pods: Vec, + config: Box, + }, + DmFlakey(Box), +} + +struct AppliedFaults { + items: Vec, +} + +impl AppliedFaults { + fn apply( + config: &FaultTestConfig, + collector: &ArtifactCollector, + scenario: &FaultScenario, + plan: &FaultPlan, + run_id: &str, + ) -> Result { + ensure!( + !plan.faults().is_empty(), + "fault plan {} did not contain any faults", + plan.scenario + ); + + let total = plan.faults().len(); + let mut items = Vec::with_capacity(total); + for (index, injection) in plan.faults().iter().enumerate() { + let manifest_name = chaos_manifest_artifact_name(total, index, injection); + let resource_name_suffix = chaos_resource_name_suffix(total, index); + items.push(AppliedFault::apply_one( + config, + collector, + scenario, + injection, + run_id, + &manifest_name, + &resource_name_suffix, + )?); + } + + Ok(Self { items }) + } + + fn len(&self) -> usize { + self.items.len() + } + + fn wait_active(&self, timeout: Duration) -> Result<()> { + for fault in &self.items { + fault.wait_active(timeout)?; + } + Ok(()) + } + + fn ensure_active(&self, stage: &str) -> Result<()> { + for fault in &self.items { + fault.ensure_active(stage)?; + } + Ok(()) + } + + fn delete(&mut self, timeout: Duration) -> Result<()> { + for fault in self.items.iter_mut().rev() { + fault.delete(timeout)?; + } + Ok(()) + } + + fn snapshot(&self, stage: &str) -> Result { + ensure!( + self.items.len() == 1, + "single fault snapshot requested for {} applied faults", + self.items.len() + ); + self.items[0].snapshot(stage) + } + + fn snapshots(&self, stage: &str) -> Result> { + self.items + .iter() + .map(|fault| fault.snapshot(stage)) + .collect() + } + + fn recovery_dm_snapshot(&self) -> Option { + self.items + .iter() + .find_map(AppliedFault::recovery_dm_snapshot) + } + + fn chaos_guards(&self) -> Vec<&ChaosGuard> { + self.items + .iter() + .filter_map(AppliedFault::chaos_guard) + .collect() + } +} + +impl AppliedFault { + fn apply_one( + config: &FaultTestConfig, + collector: &ArtifactCollector, + scenario: &FaultScenario, + injection: &FaultInjection, + run_id: &str, + manifest_name: &str, + resource_name_suffix: &str, + ) -> Result { + let cluster = &config.cluster; + match injection.kind() { + FaultKind::RustfsVolumeEnospc => { + let chaos = IoChaosSpec::enospc_on_rustfs_volume( + cluster, + &config.chaos_namespace, + run_id, + &scenario.name, + injection.rustfs_volume_path()?, + injection.percent()?, + injection.duration(), + )? + .with_name_suffix(resource_name_suffix); + collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; + Ok(Self::Chaos { + guard: Box::new(chaos_mesh::apply_iochaos(cluster, &chaos)?), + active_required: true, + }) + } + FaultKind::RustfsVolumeReadMistake => { + let chaos = IoChaosSpec::read_mistake_on_rustfs_volume( + cluster, + &config.chaos_namespace, + run_id, + &scenario.name, + injection.rustfs_volume_path()?, + injection.percent()?, + injection.duration(), + )? + .with_name_suffix(resource_name_suffix); + collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; + Ok(Self::Chaos { + guard: Box::new(chaos_mesh::apply_iochaos(cluster, &chaos)?), + active_required: true, + }) + } + FaultKind::RustfsVolumeIoError => { + let chaos = IoChaosSpec::eio_on_rustfs_volume( + cluster, + &config.chaos_namespace, + run_id, + &scenario.name, + injection.rustfs_volume_path()?, + injection.percent()?, + injection.duration(), + )? + .with_name_suffix(resource_name_suffix); + collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; + Ok(Self::Chaos { + guard: Box::new(chaos_mesh::apply_iochaos(cluster, &chaos)?), + active_required: true, + }) + } + FaultKind::RustfsServerPodKill => { + let before_pods = rustfs_pod_identities(cluster)?; + let chaos = PodChaosSpec::kill_one_rustfs_pod( + cluster, + &config.chaos_namespace, + run_id, + &scenario.name, + ) + .with_name_suffix(resource_name_suffix); + collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; + Ok(Self::PodKill { + guard: Box::new(chaos_mesh::apply_podchaos(cluster, &chaos)?), + before_pods, + config: Box::new(cluster.clone()), + }) + } + FaultKind::RustfsServerNetworkPartition => { + let chaos = NetworkChaosSpec::partition_one_rustfs_pod( + cluster, + &config.chaos_namespace, + run_id, + &scenario.name, + injection.duration(), + )? + .with_name_suffix(resource_name_suffix); + collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; + Ok(Self::Chaos { + guard: Box::new(chaos_mesh::apply_networkchaos(cluster, &chaos)?), + active_required: true, + }) + } + FaultKind::RustfsBlockDeviceFlakey => { + let name = config + .dm_name + .as_deref() + .context("RUSTFS_FAULT_TEST_DM_NAME is required for dm-flakey")?; + let fault_table = config + .dm_fault_table + .as_deref() + .context("RUSTFS_FAULT_TEST_DM_FAULT_TABLE is required for dm-flakey")?; + let node = config + .dm_node + .as_deref() + .context("RUSTFS_FAULT_TEST_DM_NODE is required for dm-flakey")?; + let mount_path = config + .dm_mount_path + .as_deref() + .context("RUSTFS_FAULT_TEST_DM_MOUNT_PATH is required for dm-flakey")?; + Ok(Self::DmFlakey(Box::new(host::apply_dm_flakey( + cluster, + &DmFlakeySpec { + node, + mount_path, + helper_image: &config.dm_helper_image, + name, + fault_table, + recovery_table: config.dm_recovery_table.as_deref(), + run_id, + }, + collector, + scenario.case_name, + )?))) + } + } + } + + fn wait_active(&self, timeout: Duration) -> Result<()> { + match self { + Self::Chaos { + guard, + active_required, + } if *active_required => guard.wait_active(timeout), + Self::PodKill { + before_pods, + config, + .. + } => wait_for_rustfs_pod_deletion(config, before_pods, timeout), + Self::Chaos { .. } | Self::DmFlakey(_) => Ok(()), + } + } + + fn ensure_active(&self, stage: &str) -> Result<()> { + match self { + Self::Chaos { + guard, + active_required, + } if *active_required => guard.ensure_active(stage), + Self::PodKill { .. } | Self::Chaos { .. } => Ok(()), + Self::DmFlakey(guard) => { + guard.ensure_active("after fault workload")?; + Ok(()) + } + } + } + + fn delete(&mut self, timeout: Duration) -> Result<()> { + match self { + Self::Chaos { guard, .. } => guard.delete(), + Self::PodKill { + guard, + before_pods, + config, + } => { + guard.delete()?; + wait_for_rustfs_pod_replacement(config, before_pods, timeout) + } + Self::DmFlakey(guard) => guard.restore(), + } + } + + fn chaos_guard(&self) -> Option<&ChaosGuard> { + match self { + Self::Chaos { guard, .. } | Self::PodKill { guard, .. } => Some(guard.as_ref()), + Self::DmFlakey(_) => None, + } + } + + fn snapshot(&self, stage: &str) -> Result { + match self { + Self::Chaos { guard, .. } | Self::PodKill { guard, .. } => Ok(FaultStatusSnapshot { + stage: stage.to_string(), + resource_kind: Some(guard.kind().to_string()), + resource_name: Some(guard.name().to_string()), + chaos_status: Some(serde_json::from_str(&guard.json()?)?), + dm_status: None, + }), + Self::DmFlakey(guard) => Ok(FaultStatusSnapshot { + stage: stage.to_string(), + resource_kind: Some("device-mapper".to_string()), + resource_name: None, + chaos_status: None, + dm_status: Some(guard.snapshot(stage)?), + }), + } + } + + fn recovery_dm_snapshot(&self) -> Option { + match self { + Self::DmFlakey(guard) => guard.recovery_snapshot().cloned(), + Self::Chaos { .. } | Self::PodKill { .. } => None, + } + } +} + +fn chaos_manifest_artifact_name(total: usize, index: usize, injection: &FaultInjection) -> String { + if total == 1 { + "chaos-manifest.yaml".to_string() + } else { + format!( + "chaos-manifest-{index:02}-{}.yaml", + injection.kind().as_str() + ) + } +} + +fn chaos_resource_name_suffix(total: usize, index: usize) -> String { + if total == 1 { + String::new() + } else { + format!("-{index:02}") + } +} + +#[derive(Debug, Clone, Serialize)] +struct FaultStatusSnapshot { + stage: String, + resource_kind: Option, + resource_name: Option, + chaos_status: Option, + dm_status: Option, +} + +#[derive(Debug, Clone, Serialize)] +struct FaultEvidence { + scenario: String, + backend: String, + target: String, + injected: bool, + active_during_workload: bool, + recovered: bool, + client_disruptions: usize, + workload_plan: WorkloadPlan, + pods_before: Vec, + pods_after: Vec, + active_snapshots: Vec, + workload_snapshots: Vec, + dm_recovery_snapshot: Option, +} + +#[derive(Debug, Clone, Serialize)] +struct RunMetadata { + scenario: String, + case_name: String, + run_id: String, + bucket: String, + backend: String, + target: String, + context: String, + namespace: String, + tenant: String, + storage_class: String, + rustfs_image: String, + artifacts_dir: String, + duration_seconds: u64, + percent: Option, + fault_selection: Vec, + workload_objects: usize, + workload_concurrency: usize, + request_timeout_seconds: u64, + use_cluster_ip: bool, + require_client_disruption: bool, + chaos_namespace: String, +} + +impl RunMetadata { + fn from_case( + config: &FaultTestConfig, + scenario: &FaultScenario, + plan: &FaultPlan, + run_id: &str, + bucket: &str, + ) -> Self { + Self { + scenario: scenario.name.clone(), + case_name: scenario.case_name.to_string(), + run_id: run_id.to_string(), + bucket: bucket.to_string(), + backend: plan.backend_summary(), + target: plan.target_summary(), + context: config.cluster.context.clone(), + namespace: config.cluster.test_namespace.clone(), + tenant: config.cluster.tenant_name.clone(), + storage_class: config.cluster.storage_class.clone(), + rustfs_image: config.cluster.rustfs_image.clone(), + artifacts_dir: config.cluster.artifacts_dir.display().to_string(), + duration_seconds: scenario.duration.as_secs(), + percent: plan + .faults() + .iter() + .find_map(|fault| match fault.selection() { + FaultSelection::Percent(percent) => Some(percent), + FaultSelection::FixedTargets(_) => None, + }), + fault_selection: plan + .faults() + .iter() + .map(|fault| fault.selection().summary()) + .collect(), + workload_objects: scenario.object_count, + workload_concurrency: config.workload.concurrency, + request_timeout_seconds: config.request_timeout.as_secs(), + use_cluster_ip: config.use_cluster_ip, + require_client_disruption: config.require_client_disruption, + chaos_namespace: config.chaos_namespace.clone(), + } + } +} + +#[derive(Debug, Clone, Serialize)] +struct FailureSummary { + scenario: String, + stage: String, + classification: String, + message: String, +} + +impl FailureSummary { + fn new( + scenario: impl Into, + stage: impl Into, + classification: impl Into, + message: impl Into, + ) -> Self { + Self { + scenario: scenario.into(), + stage: stage.into(), + classification: classification.into(), + message: message.into(), + } + } +} + +fn write_failure_summary( + collector: &ArtifactCollector, + case_name: &str, + summary: FailureSummary, +) -> Result<()> { + collector.write_text( + case_name, + "failure-summary.json", + &serde_json::to_string_pretty(&summary)?, + )?; + Ok(()) +} + +fn write_failure_summary_if_absent( + collector: &ArtifactCollector, + case_name: &str, + summary: FailureSummary, +) -> Result<()> { + let path = collector.case_dir(case_name).join("failure-summary.json"); + if path.exists() { + return Ok(()); + } + write_failure_summary(collector, case_name, summary) +} + +fn collect_fault_artifacts( + collector: &ArtifactCollector, + case_name: &str, + fault: &AppliedFaults, + suffix: &str, +) -> Result<()> { + let status = if fault.len() == 1 { + fault + .snapshot(suffix) + .and_then(|snapshot| serde_json::to_string_pretty(&snapshot).map_err(Into::into)) + } else { + fault + .snapshots(suffix) + .and_then(|snapshots| serde_json::to_string_pretty(&snapshots).map_err(Into::into)) + } + .unwrap_or_else(|error| format!("failed to collect fault status: {error}")); + collector.write_text(case_name, &format!("fault-status-{suffix}.json"), &status)?; + + let guards = fault.chaos_guards(); + for (index, guard) in guards.iter().enumerate() { + let describe = guard + .describe() + .unwrap_or_else(|error| format!("failed to describe chaos before cleanup: {error}")); + let describe_name = + chaos_artifact_name(guards.len(), index, "chaos-describe", suffix, "txt"); + collector.write_text(case_name, &describe_name, &describe)?; + + let yaml = guard + .yaml() + .unwrap_or_else(|error| format!("failed to get chaos yaml before cleanup: {error}")); + let yaml_name = chaos_artifact_name(guards.len(), index, "chaos", suffix, "yaml"); + collector.write_text(case_name, &yaml_name, &yaml)?; + } + + Ok(()) +} + +fn chaos_artifact_name( + total: usize, + index: usize, + prefix: &str, + suffix: &str, + extension: &str, +) -> String { + if total == 1 { + format!("{prefix}-{suffix}.{extension}") + } else { + format!("{prefix}-{suffix}-{index:02}.{extension}") + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +struct PodIdentity { + name: String, + uid: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct PodRuntimeState { + name: String, + uid: String, + phase: String, + containers_ready: bool, + restart_count: u64, + terminating: bool, +} + +fn rustfs_pod_identities(config: &ClusterTestConfig) -> Result> { + let selector = format!("rustfs.tenant={}", config.tenant_name); + let output = Kubectl::new(config) + .namespaced(&config.test_namespace) + .command(["get", "pod", "-l", &selector, "-o", "json"]) + .run_checked()?; + let value = serde_json::from_str::(&output.stdout) + .context("parse RustFS pod list json")?; + let items = value + .pointer("/items") + .and_then(serde_json::Value::as_array) + .context("RustFS pod list did not contain an items array")?; + let pods = items + .iter() + .filter_map(|item| { + let metadata = item.get("metadata")?; + Some(PodIdentity { + name: metadata.get("name")?.as_str()?.to_string(), + uid: metadata.get("uid")?.as_str()?.to_string(), + }) + }) + .collect::>(); + ensure!( + !pods.is_empty(), + "no RustFS pods found for selector {selector} in namespace {}", + config.test_namespace + ); + Ok(pods) +} + +fn rustfs_pod_runtime_states(config: &ClusterTestConfig) -> Result> { + let selector = format!("rustfs.tenant={}", config.tenant_name); + let output = Kubectl::new(config) + .namespaced(&config.test_namespace) + .command(["get", "pod", "-l", &selector, "-o", "json"]) + .run_checked()?; + let value = serde_json::from_str::(&output.stdout) + .context("parse RustFS pod list json")?; + let items = value + .pointer("/items") + .and_then(serde_json::Value::as_array) + .context("RustFS pod list did not contain an items array")?; + let mut pods = items + .iter() + .map(|item| { + let metadata = item + .get("metadata") + .context("RustFS pod did not contain metadata")?; + let name = metadata + .get("name") + .and_then(serde_json::Value::as_str) + .context("RustFS pod metadata did not contain a name")?; + let uid = metadata + .get("uid") + .and_then(serde_json::Value::as_str) + .context("RustFS pod metadata did not contain a uid")?; + let phase = item + .pointer("/status/phase") + .and_then(serde_json::Value::as_str) + .unwrap_or("Unknown"); + let container_statuses = item + .pointer("/status/containerStatuses") + .and_then(serde_json::Value::as_array); + let containers_ready = container_statuses.is_some_and(|statuses| { + !statuses.is_empty() + && statuses.iter().all(|status| { + status + .get("ready") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + }) + }); + let restart_count = container_statuses + .into_iter() + .flatten() + .filter_map(|status| status.get("restartCount")) + .filter_map(serde_json::Value::as_u64) + .sum(); + + Ok(PodRuntimeState { + name: name.to_string(), + uid: uid.to_string(), + phase: phase.to_string(), + containers_ready, + restart_count, + terminating: metadata.get("deletionTimestamp").is_some(), + }) + }) + .collect::>>()?; + pods.sort_by(|left, right| left.name.cmp(&right.name)); + Ok(pods) +} + +fn stable_pod_fingerprint(pods: &[PodRuntimeState]) -> Option> { + if pods.len() != FAULT_TENANT_POD_COUNT + || pods + .iter() + .any(|pod| pod.phase != "Running" || !pod.containers_ready || pod.terminating) + { + return None; + } + + Some( + pods.iter() + .map(|pod| (pod.uid.clone(), pod.restart_count)) + .collect(), + ) +} + +async fn wait_for_stable_rustfs_pods( + config: &ClusterTestConfig, + stable_window: Duration, +) -> Result<()> { + let deadline = Instant::now() + config.timeout; + let mut stable_since = None; + let mut stable_fingerprint = None; + let mut last_snapshot = Vec::new(); + let mut last_error = "not checked yet".to_string(); + + eprintln!( + "waiting for {FAULT_TENANT_POD_COUNT} RustFS pods to remain ready without restarts for {stable_window:?}" + ); + loop { + if Instant::now() >= deadline { + bail!( + "timed out waiting for stable RustFS pods after {:?}\nlast: {last_snapshot:?}\nlast error: {last_error}", + config.timeout + ); + } + + match rustfs_pod_runtime_states(config) { + Ok(current) => { + if let Some(fingerprint) = stable_pod_fingerprint(¤t) { + if stable_fingerprint.as_ref() != Some(&fingerprint) { + stable_since = Some(Instant::now()); + stable_fingerprint = Some(fingerprint); + } + if stable_since.is_some_and(|started| started.elapsed() >= stable_window) { + eprintln!("RustFS pods remained stable for {stable_window:?}"); + return Ok(()); + } + } else { + stable_since = None; + stable_fingerprint = None; + } + last_snapshot = current; + last_error = "none".to_string(); + } + Err(error) => { + stable_since = None; + stable_fingerprint = None; + last_error = error.to_string(); + } + } + + async_sleep(Duration::from_secs(1)).await; + } +} + +fn wait_for_rustfs_pod_replacement( + config: &ClusterTestConfig, + before: &[PodIdentity], + timeout: Duration, +) -> Result<()> { + let deadline = Instant::now() + timeout; + let mut last_snapshot = Vec::new(); + let mut last_error = "not checked yet".to_string(); + + loop { + if Instant::now() >= deadline { + bail!( + "timed out waiting for PodChaos to replace a RustFS pod after {timeout:?}\nbefore: {before:?}\nlast: {last_snapshot:?}\nlast error: {last_error}", + ); + } + + match rustfs_pod_identities(config) { + Ok(current) => { + if pod_replacement_observed(before, ¤t) { + return Ok(()); + } + last_snapshot = current; + last_error = "none".to_string(); + } + Err(error) => { + last_error = error.to_string(); + } + } + + sleep(Duration::from_secs(1)); + } +} + +fn wait_for_rustfs_pod_deletion( + config: &ClusterTestConfig, + before: &[PodIdentity], + timeout: Duration, +) -> Result<()> { + let deadline = Instant::now() + timeout; + let mut last_snapshot = Vec::new(); + let mut last_error = "not checked yet".to_string(); + + loop { + if Instant::now() >= deadline { + bail!( + "timed out waiting for PodChaos to delete a RustFS pod after {timeout:?}\nbefore: {before:?}\nlast: {last_snapshot:?}\nlast error: {last_error}", + ); + } + + match rustfs_pod_identities(config) { + Ok(current) => { + if pod_deletion_observed(before, ¤t) { + return Ok(()); + } + last_snapshot = current; + last_error = "none".to_string(); + } + Err(error) => { + last_error = error.to_string(); + } + } + + sleep(Duration::from_millis(250)); + } +} + +fn pod_deletion_observed(before: &[PodIdentity], current: &[PodIdentity]) -> bool { + let current_uids = current + .iter() + .map(|pod| pod.uid.as_str()) + .collect::>(); + !before.is_empty() + && before + .iter() + .any(|pod| !current_uids.contains(pod.uid.as_str())) +} + +fn pod_replacement_observed(before: &[PodIdentity], current: &[PodIdentity]) -> bool { + if before.is_empty() || current.is_empty() { + return false; + } + + let before_uids = before + .iter() + .map(|pod| pod.uid.as_str()) + .collect::>(); + let current_uids = current + .iter() + .map(|pod| pod.uid.as_str()) + .collect::>(); + let old_uid_removed = before_uids.iter().any(|uid| !current_uids.contains(uid)); + let new_uid_added = current_uids.iter().any(|uid| !before_uids.contains(uid)); + + old_uid_removed && new_uid_added +} + +async fn wait_for_ready_tenant(config: &ClusterTestConfig) -> Result { + let client = kube_client::default_client().await?; + let tenants: Api = kube_client::tenant_api(client, &config.test_namespace); + wait::wait_for_tenant_ready(tenants, &config.tenant_name, config.timeout).await +} + +fn s3_access(config: &FaultTestConfig) -> Result<(String, Option)> { + let cluster = &config.cluster; + if config.use_cluster_ip { + let service = format!("{}-io", cluster.tenant_name); + let output = Kubectl::new(cluster) + .namespaced(&cluster.test_namespace) + .command([ + "get".to_string(), + "service".to_string(), + service.clone(), + "-o".to_string(), + "jsonpath={.spec.clusterIP}".to_string(), + ]) + .run_checked() + .with_context(|| format!("read ClusterIP for fault-test service {service:?}"))?; + let cluster_ip = output.stdout.trim(); + ensure!( + !cluster_ip.is_empty() && cluster_ip != "None", + "fault-test service {service:?} does not have a ClusterIP" + ); + let host = if cluster_ip.contains(':') { + format!("[{cluster_ip}]") + } else { + cluster_ip.to_string() + }; + return Ok((format!("http://{host}:9000"), None)); + } + + let spec = PortForwardSpec::tenant_io(&cluster.test_namespace, &cluster.tenant_name); + let endpoint = spec.local_base_url(); + Ok((endpoint, Some(PortForwardSpec::start_tenant_io(cluster)?))) +} + +async fn ensure_s3_access( + port_forward: &mut Option, + config: &ClusterTestConfig, + endpoint: &str, +) -> Result<()> { + if let Some(guard) = port_forward { + if guard.ensure_running().is_err() { + *guard = PortForwardSpec::start_tenant_io(config)?; + } + return wait_for_tenant_s3(guard, endpoint, config.timeout).await; + } + + wait_for_s3_endpoint(endpoint, config.timeout).await +} + +async fn wait_for_tenant_s3( + port_forward: &mut PortForwardGuard, + endpoint: &str, + timeout: Duration, +) -> Result<()> { + port_forward.ensure_running()?; + wait_for_s3_endpoint(endpoint, timeout) + .await + .with_context(|| { + format!( + "S3 port-forward was not ready; command: {}; log {}:\n{}", + port_forward.command_display(), + port_forward.log_path().display(), + port_forward.log_contents() + ) + }) +} + +async fn prefill_objects( + s3: &S3WorkloadClient, + history: &Recorder, + run_id: &str, + plan: &WorkloadPlan, + count: usize, +) -> Result> { + let tasks = (0..count).map(|index| { + let s3 = s3.clone(); + let history = history.clone(); + let run_id = run_id.to_string(); + let size_bytes = plan.size_at(index); + let seed = plan.seed; + async move { + let object = ObjectSpec::prepare_seeded(&run_id, index, size_bytes, seed); + let spec = object.spec.clone(); + let put_outcome = s3.put_object(&object, &history).await?; + ensure!( + put_outcome == OperationOutcome::Ok, + "prefill PUT failed before fault injection for key {}: {put_outcome:?}", + spec.key + ); + let head_outcome = s3.head_object(&spec.key, &history).await?; + ensure!( + head_outcome == OperationOutcome::Ok, + "prefill HEAD failed before fault injection for key {}: {head_outcome:?}", + spec.key + ); + Ok::<_, anyhow::Error>((index, spec)) + } + }); + let mut objects = stream::iter(tasks) + .buffer_unordered(plan.concurrency) + .try_collect::>() + .await?; + objects.sort_by_key(|(index, _)| *index); + + Ok(objects.into_iter().map(|(_, object)| object).collect()) +} + +async fn run_mixed_workload( + s3: &S3WorkloadClient, + history: &Recorder, + run_id: &str, + plan: &WorkloadPlan, + prefilled: &[ObjectSpec], + start_index: usize, + count: usize, +) -> Result { + let tasks = (0..count).map(|offset| { + let s3 = s3.clone(); + let history = history.clone(); + let run_id = run_id.to_string(); + let index = start_index + offset; + let size_bytes = plan.size_at(index); + let seed = plan.seed; + let existing = prefilled[offset % prefilled.len()].clone(); + async move { + let object = ObjectSpec::prepare_seeded(&run_id, index, size_bytes, seed); + let spec = object.spec.clone(); + let put_outcome = s3.put_object(&object, &history).await?; + let get_outcome = s3.get_object_result(&existing.key, &history).await?.outcome; + Ok::<_, anyhow::Error>(MixedTaskResult { + index, + object: spec, + put_outcome, + get_outcome, + }) + } + }); + let results = stream::iter(tasks) + .buffer_unordered(plan.concurrency) + .collect::>() + .await; + let mut completed = Vec::with_capacity(count); + for result in results { + completed.push(result?); + } + completed.sort_by_key(|result| result.index); + + let mut summary = WorkloadSummary::new(plan); + let mut unconfirmed_puts = Vec::new(); + for result in completed { + summary.puts.record(result.put_outcome); + summary.gets.record(result.get_outcome); + if result.put_outcome != OperationOutcome::Ok { + unconfirmed_puts.push(result.object); + } + } + + summary.require_exercised()?; + Ok(MixedWorkloadResult { + summary, + unconfirmed_puts, + }) +} + +async fn recommit_unconfirmed_objects( + s3: &S3WorkloadClient, + history: &Recorder, + objects: &[ObjectSpec], + concurrency: usize, +) -> RecommitReport { + let tasks = objects.iter().cloned().map(|object| { + let s3 = s3.clone(); + let history = history.clone(); + async move { + let prepared = object.prepare(); + match s3.put_object_record(&prepared, &history).await { + Ok(record) => RecommitAttempt::from_record(object, record), + Err(error) => { + RecommitAttempt::from_harness_error(object, format!("record PUT: {error}")) + } + } + } + }); + let mut attempts = stream::iter(tasks) + .buffer_unordered(concurrency) + .collect::>() + .await; + attempts.sort_by(|left, right| left.key.cmp(&right.key)); + RecommitReport::from_attempts(attempts) +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +struct RecommitReport { + attempted: usize, + committed: usize, + failed: usize, + harness_errors: usize, + attempts: Vec, +} + +impl RecommitReport { + fn from_attempts(attempts: Vec) -> Self { + let committed = attempts + .iter() + .filter(|attempt| attempt.outcome == Some(OperationOutcome::Ok)) + .count(); + let failed = attempts + .iter() + .filter(|attempt| attempt.is_s3_failure()) + .count(); + let harness_errors = attempts + .iter() + .filter(|attempt| attempt.is_harness_error()) + .count(); + Self { + attempted: attempts.len(), + committed, + failed, + harness_errors, + attempts, + } + } + + fn has_failures(&self) -> bool { + self.failed > 0 || self.harness_errors > 0 + } + + fn failure_classification(&self) -> &'static str { + if self.harness_errors > 0 { + "test_harness" + } else { + "product_or_environment" + } + } + + fn failure_message(&self) -> String { + let sample = self + .attempts + .iter() + .filter_map(RecommitAttempt::failure_sample) + .take(5) + .collect::>() + .join(", "); + format!( + "{} of {} previously unconfirmed PUTs did not commit after recovery; harness_errors={}{}", + self.failed, + self.attempted, + self.harness_errors, + if sample.is_empty() { + String::new() + } else { + format!("; sample: {sample}") + } + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +struct RecommitAttempt { + key: String, + size_bytes: usize, + sha256: String, + outcome: Option, + http_status: Option, + error: Option, + harness_error: Option, +} + +impl RecommitAttempt { + fn from_record(object: ObjectSpec, record: OperationRecord) -> Self { + Self { + key: object.key, + size_bytes: object.size_bytes, + sha256: object.sha256, + outcome: Some(record.outcome), + http_status: record.http_status, + error: record.error, + harness_error: None, + } + } + + fn from_harness_error(object: ObjectSpec, error: String) -> Self { + Self { + key: object.key, + size_bytes: object.size_bytes, + sha256: object.sha256, + outcome: None, + http_status: None, + error: None, + harness_error: Some(error), + } + } + + fn is_s3_failure(&self) -> bool { + matches!( + self.outcome, + Some(OperationOutcome::Failed | OperationOutcome::Timeout | OperationOutcome::Unknown) + ) + } + + fn is_harness_error(&self) -> bool { + self.harness_error.is_some() + } + + fn failure_sample(&self) -> Option { + if let Some(error) = &self.harness_error { + return Some(format!("{}=harness_error({error})", self.key)); + } + let outcome = self.outcome?; + if outcome == OperationOutcome::Ok { + return None; + } + let status = self + .http_status + .map(|status| format!(" status={status}")) + .unwrap_or_default(); + let error = self + .error + .as_ref() + .map(|error| format!(" error={error}")) + .unwrap_or_default(); + Some(format!("{}={outcome:?}{status}{error}", self.key)) + } +} + +#[derive(Debug)] +struct MixedTaskResult { + index: usize, + object: ObjectSpec, + put_outcome: OperationOutcome, + get_outcome: OperationOutcome, +} + +#[derive(Debug)] +struct MixedWorkloadResult { + summary: WorkloadSummary, + unconfirmed_puts: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +struct WorkloadSummary { + seed: u64, + object_count: usize, + concurrency: usize, + total_payload_bytes: u64, + puts: OutcomeCounts, + gets: OutcomeCounts, + recommitted_after_recovery: usize, +} + +impl WorkloadSummary { + fn new(plan: &WorkloadPlan) -> Self { + Self { + seed: plan.seed, + object_count: plan.object_count, + concurrency: plan.concurrency, + total_payload_bytes: plan.total_payload_bytes, + puts: OutcomeCounts::default(), + gets: OutcomeCounts::default(), + recommitted_after_recovery: 0, + } + } + + fn require_exercised(&self) -> Result<()> { + ensure!( + self.puts.total() > 0 && self.gets.total() > 0, + "fault workload did not exercise both PUT and GET paths: {self:?}" + ); + Ok(()) + } + + fn require_fault_evidence(&self, require_client_disruption: bool) -> Result<()> { + if require_client_disruption { + ensure!( + self.disrupted() > 0, + "fault was applied but the S3 workload observed no client-visible disrupted operation; increase RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS or RUSTFS_FAULT_TEST_PERCENT, or set RUSTFS_FAULT_TEST_REQUIRE_CLIENT_DISRUPTION=0 if this is expected" + ); + } else if self.disrupted() == 0 { + eprintln!( + "fault was applied, but the S3 workload observed no client-visible disrupted operation" + ); + } + Ok(()) + } + + fn disrupted(&self) -> usize { + self.puts.disrupted() + self.gets.disrupted() + } +} + +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)] +struct OutcomeCounts { + ok: usize, + failed: usize, + timeout: usize, + unknown: usize, +} + +impl OutcomeCounts { + fn record(&mut self, outcome: OperationOutcome) { + match outcome { + OperationOutcome::Ok => self.ok += 1, + OperationOutcome::Failed => self.failed += 1, + OperationOutcome::Timeout => self.timeout += 1, + OperationOutcome::Unknown => self.unknown += 1, + } + } + + fn total(&self) -> usize { + self.ok + self.failed + self.timeout + self.unknown + } + + fn disrupted(&self) -> usize { + self.failed + self.timeout + self.unknown + } +} + +fn bucket_name(run_id: &str) -> String { + let suffix = run_id + .chars() + .filter(|ch| ch.is_ascii_alphanumeric()) + .take(16) + .collect::() + .to_ascii_lowercase(); + format!("rustfs-fault-{suffix}") +} + +fn generated_seed() -> u64 { + let run = Uuid::new_v4(); + let mut bytes = [0; 8]; + bytes.copy_from_slice(&run.as_bytes()[..8]); + u64::from_le_bytes(bytes) +} + +fn warp_bucket_name(run_id: &str) -> String { + format!("{}-warp", bucket_name(run_id)) +} + +#[cfg(test)] +mod tests { + use super::{ + OutcomeCounts, PodIdentity, PodRuntimeState, RecommitAttempt, RecommitReport, + WorkloadSummary, bucket_name, chaos_manifest_artifact_name, chaos_resource_name_suffix, + pod_deletion_observed, pod_replacement_observed, stable_pod_fingerprint, warp_bucket_name, + }; + use crate::fault::history::OperationOutcome; + use crate::fault::plan::{ + DEFAULT_RUSTFS_DATA_VOLUME, FaultInjection, FaultKind, FaultSelection, FaultTarget, + }; + use crate::fault::scenarios::FaultBackend; + use crate::fault::workload::WorkloadPlan; + + #[test] + fn fault_bucket_name_is_s3_compatible_and_run_scoped() { + assert_eq!( + bucket_name("run-12345678-abcd-efgh"), + "rustfs-fault-run12345678abcde" + ); + assert_eq!( + warp_bucket_name("run-12345678-abcd-efgh"), + "rustfs-fault-run12345678abcde-warp" + ); + } + + #[test] + fn composite_fault_artifacts_and_resource_names_are_indexed() { + let injection = FaultInjection::new( + FaultKind::RustfsVolumeIoError, + FaultBackend::ChaosMeshIoChaos, + FaultTarget::RustfsVolume { + path: DEFAULT_RUSTFS_DATA_VOLUME, + }, + FaultSelection::Percent(20), + std::time::Duration::from_secs(60), + ) + .expect("valid fault"); + + assert_eq!( + chaos_manifest_artifact_name(1, 0, &injection), + "chaos-manifest.yaml" + ); + assert_eq!(chaos_resource_name_suffix(1, 0), ""); + assert_eq!( + chaos_manifest_artifact_name(2, 1, &injection), + "chaos-manifest-01-rustfs_volume_io_error.yaml" + ); + assert_eq!(chaos_resource_name_suffix(2, 1), "-01"); + } + + #[test] + fn workload_summary_counts_disrupted_operations() { + let mut summary = WorkloadSummary::new(&WorkloadPlan::seeded(42, 40000, 80)); + summary.puts.record(OperationOutcome::Ok); + summary.gets.record(OperationOutcome::Timeout); + + assert_eq!(summary.puts.total(), 1); + assert_eq!(summary.gets.total(), 1); + assert_eq!(summary.disrupted(), 1); + assert!(summary.require_exercised().is_ok()); + assert!(summary.require_fault_evidence(true).is_ok()); + } + + #[test] + fn workload_summary_can_require_fault_evidence() { + let summary = WorkloadSummary { + seed: 42, + object_count: 40000, + concurrency: 80, + total_payload_bytes: 20_337_459_200, + puts: OutcomeCounts { + ok: 1, + ..OutcomeCounts::default() + }, + gets: OutcomeCounts { + ok: 1, + ..OutcomeCounts::default() + }, + recommitted_after_recovery: 0, + }; + + assert!(summary.require_fault_evidence(false).is_ok()); + assert!(summary.require_fault_evidence(true).is_err()); + } + + #[test] + fn recommit_report_counts_and_summarizes_failed_attempts() { + let report = RecommitReport::from_attempts(vec![ + RecommitAttempt { + key: "object-a".to_string(), + size_bytes: 4096, + sha256: "sha-a".to_string(), + outcome: Some(OperationOutcome::Ok), + http_status: Some(200), + error: None, + harness_error: None, + }, + RecommitAttempt { + key: "object-b".to_string(), + size_bytes: 4096, + sha256: "sha-b".to_string(), + outcome: Some(OperationOutcome::Failed), + http_status: Some(503), + error: Some("service unavailable".to_string()), + harness_error: None, + }, + ]); + + assert_eq!(report.attempted, 2); + assert_eq!(report.committed, 1); + assert_eq!(report.failed, 1); + assert_eq!(report.harness_errors, 0); + assert!(report.has_failures()); + assert!( + report + .failure_message() + .contains("object-b=Failed status=503") + ); + assert_eq!(report.failure_classification(), "product_or_environment"); + } + + #[test] + fn recommit_report_separates_harness_errors_from_s3_failures() { + let report = RecommitReport::from_attempts(vec![RecommitAttempt { + key: "object-a".to_string(), + size_bytes: 4096, + sha256: "sha-a".to_string(), + outcome: None, + http_status: None, + error: None, + harness_error: Some("record PUT: disk full".to_string()), + }]); + + assert_eq!(report.attempted, 1); + assert_eq!(report.committed, 0); + assert_eq!(report.failed, 0); + assert_eq!(report.harness_errors, 1); + assert!(report.has_failures()); + assert_eq!(report.failure_classification(), "test_harness"); + assert!( + report + .failure_message() + .contains("object-a=harness_error(record PUT: disk full)") + ); + } + + #[test] + fn pod_replacement_requires_old_uid_removed_and_new_uid_added() { + let before = vec![ + PodIdentity { + name: "rustfs-0".to_string(), + uid: "uid-a".to_string(), + }, + PodIdentity { + name: "rustfs-1".to_string(), + uid: "uid-b".to_string(), + }, + ]; + + assert!(!pod_replacement_observed(&before, &before)); + assert!(!pod_replacement_observed(&before, &before[..1])); + assert!(!pod_deletion_observed(&before, &before)); + assert!(pod_deletion_observed(&before, &before[..1])); + assert!(pod_replacement_observed( + &before, + &[ + PodIdentity { + name: "rustfs-0".to_string(), + uid: "uid-c".to_string(), + }, + before[1].clone(), + ], + )); + } + + #[test] + fn stable_pod_fingerprint_requires_four_ready_unchanged_pods() { + let pods = (0..4) + .map(|index| PodRuntimeState { + name: format!("rustfs-{index}"), + uid: format!("uid-{index}"), + phase: "Running".to_string(), + containers_ready: true, + restart_count: index, + terminating: false, + }) + .collect::>(); + + assert_eq!( + stable_pod_fingerprint(&pods), + Some(vec![ + ("uid-0".to_string(), 0), + ("uid-1".to_string(), 1), + ("uid-2".to_string(), 2), + ("uid-3".to_string(), 3), + ]) + ); + assert!(stable_pod_fingerprint(&pods[..3]).is_none()); + + let mut unready = pods; + unready[0].containers_ready = false; + assert!(stable_pod_fingerprint(&unready).is_none()); + } +} diff --git a/e2e/src/framework/fault_scenarios.rs b/e2e/src/fault/scenarios.rs similarity index 99% rename from e2e/src/framework/fault_scenarios.rs rename to e2e/src/fault/scenarios.rs index d756971..a920d36 100644 --- a/e2e/src/framework/fault_scenarios.rs +++ b/e2e/src/fault/scenarios.rs @@ -15,7 +15,7 @@ use anyhow::{Result, ensure}; use std::time::Duration; -use crate::framework::fault_config::FaultTestConfig; +use crate::fault::config::FaultTestConfig; pub const IO_EIO_SCENARIO: &str = "io-eio"; pub const POD_KILL_ONE_SCENARIO: &str = "pod-kill-one"; @@ -265,7 +265,7 @@ mod tests { FaultScenario, FaultScenarioStatus, IO_EIO_SCENARIO, POD_KILL_ONE_SCENARIO, scenario_catalog, }; - use crate::framework::fault_config::{FaultTestConfig, FaultWorkloadProfile}; + use crate::fault::config::{FaultTestConfig, FaultWorkloadProfile}; use std::time::Duration; #[test] diff --git a/e2e/src/framework/s3_workload.rs b/e2e/src/fault/workload.rs similarity index 99% rename from e2e/src/framework/s3_workload.rs rename to e2e/src/fault/workload.rs index fa1e3fa..6c2bb32 100644 --- a/e2e/src/framework/s3_workload.rs +++ b/e2e/src/fault/workload.rs @@ -21,7 +21,7 @@ use sha2::{Digest, Sha256}; use std::time::Duration; use tokio::time::timeout; -use crate::framework::history::{OperationKind, OperationOutcome, OperationRecord, Recorder}; +use crate::fault::history::{OperationKind, OperationOutcome, OperationRecord, Recorder}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct ObjectSpec { diff --git a/e2e/src/framework/mod.rs b/e2e/src/framework/mod.rs index c638034..4d8f32f 100644 --- a/e2e/src/framework/mod.rs +++ b/e2e/src/framework/mod.rs @@ -15,17 +15,10 @@ pub mod artifacts; pub mod assertions; pub mod cert_manager_tls; -pub mod chaos_mesh; -pub mod checker; pub mod command; pub mod config; pub mod console_client; pub mod deploy; -pub mod fault_config; -pub mod fault_plan; -pub mod fault_scenarios; -pub mod history; -pub mod host_faults; pub mod images; pub mod kind; pub mod kube_client; @@ -33,7 +26,6 @@ pub mod kubectl; pub mod live; pub mod port_forward; pub mod resources; -pub mod s3_workload; pub mod storage; pub mod tenant_factory; pub mod tools; diff --git a/e2e/src/framework/resources.rs b/e2e/src/framework/resources.rs index 40b51a2..5288547 100644 --- a/e2e/src/framework/resources.rs +++ b/e2e/src/framework/resources.rs @@ -12,8 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::{Context, Result, bail, ensure}; -use serde_json::Value; +use anyhow::{Context, Result, bail}; use std::thread::sleep; use std::time::{Duration, Instant}; @@ -29,9 +28,6 @@ const TEST_ACCESS_KEY: &str = "testaccess"; const TEST_SECRET_KEY: &str = "testsecret"; const RESOURCE_RESET_TIMEOUT: Duration = Duration::from_secs(120); const RESOURCE_RESET_POLL_INTERVAL: Duration = Duration::from_secs(2); -const MANAGED_BY_LABEL: &str = "app.kubernetes.io/managed-by"; -const FAULT_TEST_MANAGER: &str = "rustfs-operator-fault-test"; -const FAULT_TEST_TENANT_ANNOTATION: &str = "rustfs.com/fault-test-tenant"; pub fn credential_secret_name(config: &ClusterTestConfig) -> String { format!("{}-credentials", config.tenant_name) @@ -51,25 +47,6 @@ metadata: ) } -pub fn fault_namespace_manifest(config: &ClusterTestConfig) -> String { - format!( - r#"apiVersion: v1 -kind: Namespace -metadata: - name: {namespace} - labels: - {managed_by_label}: {manager} - annotations: - {tenant_annotation}: {tenant_name} -"#, - namespace = config.test_namespace, - managed_by_label = MANAGED_BY_LABEL, - manager = FAULT_TEST_MANAGER, - tenant_annotation = FAULT_TEST_TENANT_ANNOTATION, - tenant_name = config.tenant_name, - ) -} - pub fn credential_secret_manifest(config: &ClusterTestConfig) -> String { format!( r#"apiVersion: v1 @@ -114,17 +91,6 @@ pub fn smoke_tenant_manifest(config: &ClusterTestConfig) -> Result { )?) } -pub fn fault_tenant_manifest(config: &ClusterTestConfig) -> Result { - let template = TenantTemplate::real_cluster( - &config.test_namespace, - &config.tenant_name, - &config.rustfs_image, - &config.storage_class, - credential_secret_name(config), - ); - Ok(serde_yaml_ng::to_string(&template.build())?) -} - pub fn apply_smoke_tenant_resources(config: &ClusterTestConfig) -> Result<()> { let kubectl = Kubectl::new(config); kubectl @@ -139,35 +105,6 @@ pub fn apply_smoke_tenant_resources(config: &ClusterTestConfig) -> Result<()> { Ok(()) } -pub fn apply_fault_tenant_resources(config: &ClusterTestConfig) -> Result<()> { - let kubectl = Kubectl::new(config); - if !ensure_fault_namespace_owned_or_absent(config)? { - kubectl - .create_yaml_command(fault_namespace_manifest(config)) - .run_checked() - .with_context(|| { - format!( - "create dedicated fault-test namespace {:?}", - config.test_namespace - ) - })?; - } - kubectl - .apply_yaml_command(credential_secret_manifest(config)) - .run_checked()?; - kubectl - .apply_yaml_command(fault_tenant_manifest(config)?) - .run_checked()?; - Ok(()) -} - -pub fn reset_fault_tenant_resources(config: &ClusterTestConfig) -> Result<()> { - if !ensure_fault_namespace_owned_or_absent(config)? { - return Ok(()); - } - reset_tenant_resources(config) -} - pub fn reset_and_apply_smoke_tenant_resources(config: &ClusterTestConfig) -> Result<()> { reset_tenant_resources(config)?; apply_smoke_tenant_resources(config) @@ -286,52 +223,6 @@ fn namespace_exists(kubectl: &Kubectl, namespace: &str) -> Result { Ok(output.code == Some(0)) } -fn ensure_fault_namespace_owned_or_absent(config: &ClusterTestConfig) -> Result { - let output = Kubectl::new(config) - .command(["get", "namespace", &config.test_namespace, "-o", "json"]) - .run()?; - - match output.code { - Some(0) => { - validate_fault_namespace_ownership( - &output.stdout, - &config.test_namespace, - &config.tenant_name, - )?; - Ok(true) - } - _ if is_not_found(&output) => Ok(false), - _ => bail!( - "failed to inspect fault-test namespace {:?} before destructive operation\nexit: {:?}\nstdout:\n{}\nstderr:\n{}", - config.test_namespace, - output.code, - output.stdout, - output.stderr - ), - } -} - -fn validate_fault_namespace_ownership(raw: &str, namespace: &str, tenant_name: &str) -> Result<()> { - let value = serde_json::from_str::(raw) - .with_context(|| format!("parse namespace {namespace:?} json"))?; - let manager = value - .pointer("/metadata/labels/app.kubernetes.io~1managed-by") - .and_then(Value::as_str); - let owned_tenant = value - .pointer("/metadata/annotations/rustfs.com~1fault-test-tenant") - .and_then(Value::as_str); - - ensure!( - manager == Some(FAULT_TEST_MANAGER) && owned_tenant == Some(tenant_name), - "refusing destructive fault-test operation in namespace {namespace:?}: expected label \ - {MANAGED_BY_LABEL}={FAULT_TEST_MANAGER:?} and annotation \ - {FAULT_TEST_TENANT_ANNOTATION}={tenant_name:?}, got manager={manager:?}, \ - tenant={owned_tenant:?}; use a dedicated namespace or explicitly label and annotate it \ - only after verifying that it contains no non-test workloads" - ); - Ok(()) -} - fn run_delete(command: CommandSpec) -> Result<()> { command.run_checked()?; Ok(()) @@ -413,12 +304,8 @@ fn is_not_found(output: &CommandOutput) -> bool { #[cfg(test)] mod tests { - use super::{ - credential_secret_manifest, credential_secret_name, fault_namespace_manifest, - fault_tenant_manifest, smoke_tenant_manifest, validate_fault_namespace_ownership, - }; + use super::{credential_secret_manifest, credential_secret_name, smoke_tenant_manifest}; use crate::framework::config::E2eConfig; - use crate::framework::fault_config::FaultTestConfig; #[test] fn smoke_tenant_manifest_wires_secret_storage_and_image() { @@ -442,55 +329,4 @@ mod tests { assert!(manifest.contains("accesskey:")); assert!(manifest.contains("secretkey:")); } - - #[test] - fn fault_tenant_manifest_uses_real_cluster_defaults() { - let config = FaultTestConfig::for_test("real-cluster", "fast-csi"); - let manifest = fault_tenant_manifest(&config.cluster).expect("fault tenant manifest"); - - assert!(manifest.contains("namespace: rustfs-fault-test")); - assert!(manifest.contains("storageClassName: fast-csi")); - assert!(manifest.contains("storage: 100Gi")); - assert!(!manifest.contains("rustfs-storage")); - assert!(!manifest.contains("RUSTFS_UNSAFE_BYPASS_DISK_CHECK")); - } - - #[test] - fn fault_namespace_manifest_records_destructive_test_ownership() { - let config = FaultTestConfig::for_test("real-cluster", "fast-csi"); - let manifest = fault_namespace_manifest(&config.cluster); - - assert!(manifest.contains("name: rustfs-fault-test")); - assert!(manifest.contains("app.kubernetes.io/managed-by: rustfs-operator-fault-test")); - assert!(manifest.contains("rustfs.com/fault-test-tenant: fault-test-tenant")); - } - - #[test] - fn fault_namespace_ownership_requires_matching_manager_and_tenant() { - let owned = r#"{ - "metadata": { - "labels": { - "app.kubernetes.io/managed-by": "rustfs-operator-fault-test" - }, - "annotations": { - "rustfs.com/fault-test-tenant": "fault-test-tenant" - } - } - }"#; - assert!( - validate_fault_namespace_ownership(owned, "rustfs-fault-test", "fault-test-tenant") - .is_ok() - ); - - let unowned = r#"{"metadata":{"labels":{},"annotations":{}}}"#; - assert!( - validate_fault_namespace_ownership(unowned, "rustfs-fault-test", "fault-test-tenant") - .is_err() - ); - - assert!( - validate_fault_namespace_ownership(owned, "rustfs-fault-test", "another-tenant") - .is_err() - ); - } } diff --git a/e2e/src/lib.rs b/e2e/src/lib.rs index fd09500..bb5eec3 100644 --- a/e2e/src/lib.rs +++ b/e2e/src/lib.rs @@ -15,4 +15,5 @@ #![forbid(unsafe_code)] pub mod cases; +pub mod fault; pub mod framework; diff --git a/e2e/tests/faults.rs b/e2e/tests/faults.rs index 2401a57..edc5967 100644 --- a/e2e/tests/faults.rs +++ b/e2e/tests/faults.rs @@ -12,2176 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::{Context, Result, bail, ensure}; -use futures::{StreamExt, TryStreamExt, stream}; -use kube::Api; -use operator::types::v1alpha1::tenant::Tenant; -use rustfs_operator_e2e::framework::{ - artifacts::ArtifactCollector, - chaos_mesh::{self, ChaosGuard, IoChaosSpec, NetworkChaosSpec, PodChaosSpec}, - checker, - command::CommandSpec, - config::ClusterTestConfig, - fault_config::FaultTestConfig, - fault_plan::{FaultInjection, FaultKind, FaultPlan, FaultSelection}, - fault_scenarios::{self, FaultBackend, FaultIsolation, FaultScenario}, - history::{OperationOutcome, OperationRecord, Recorder}, - host_faults::{self, DmFlakeyGuard, DmFlakeySpec, DmStatusSnapshot}, - kube_client, - kubectl::Kubectl, - port_forward::{PortForwardGuard, PortForwardSpec}, - resources, - s3_workload::{ObjectSpec, S3WorkloadClient, WorkloadPlan, wait_for_s3_endpoint}, - wait, -}; -use serde::Serialize; -use std::collections::BTreeSet; -use std::thread::sleep; -use std::time::{Duration, Instant}; -use tokio::time::sleep as async_sleep; -use uuid::Uuid; - -const FAULT_TENANT_POD_COUNT: usize = 4; -const RUSTFS_POD_STABLE_WINDOW: Duration = Duration::from_secs(60); +use anyhow::Result; #[tokio::test] #[ignore = "destructive RustFS workload fault scenario; select with RUSTFS_FAULT_TEST_SCENARIO"] async fn fault_selected_scenario() -> Result<()> { - let config = FaultTestConfig::from_env()?; - let scenario = FaultScenario::from_config(&config)?; - let spec = fault_scenarios::scenario_spec(&scenario.name)?; - let plan = FaultPlan::from_scenario(&scenario, spec)?; - - config.require_destructive_enabled()?; - config.validate_cluster(plan.requires_static_storage())?; - eprintln!( - "running destructive RustFS fault scenario {} against real Kubernetes context: {}", - scenario.name, config.cluster.context - ); - - let collector = ArtifactCollector::new(&config.cluster.artifacts_dir); - let result = run_fault_case(&config, &collector, &scenario, &plan).await; - - if let Err(error) = &result { - write_failure_summary_if_absent( - &collector, - scenario.case_name, - FailureSummary::new(&scenario.name, "scenario", "unknown", error.to_string()), - ) - .ok(); - match collector.collect_kubernetes_snapshot(scenario.case_name, &config.cluster) { - Ok(report) => { - eprintln!( - "collected fault-test artifacts under {}", - report.dir.display() - ); - eprintln!("{}", report.diagnosis); - } - Err(artifact_error) => { - eprintln!("failed to collect fault-test artifacts after {error}: {artifact_error}"); - } - } - } - - result -} - -async fn run_fault_case( - config: &FaultTestConfig, - collector: &ArtifactCollector, - scenario: &FaultScenario, - plan: &FaultPlan, -) -> Result<()> { - let spec = fault_scenarios::scenario_spec(&scenario.name)?; - if let Err(error) = require_fault_backends(config, plan) { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "fault-backend-preflight", - "environment_or_fault_backend", - error.to_string(), - ), - )?; - return Err(error); - } - if let Err(error) = cleanup_fault_backends(config, plan) { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "fault-backend-pre-cleanup", - "environment_or_fault_backend", - error.to_string(), - ), - )?; - return Err(error); - } - - if let Err(error) = prepare_fault_fixture(&config.cluster, spec.isolation) { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "fixture-prepare", - "test_or_environment", - error.to_string(), - ), - )?; - return Err(error); - } - if let Err(error) = wait_for_ready_tenant(&config.cluster).await { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "tenant-ready-before-fault", - "product_or_environment", - error.to_string(), - ), - )?; - return Err(error); - } - if let Err(error) = wait_for_stable_rustfs_pods(&config.cluster, RUSTFS_POD_STABLE_WINDOW).await - { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "pod-stability-before-fault", - "product_or_environment", - error.to_string(), - ), - )?; - return Err(error); - } - - let run_id = format!("run-{}", Uuid::new_v4()); - let workload_seed = config.workload_seed.unwrap_or_else(generated_seed); - let workload_plan = WorkloadPlan::seeded( - workload_seed, - scenario.object_count, - config.workload.concurrency, - ); - let bucket = bucket_name(&run_id); - let history_path = collector.case_dir(scenario.case_name).join("history.jsonl"); - let history = Recorder::create(history_path, &scenario.name, &run_id)?; - collector.write_text( - scenario.case_name, - "run-metadata.json", - &serde_json::to_string_pretty(&RunMetadata::from_case( - config, scenario, plan, &run_id, &bucket, - ))?, - )?; - collector.write_text( - scenario.case_name, - "workload-plan.json", - &serde_json::to_string_pretty(&workload_plan)?, - )?; - eprintln!( - "fault workload seed={} objects={} concurrency={} payload_bytes={}", - workload_plan.seed, - workload_plan.object_count, - workload_plan.concurrency, - workload_plan.total_payload_bytes - ); - - let cluster = &config.cluster; - let (endpoint, mut port_forward) = match s3_access(config) { - Ok(access) => access, - Err(error) => { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "s3-endpoint", - "test_or_environment", - error.to_string(), - ), - )?; - return Err(error); - } - }; - if let Err(error) = ensure_s3_access(&mut port_forward, cluster, &endpoint).await { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "initial-s3-access", - "product_or_environment", - error.to_string(), - ), - )?; - return Err(error); - } - - let (access_key, secret_key) = resources::test_credentials(); - let s3 = match S3WorkloadClient::new( - &endpoint, - &bucket, - access_key, - secret_key, - config.request_timeout, - ) - .await - { - Ok(client) => client, - Err(error) => { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "s3-client", - "test_or_environment", - error.to_string(), - ), - )?; - return Err(error); - } - }; - let bucket_outcome = match s3.create_bucket(&history).await { - Ok(outcome) => outcome, - Err(error) => { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "bucket-create", - "test_harness", - error.to_string(), - ), - )?; - return Err(error); - } - }; - if bucket_outcome != OperationOutcome::Ok { - let message = format!("fault workload bucket creation did not succeed: {bucket_outcome:?}"); - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "bucket-create", - "product_or_environment", - message.clone(), - ), - )?; - bail!("{message}"); - } - - let prefilled = match prefill_objects( - &s3, - &history, - &run_id, - &workload_plan, - scenario.prefill_count(), - ) - .await - { - Ok(prefilled) => prefilled, - Err(error) => { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "prefill", - "product_or_environment", - error.to_string(), - ), - )?; - return Err(error); - } - }; - let pods_before = match rustfs_pod_identities(cluster) { - Ok(pods) => pods, - Err(error) => { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "pod-identity-before-fault", - "test_or_environment", - error.to_string(), - ), - )?; - return Err(error); - } - }; - let mut fault = match AppliedFaults::apply(config, collector, scenario, plan, &run_id) { - Ok(fault) => fault, - Err(error) => { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "fault-apply", - "environment_or_fault_backend", - error.to_string(), - ), - )?; - return Err(error); - } - }; - - if let Err(error) = fault.wait_active(cluster.timeout) { - collect_fault_artifacts(collector, scenario.case_name, &fault, "wait-active-failed")?; - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "wait-active", - "environment_or_fault_backend", - error.to_string(), - ), - )?; - return Err(error); - } - let active_snapshots = fault.snapshots("active")?; - - if let Err(error) = ensure_s3_access(&mut port_forward, cluster, &endpoint).await { - collect_fault_artifacts(collector, scenario.case_name, &fault, "port-forward-failed")?; - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "s3-access-under-fault", - "environment_or_workload", - error.to_string(), - ), - )?; - return Err(error); - } - - if plan.workload_mode.runs_warp() { - let warp_bucket = warp_bucket_name(&run_id); - if let Err(error) = host_faults::run_warp_mixed( - config.warp_duration, - collector, - scenario.case_name, - &endpoint, - &warp_bucket, - access_key, - secret_key, - ) { - collect_fault_artifacts(collector, scenario.case_name, &fault, "warp-failed")?; - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "warp-workload", - "workload_or_product", - error.to_string(), - ), - )?; - return Err(error); - } - - if let Err(error) = ensure_s3_access(&mut port_forward, cluster, &endpoint).await { - collect_fault_artifacts( - collector, - scenario.case_name, - &fault, - "post-warp-port-forward-failed", - )?; - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "post-warp-s3-access", - "environment_or_workload", - error.to_string(), - ), - )?; - return Err(error); - } - } - - let mut workload = match run_mixed_workload( - &s3, - &history, - &run_id, - &workload_plan, - &prefilled, - scenario.prefill_count(), - scenario.mixed_workload_count(), - ) - .await - { - Ok(workload) => workload, - Err(error) => { - collect_fault_artifacts(collector, scenario.case_name, &fault, "workload-failed")?; - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "mixed-workload", - "workload_or_product", - error.to_string(), - ), - )?; - return Err(error); - } - }; - collector.write_text( - scenario.case_name, - "workload-summary.json", - &serde_json::to_string_pretty(&workload.summary)?, - )?; - if let Err(error) = workload - .summary - .require_fault_evidence(config.require_client_disruption) - { - collect_fault_artifacts( - collector, - scenario.case_name, - &fault, - "workload-no-fault-evidence", - )?; - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "fault-evidence", - "test_or_environment", - error.to_string(), - ), - )?; - return Err(error); - } - if let Err(error) = fault.ensure_active("after fault workload") { - collect_fault_artifacts( - collector, - scenario.case_name, - &fault, - "workload-outlived-fault", - )?; - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "fault-still-active", - "test_or_environment", - error.to_string(), - ), - )?; - return Err(error); - } - let workload_snapshots = fault.snapshots("after-workload")?; - - if let Err(error) = fault.delete(cluster.timeout) { - collect_fault_artifacts(collector, scenario.case_name, &fault, "delete-failed")?; - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "fault-delete", - "environment_or_fault_backend", - error.to_string(), - ), - )?; - return Err(error); - } - - if let Err(error) = wait_for_ready_tenant(cluster).await { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "tenant-recovery", - "product_or_environment", - error.to_string(), - ), - )?; - return Err(error); - } - if let Err(error) = wait_for_stable_rustfs_pods(cluster, RUSTFS_POD_STABLE_WINDOW).await { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "pod-stability-after-recovery", - "product_or_environment", - error.to_string(), - ), - )?; - return Err(error); - } - let pods_after = rustfs_pod_identities(cluster)?; - if let Err(error) = ensure_s3_access(&mut port_forward, cluster, &endpoint).await { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "s3-access-after-recovery", - "product_or_environment", - error.to_string(), - ), - )?; - return Err(error); - } - let recovered_evidence = FaultEvidence { - scenario: scenario.name.clone(), - backend: plan.backend_summary(), - target: plan.target_summary(), - injected: true, - active_during_workload: true, - recovered: true, - client_disruptions: workload.summary.disrupted(), - workload_plan: workload_plan.clone(), - pods_before: pods_before.clone(), - pods_after: pods_after.clone(), - active_snapshots: active_snapshots.clone(), - workload_snapshots: workload_snapshots.clone(), - dm_recovery_snapshot: fault.recovery_dm_snapshot(), - }; - collector.write_text( - scenario.case_name, - "fault-evidence.json", - &serde_json::to_string_pretty(&recovered_evidence)?, - )?; - let recommit_report = recommit_unconfirmed_objects( - &s3, - &history, - &workload.unconfirmed_puts, - workload_plan.concurrency, - ) - .await; - collector.write_text( - scenario.case_name, - "recommit-report.json", - &serde_json::to_string_pretty(&recommit_report)?, - )?; - workload.summary.recommitted_after_recovery = recommit_report.committed; - collector.write_text( - scenario.case_name, - "workload-summary.json", - &serde_json::to_string_pretty(&workload.summary)?, - )?; - if recommit_report.has_failures() { - let message = recommit_report.failure_message(); - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "recommit-unconfirmed", - recommit_report.failure_classification(), - message.clone(), - ), - )?; - bail!("{message}"); - } - let report = checker::check_s3_history(&s3, &history, true, workload_plan.concurrency).await?; - collector.write_text( - scenario.case_name, - "checker-report.json", - &serde_json::to_string_pretty(&report)?, - )?; - let evidence = FaultEvidence { - scenario: scenario.name.clone(), - backend: plan.backend_summary(), - target: plan.target_summary(), - injected: true, - active_during_workload: true, - recovered: report.tenant_recovered, - client_disruptions: workload.summary.disrupted(), - workload_plan, - pods_before, - pods_after, - active_snapshots, - workload_snapshots, - dm_recovery_snapshot: fault.recovery_dm_snapshot(), - }; - collector.write_text( - scenario.case_name, - "fault-evidence.json", - &serde_json::to_string_pretty(&evidence)?, - )?; - if report.committed_puts != scenario.object_count { - let message = format!( - "fault scenario {} expected {} committed objects after recovery reconciliation, got {}", - scenario.name, scenario.object_count, report.committed_puts - ); - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "checker-committed-count", - "product_or_environment", - message.clone(), - ), - )?; - bail!("{message}"); - } - if let Err(error) = report.require_success() { - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "checker-verdict", - "product_or_environment", - error.to_string(), - ), - )?; - return Err(error); - } - - Ok(()) -} - -fn require_fault_backends(config: &FaultTestConfig, plan: &FaultPlan) -> Result<()> { - for backend in plan.required_backends() { - require_fault_backend(config, backend)?; - } - Ok(()) -} - -fn require_fault_backend(config: &FaultTestConfig, backend: FaultBackend) -> Result<()> { - let cluster = &config.cluster; - match backend { - FaultBackend::ChaosMeshIoChaos => chaos_mesh::require_iochaos_crd(cluster), - FaultBackend::MinioWarpWithChaos => { - chaos_mesh::require_iochaos_crd(cluster)?; - require_tool("warp", ["--help"]) - } - FaultBackend::ChaosMeshPodChaos => chaos_mesh::require_podchaos_crd(cluster), - FaultBackend::ChaosMeshNetworkChaos => chaos_mesh::require_networkchaos_crd(cluster), - FaultBackend::DeviceMapper => require_dm_flakey_preflight(config), - } -} - -fn require_tool(program: &'static str, args: I) -> Result<()> -where - I: IntoIterator, - S: Into, -{ - CommandSpec::new(program) - .args(args) - .run_checked() - .with_context(|| format!("{program} is required for the selected fault scenario"))?; - Ok(()) -} - -fn require_dm_flakey_preflight(config: &FaultTestConfig) -> Result<()> { - config - .dm_name - .as_deref() - .context("RUSTFS_FAULT_TEST_DM_NAME is required for dm-flakey")?; - config - .dm_node - .as_deref() - .context("RUSTFS_FAULT_TEST_DM_NODE is required for dm-flakey")?; - config - .dm_mount_path - .as_deref() - .context("RUSTFS_FAULT_TEST_DM_MOUNT_PATH is required for dm-flakey")?; - config - .dm_fault_table - .as_deref() - .context("RUSTFS_FAULT_TEST_DM_FAULT_TABLE is required for dm-flakey")?; - Ok(()) -} - -fn cleanup_fault_backends(config: &FaultTestConfig, plan: &FaultPlan) -> Result<()> { - for backend in plan.required_backends() { - cleanup_fault_backend(config, backend)?; - } - Ok(()) -} - -fn cleanup_fault_backend(config: &FaultTestConfig, backend: FaultBackend) -> Result<()> { - match backend { - FaultBackend::ChaosMeshIoChaos | FaultBackend::MinioWarpWithChaos => { - chaos_mesh::cleanup_managed_iochaos(&config.cluster, &config.chaos_namespace) - } - FaultBackend::ChaosMeshPodChaos => { - chaos_mesh::cleanup_managed_podchaos(&config.cluster, &config.chaos_namespace) - } - FaultBackend::ChaosMeshNetworkChaos => { - chaos_mesh::cleanup_managed_networkchaos(&config.cluster, &config.chaos_namespace) - } - FaultBackend::DeviceMapper => Ok(()), - } -} - -fn prepare_fault_fixture(config: &ClusterTestConfig, isolation: FaultIsolation) -> Result<()> { - match isolation { - FaultIsolation::ReusableTenant => resources::apply_fault_tenant_resources(config)?, - FaultIsolation::FreshTenant | FaultIsolation::DedicatedLinuxBlockDevice => { - resources::reset_fault_tenant_resources(config)?; - resources::apply_fault_tenant_resources(config)?; - } - } - Ok(()) -} - -enum AppliedFault { - Chaos { - guard: Box, - active_required: bool, - }, - PodKill { - guard: Box, - before_pods: Vec, - config: Box, - }, - DmFlakey(Box), -} - -struct AppliedFaults { - items: Vec, -} - -impl AppliedFaults { - fn apply( - config: &FaultTestConfig, - collector: &ArtifactCollector, - scenario: &FaultScenario, - plan: &FaultPlan, - run_id: &str, - ) -> Result { - ensure!( - !plan.faults().is_empty(), - "fault plan {} did not contain any faults", - plan.scenario - ); - - let total = plan.faults().len(); - let mut items = Vec::with_capacity(total); - for (index, injection) in plan.faults().iter().enumerate() { - let manifest_name = chaos_manifest_artifact_name(total, index, injection); - let resource_name_suffix = chaos_resource_name_suffix(total, index); - items.push(AppliedFault::apply_one( - config, - collector, - scenario, - injection, - run_id, - &manifest_name, - &resource_name_suffix, - )?); - } - - Ok(Self { items }) - } - - fn len(&self) -> usize { - self.items.len() - } - - fn wait_active(&self, timeout: Duration) -> Result<()> { - for fault in &self.items { - fault.wait_active(timeout)?; - } - Ok(()) - } - - fn ensure_active(&self, stage: &str) -> Result<()> { - for fault in &self.items { - fault.ensure_active(stage)?; - } - Ok(()) - } - - fn delete(&mut self, timeout: Duration) -> Result<()> { - for fault in self.items.iter_mut().rev() { - fault.delete(timeout)?; - } - Ok(()) - } - - fn snapshot(&self, stage: &str) -> Result { - ensure!( - self.items.len() == 1, - "single fault snapshot requested for {} applied faults", - self.items.len() - ); - self.items[0].snapshot(stage) - } - - fn snapshots(&self, stage: &str) -> Result> { - self.items - .iter() - .map(|fault| fault.snapshot(stage)) - .collect() - } - - fn recovery_dm_snapshot(&self) -> Option { - self.items - .iter() - .find_map(AppliedFault::recovery_dm_snapshot) - } - - fn chaos_guards(&self) -> Vec<&ChaosGuard> { - self.items - .iter() - .filter_map(AppliedFault::chaos_guard) - .collect() - } -} - -impl AppliedFault { - fn apply_one( - config: &FaultTestConfig, - collector: &ArtifactCollector, - scenario: &FaultScenario, - injection: &FaultInjection, - run_id: &str, - manifest_name: &str, - resource_name_suffix: &str, - ) -> Result { - let cluster = &config.cluster; - match injection.kind() { - FaultKind::RustfsVolumeEnospc => { - let chaos = IoChaosSpec::enospc_on_rustfs_volume( - cluster, - &config.chaos_namespace, - run_id, - &scenario.name, - injection.rustfs_volume_path()?, - injection.percent()?, - injection.duration(), - )? - .with_name_suffix(resource_name_suffix); - collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; - Ok(Self::Chaos { - guard: Box::new(chaos_mesh::apply_iochaos(cluster, &chaos)?), - active_required: true, - }) - } - FaultKind::RustfsVolumeReadMistake => { - let chaos = IoChaosSpec::read_mistake_on_rustfs_volume( - cluster, - &config.chaos_namespace, - run_id, - &scenario.name, - injection.rustfs_volume_path()?, - injection.percent()?, - injection.duration(), - )? - .with_name_suffix(resource_name_suffix); - collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; - Ok(Self::Chaos { - guard: Box::new(chaos_mesh::apply_iochaos(cluster, &chaos)?), - active_required: true, - }) - } - FaultKind::RustfsVolumeIoError => { - let chaos = IoChaosSpec::eio_on_rustfs_volume( - cluster, - &config.chaos_namespace, - run_id, - &scenario.name, - injection.rustfs_volume_path()?, - injection.percent()?, - injection.duration(), - )? - .with_name_suffix(resource_name_suffix); - collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; - Ok(Self::Chaos { - guard: Box::new(chaos_mesh::apply_iochaos(cluster, &chaos)?), - active_required: true, - }) - } - FaultKind::RustfsServerPodKill => { - let before_pods = rustfs_pod_identities(cluster)?; - let chaos = PodChaosSpec::kill_one_rustfs_pod( - cluster, - &config.chaos_namespace, - run_id, - &scenario.name, - ) - .with_name_suffix(resource_name_suffix); - collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; - Ok(Self::PodKill { - guard: Box::new(chaos_mesh::apply_podchaos(cluster, &chaos)?), - before_pods, - config: Box::new(cluster.clone()), - }) - } - FaultKind::RustfsServerNetworkPartition => { - let chaos = NetworkChaosSpec::partition_one_rustfs_pod( - cluster, - &config.chaos_namespace, - run_id, - &scenario.name, - injection.duration(), - )? - .with_name_suffix(resource_name_suffix); - collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; - Ok(Self::Chaos { - guard: Box::new(chaos_mesh::apply_networkchaos(cluster, &chaos)?), - active_required: true, - }) - } - FaultKind::RustfsBlockDeviceFlakey => { - let name = config - .dm_name - .as_deref() - .context("RUSTFS_FAULT_TEST_DM_NAME is required for dm-flakey")?; - let fault_table = config - .dm_fault_table - .as_deref() - .context("RUSTFS_FAULT_TEST_DM_FAULT_TABLE is required for dm-flakey")?; - let node = config - .dm_node - .as_deref() - .context("RUSTFS_FAULT_TEST_DM_NODE is required for dm-flakey")?; - let mount_path = config - .dm_mount_path - .as_deref() - .context("RUSTFS_FAULT_TEST_DM_MOUNT_PATH is required for dm-flakey")?; - Ok(Self::DmFlakey(Box::new(host_faults::apply_dm_flakey( - cluster, - &DmFlakeySpec { - node, - mount_path, - helper_image: &config.dm_helper_image, - name, - fault_table, - recovery_table: config.dm_recovery_table.as_deref(), - run_id, - }, - collector, - scenario.case_name, - )?))) - } - } - } - - fn wait_active(&self, timeout: Duration) -> Result<()> { - match self { - Self::Chaos { - guard, - active_required, - } if *active_required => guard.wait_active(timeout), - Self::PodKill { - before_pods, - config, - .. - } => wait_for_rustfs_pod_deletion(config, before_pods, timeout), - Self::Chaos { .. } | Self::DmFlakey(_) => Ok(()), - } - } - - fn ensure_active(&self, stage: &str) -> Result<()> { - match self { - Self::Chaos { - guard, - active_required, - } if *active_required => guard.ensure_active(stage), - Self::PodKill { .. } | Self::Chaos { .. } => Ok(()), - Self::DmFlakey(guard) => { - guard.ensure_active("after fault workload")?; - Ok(()) - } - } - } - - fn delete(&mut self, timeout: Duration) -> Result<()> { - match self { - Self::Chaos { guard, .. } => guard.delete(), - Self::PodKill { - guard, - before_pods, - config, - } => { - guard.delete()?; - wait_for_rustfs_pod_replacement(config, before_pods, timeout) - } - Self::DmFlakey(guard) => guard.restore(), - } - } - - fn chaos_guard(&self) -> Option<&ChaosGuard> { - match self { - Self::Chaos { guard, .. } | Self::PodKill { guard, .. } => Some(guard.as_ref()), - Self::DmFlakey(_) => None, - } - } - - fn snapshot(&self, stage: &str) -> Result { - match self { - Self::Chaos { guard, .. } | Self::PodKill { guard, .. } => Ok(FaultStatusSnapshot { - stage: stage.to_string(), - resource_kind: Some(guard.kind().to_string()), - resource_name: Some(guard.name().to_string()), - chaos_status: Some(serde_json::from_str(&guard.json()?)?), - dm_status: None, - }), - Self::DmFlakey(guard) => Ok(FaultStatusSnapshot { - stage: stage.to_string(), - resource_kind: Some("device-mapper".to_string()), - resource_name: None, - chaos_status: None, - dm_status: Some(guard.snapshot(stage)?), - }), - } - } - - fn recovery_dm_snapshot(&self) -> Option { - match self { - Self::DmFlakey(guard) => guard.recovery_snapshot().cloned(), - Self::Chaos { .. } | Self::PodKill { .. } => None, - } - } -} - -fn chaos_manifest_artifact_name(total: usize, index: usize, injection: &FaultInjection) -> String { - if total == 1 { - "chaos-manifest.yaml".to_string() - } else { - format!( - "chaos-manifest-{index:02}-{}.yaml", - injection.kind().as_str() - ) - } -} - -fn chaos_resource_name_suffix(total: usize, index: usize) -> String { - if total == 1 { - String::new() - } else { - format!("-{index:02}") - } -} - -#[derive(Debug, Clone, Serialize)] -struct FaultStatusSnapshot { - stage: String, - resource_kind: Option, - resource_name: Option, - chaos_status: Option, - dm_status: Option, -} - -#[derive(Debug, Clone, Serialize)] -struct FaultEvidence { - scenario: String, - backend: String, - target: String, - injected: bool, - active_during_workload: bool, - recovered: bool, - client_disruptions: usize, - workload_plan: WorkloadPlan, - pods_before: Vec, - pods_after: Vec, - active_snapshots: Vec, - workload_snapshots: Vec, - dm_recovery_snapshot: Option, -} - -#[derive(Debug, Clone, Serialize)] -struct RunMetadata { - scenario: String, - case_name: String, - run_id: String, - bucket: String, - backend: String, - target: String, - context: String, - namespace: String, - tenant: String, - storage_class: String, - rustfs_image: String, - artifacts_dir: String, - duration_seconds: u64, - percent: Option, - fault_selection: Vec, - workload_objects: usize, - workload_concurrency: usize, - request_timeout_seconds: u64, - use_cluster_ip: bool, - require_client_disruption: bool, - chaos_namespace: String, -} - -impl RunMetadata { - fn from_case( - config: &FaultTestConfig, - scenario: &FaultScenario, - plan: &FaultPlan, - run_id: &str, - bucket: &str, - ) -> Self { - Self { - scenario: scenario.name.clone(), - case_name: scenario.case_name.to_string(), - run_id: run_id.to_string(), - bucket: bucket.to_string(), - backend: plan.backend_summary(), - target: plan.target_summary(), - context: config.cluster.context.clone(), - namespace: config.cluster.test_namespace.clone(), - tenant: config.cluster.tenant_name.clone(), - storage_class: config.cluster.storage_class.clone(), - rustfs_image: config.cluster.rustfs_image.clone(), - artifacts_dir: config.cluster.artifacts_dir.display().to_string(), - duration_seconds: scenario.duration.as_secs(), - percent: plan - .faults() - .iter() - .find_map(|fault| match fault.selection() { - FaultSelection::Percent(percent) => Some(percent), - FaultSelection::FixedTargets(_) => None, - }), - fault_selection: plan - .faults() - .iter() - .map(|fault| fault.selection().summary()) - .collect(), - workload_objects: scenario.object_count, - workload_concurrency: config.workload.concurrency, - request_timeout_seconds: config.request_timeout.as_secs(), - use_cluster_ip: config.use_cluster_ip, - require_client_disruption: config.require_client_disruption, - chaos_namespace: config.chaos_namespace.clone(), - } - } -} - -#[derive(Debug, Clone, Serialize)] -struct FailureSummary { - scenario: String, - stage: String, - classification: String, - message: String, -} - -impl FailureSummary { - fn new( - scenario: impl Into, - stage: impl Into, - classification: impl Into, - message: impl Into, - ) -> Self { - Self { - scenario: scenario.into(), - stage: stage.into(), - classification: classification.into(), - message: message.into(), - } - } -} - -fn write_failure_summary( - collector: &ArtifactCollector, - case_name: &str, - summary: FailureSummary, -) -> Result<()> { - collector.write_text( - case_name, - "failure-summary.json", - &serde_json::to_string_pretty(&summary)?, - )?; - Ok(()) -} - -fn write_failure_summary_if_absent( - collector: &ArtifactCollector, - case_name: &str, - summary: FailureSummary, -) -> Result<()> { - let path = collector.case_dir(case_name).join("failure-summary.json"); - if path.exists() { - return Ok(()); - } - write_failure_summary(collector, case_name, summary) -} - -fn collect_fault_artifacts( - collector: &ArtifactCollector, - case_name: &str, - fault: &AppliedFaults, - suffix: &str, -) -> Result<()> { - let status = if fault.len() == 1 { - fault - .snapshot(suffix) - .and_then(|snapshot| serde_json::to_string_pretty(&snapshot).map_err(Into::into)) - } else { - fault - .snapshots(suffix) - .and_then(|snapshots| serde_json::to_string_pretty(&snapshots).map_err(Into::into)) - } - .unwrap_or_else(|error| format!("failed to collect fault status: {error}")); - collector.write_text(case_name, &format!("fault-status-{suffix}.json"), &status)?; - - let guards = fault.chaos_guards(); - for (index, guard) in guards.iter().enumerate() { - let describe = guard - .describe() - .unwrap_or_else(|error| format!("failed to describe chaos before cleanup: {error}")); - let describe_name = - chaos_artifact_name(guards.len(), index, "chaos-describe", suffix, "txt"); - collector.write_text(case_name, &describe_name, &describe)?; - - let yaml = guard - .yaml() - .unwrap_or_else(|error| format!("failed to get chaos yaml before cleanup: {error}")); - let yaml_name = chaos_artifact_name(guards.len(), index, "chaos", suffix, "yaml"); - collector.write_text(case_name, &yaml_name, &yaml)?; - } - - Ok(()) -} - -fn chaos_artifact_name( - total: usize, - index: usize, - prefix: &str, - suffix: &str, - extension: &str, -) -> String { - if total == 1 { - format!("{prefix}-{suffix}.{extension}") - } else { - format!("{prefix}-{suffix}-{index:02}.{extension}") - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -struct PodIdentity { - name: String, - uid: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct PodRuntimeState { - name: String, - uid: String, - phase: String, - containers_ready: bool, - restart_count: u64, - terminating: bool, -} - -fn rustfs_pod_identities(config: &ClusterTestConfig) -> Result> { - let selector = format!("rustfs.tenant={}", config.tenant_name); - let output = rustfs_operator_e2e::framework::kubectl::Kubectl::new(config) - .namespaced(&config.test_namespace) - .command(["get", "pod", "-l", &selector, "-o", "json"]) - .run_checked()?; - let value = serde_json::from_str::(&output.stdout) - .context("parse RustFS pod list json")?; - let items = value - .pointer("/items") - .and_then(serde_json::Value::as_array) - .context("RustFS pod list did not contain an items array")?; - let pods = items - .iter() - .filter_map(|item| { - let metadata = item.get("metadata")?; - Some(PodIdentity { - name: metadata.get("name")?.as_str()?.to_string(), - uid: metadata.get("uid")?.as_str()?.to_string(), - }) - }) - .collect::>(); - ensure!( - !pods.is_empty(), - "no RustFS pods found for selector {selector} in namespace {}", - config.test_namespace - ); - Ok(pods) -} - -fn rustfs_pod_runtime_states(config: &ClusterTestConfig) -> Result> { - let selector = format!("rustfs.tenant={}", config.tenant_name); - let output = Kubectl::new(config) - .namespaced(&config.test_namespace) - .command(["get", "pod", "-l", &selector, "-o", "json"]) - .run_checked()?; - let value = serde_json::from_str::(&output.stdout) - .context("parse RustFS pod list json")?; - let items = value - .pointer("/items") - .and_then(serde_json::Value::as_array) - .context("RustFS pod list did not contain an items array")?; - let mut pods = items - .iter() - .map(|item| { - let metadata = item - .get("metadata") - .context("RustFS pod did not contain metadata")?; - let name = metadata - .get("name") - .and_then(serde_json::Value::as_str) - .context("RustFS pod metadata did not contain a name")?; - let uid = metadata - .get("uid") - .and_then(serde_json::Value::as_str) - .context("RustFS pod metadata did not contain a uid")?; - let phase = item - .pointer("/status/phase") - .and_then(serde_json::Value::as_str) - .unwrap_or("Unknown"); - let container_statuses = item - .pointer("/status/containerStatuses") - .and_then(serde_json::Value::as_array); - let containers_ready = container_statuses.is_some_and(|statuses| { - !statuses.is_empty() - && statuses.iter().all(|status| { - status - .get("ready") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false) - }) - }); - let restart_count = container_statuses - .into_iter() - .flatten() - .filter_map(|status| status.get("restartCount")) - .filter_map(serde_json::Value::as_u64) - .sum(); - - Ok(PodRuntimeState { - name: name.to_string(), - uid: uid.to_string(), - phase: phase.to_string(), - containers_ready, - restart_count, - terminating: metadata.get("deletionTimestamp").is_some(), - }) - }) - .collect::>>()?; - pods.sort_by(|left, right| left.name.cmp(&right.name)); - Ok(pods) -} - -fn stable_pod_fingerprint(pods: &[PodRuntimeState]) -> Option> { - if pods.len() != FAULT_TENANT_POD_COUNT - || pods - .iter() - .any(|pod| pod.phase != "Running" || !pod.containers_ready || pod.terminating) - { - return None; - } - - Some( - pods.iter() - .map(|pod| (pod.uid.clone(), pod.restart_count)) - .collect(), - ) -} - -async fn wait_for_stable_rustfs_pods( - config: &ClusterTestConfig, - stable_window: Duration, -) -> Result<()> { - let deadline = Instant::now() + config.timeout; - let mut stable_since = None; - let mut stable_fingerprint = None; - let mut last_snapshot = Vec::new(); - let mut last_error = "not checked yet".to_string(); - - eprintln!( - "waiting for {FAULT_TENANT_POD_COUNT} RustFS pods to remain ready without restarts for {stable_window:?}" - ); - loop { - if Instant::now() >= deadline { - bail!( - "timed out waiting for stable RustFS pods after {:?}\nlast: {last_snapshot:?}\nlast error: {last_error}", - config.timeout - ); - } - - match rustfs_pod_runtime_states(config) { - Ok(current) => { - if let Some(fingerprint) = stable_pod_fingerprint(¤t) { - if stable_fingerprint.as_ref() != Some(&fingerprint) { - stable_since = Some(Instant::now()); - stable_fingerprint = Some(fingerprint); - } - if stable_since.is_some_and(|started| started.elapsed() >= stable_window) { - eprintln!("RustFS pods remained stable for {stable_window:?}"); - return Ok(()); - } - } else { - stable_since = None; - stable_fingerprint = None; - } - last_snapshot = current; - last_error = "none".to_string(); - } - Err(error) => { - stable_since = None; - stable_fingerprint = None; - last_error = error.to_string(); - } - } - - async_sleep(Duration::from_secs(1)).await; - } -} - -fn wait_for_rustfs_pod_replacement( - config: &ClusterTestConfig, - before: &[PodIdentity], - timeout: Duration, -) -> Result<()> { - let deadline = Instant::now() + timeout; - let mut last_snapshot = Vec::new(); - let mut last_error = "not checked yet".to_string(); - - loop { - if Instant::now() >= deadline { - bail!( - "timed out waiting for PodChaos to replace a RustFS pod after {timeout:?}\nbefore: {before:?}\nlast: {last_snapshot:?}\nlast error: {last_error}", - ); - } - - match rustfs_pod_identities(config) { - Ok(current) => { - if pod_replacement_observed(before, ¤t) { - return Ok(()); - } - last_snapshot = current; - last_error = "none".to_string(); - } - Err(error) => { - last_error = error.to_string(); - } - } - - sleep(Duration::from_secs(1)); - } -} - -fn wait_for_rustfs_pod_deletion( - config: &ClusterTestConfig, - before: &[PodIdentity], - timeout: Duration, -) -> Result<()> { - let deadline = Instant::now() + timeout; - let mut last_snapshot = Vec::new(); - let mut last_error = "not checked yet".to_string(); - - loop { - if Instant::now() >= deadline { - bail!( - "timed out waiting for PodChaos to delete a RustFS pod after {timeout:?}\nbefore: {before:?}\nlast: {last_snapshot:?}\nlast error: {last_error}", - ); - } - - match rustfs_pod_identities(config) { - Ok(current) => { - if pod_deletion_observed(before, ¤t) { - return Ok(()); - } - last_snapshot = current; - last_error = "none".to_string(); - } - Err(error) => { - last_error = error.to_string(); - } - } - - sleep(Duration::from_millis(250)); - } -} - -fn pod_deletion_observed(before: &[PodIdentity], current: &[PodIdentity]) -> bool { - let current_uids = current - .iter() - .map(|pod| pod.uid.as_str()) - .collect::>(); - !before.is_empty() - && before - .iter() - .any(|pod| !current_uids.contains(pod.uid.as_str())) -} - -fn pod_replacement_observed(before: &[PodIdentity], current: &[PodIdentity]) -> bool { - if before.is_empty() || current.is_empty() { - return false; - } - - let before_uids = before - .iter() - .map(|pod| pod.uid.as_str()) - .collect::>(); - let current_uids = current - .iter() - .map(|pod| pod.uid.as_str()) - .collect::>(); - let old_uid_removed = before_uids.iter().any(|uid| !current_uids.contains(uid)); - let new_uid_added = current_uids.iter().any(|uid| !before_uids.contains(uid)); - - old_uid_removed && new_uid_added -} - -async fn wait_for_ready_tenant(config: &ClusterTestConfig) -> Result { - let client = kube_client::default_client().await?; - let tenants: Api = kube_client::tenant_api(client, &config.test_namespace); - wait::wait_for_tenant_ready(tenants, &config.tenant_name, config.timeout).await -} - -fn s3_access(config: &FaultTestConfig) -> Result<(String, Option)> { - let cluster = &config.cluster; - if config.use_cluster_ip { - let service = format!("{}-io", cluster.tenant_name); - let output = Kubectl::new(cluster) - .namespaced(&cluster.test_namespace) - .command([ - "get".to_string(), - "service".to_string(), - service.clone(), - "-o".to_string(), - "jsonpath={.spec.clusterIP}".to_string(), - ]) - .run_checked() - .with_context(|| format!("read ClusterIP for fault-test service {service:?}"))?; - let cluster_ip = output.stdout.trim(); - ensure!( - !cluster_ip.is_empty() && cluster_ip != "None", - "fault-test service {service:?} does not have a ClusterIP" - ); - let host = if cluster_ip.contains(':') { - format!("[{cluster_ip}]") - } else { - cluster_ip.to_string() - }; - return Ok((format!("http://{host}:9000"), None)); - } - - let spec = PortForwardSpec::tenant_io(&cluster.test_namespace, &cluster.tenant_name); - let endpoint = spec.local_base_url(); - Ok((endpoint, Some(PortForwardSpec::start_tenant_io(cluster)?))) -} - -async fn ensure_s3_access( - port_forward: &mut Option, - config: &ClusterTestConfig, - endpoint: &str, -) -> Result<()> { - if let Some(guard) = port_forward { - if guard.ensure_running().is_err() { - *guard = PortForwardSpec::start_tenant_io(config)?; - } - return wait_for_tenant_s3(guard, endpoint, config.timeout).await; - } - - wait_for_s3_endpoint(endpoint, config.timeout).await -} - -async fn wait_for_tenant_s3( - port_forward: &mut PortForwardGuard, - endpoint: &str, - timeout: Duration, -) -> Result<()> { - port_forward.ensure_running()?; - wait_for_s3_endpoint(endpoint, timeout) - .await - .with_context(|| { - format!( - "S3 port-forward was not ready; command: {}; log {}:\n{}", - port_forward.command_display(), - port_forward.log_path().display(), - port_forward.log_contents() - ) - }) -} - -async fn prefill_objects( - s3: &S3WorkloadClient, - history: &Recorder, - run_id: &str, - plan: &WorkloadPlan, - count: usize, -) -> Result> { - let tasks = (0..count).map(|index| { - let s3 = s3.clone(); - let history = history.clone(); - let run_id = run_id.to_string(); - let size_bytes = plan.size_at(index); - let seed = plan.seed; - async move { - let object = ObjectSpec::prepare_seeded(&run_id, index, size_bytes, seed); - let spec = object.spec.clone(); - let put_outcome = s3.put_object(&object, &history).await?; - ensure!( - put_outcome == OperationOutcome::Ok, - "prefill PUT failed before fault injection for key {}: {put_outcome:?}", - spec.key - ); - let head_outcome = s3.head_object(&spec.key, &history).await?; - ensure!( - head_outcome == OperationOutcome::Ok, - "prefill HEAD failed before fault injection for key {}: {head_outcome:?}", - spec.key - ); - Ok::<_, anyhow::Error>((index, spec)) - } - }); - let mut objects = stream::iter(tasks) - .buffer_unordered(plan.concurrency) - .try_collect::>() - .await?; - objects.sort_by_key(|(index, _)| *index); - - Ok(objects.into_iter().map(|(_, object)| object).collect()) -} - -async fn run_mixed_workload( - s3: &S3WorkloadClient, - history: &Recorder, - run_id: &str, - plan: &WorkloadPlan, - prefilled: &[ObjectSpec], - start_index: usize, - count: usize, -) -> Result { - let tasks = (0..count).map(|offset| { - let s3 = s3.clone(); - let history = history.clone(); - let run_id = run_id.to_string(); - let index = start_index + offset; - let size_bytes = plan.size_at(index); - let seed = plan.seed; - let existing = prefilled[offset % prefilled.len()].clone(); - async move { - let object = ObjectSpec::prepare_seeded(&run_id, index, size_bytes, seed); - let spec = object.spec.clone(); - let put_outcome = s3.put_object(&object, &history).await?; - let get_outcome = s3.get_object_result(&existing.key, &history).await?.outcome; - Ok::<_, anyhow::Error>(MixedTaskResult { - index, - object: spec, - put_outcome, - get_outcome, - }) - } - }); - let results = stream::iter(tasks) - .buffer_unordered(plan.concurrency) - .collect::>() - .await; - let mut completed = Vec::with_capacity(count); - for result in results { - completed.push(result?); - } - completed.sort_by_key(|result| result.index); - - let mut summary = WorkloadSummary::new(plan); - let mut unconfirmed_puts = Vec::new(); - for result in completed { - summary.puts.record(result.put_outcome); - summary.gets.record(result.get_outcome); - if result.put_outcome != OperationOutcome::Ok { - unconfirmed_puts.push(result.object); - } - } - - summary.require_exercised()?; - Ok(MixedWorkloadResult { - summary, - unconfirmed_puts, - }) -} - -async fn recommit_unconfirmed_objects( - s3: &S3WorkloadClient, - history: &Recorder, - objects: &[ObjectSpec], - concurrency: usize, -) -> RecommitReport { - let tasks = objects.iter().cloned().map(|object| { - let s3 = s3.clone(); - let history = history.clone(); - async move { - let prepared = object.prepare(); - match s3.put_object_record(&prepared, &history).await { - Ok(record) => RecommitAttempt::from_record(object, record), - Err(error) => { - RecommitAttempt::from_harness_error(object, format!("record PUT: {error}")) - } - } - } - }); - let mut attempts = stream::iter(tasks) - .buffer_unordered(concurrency) - .collect::>() - .await; - attempts.sort_by(|left, right| left.key.cmp(&right.key)); - RecommitReport::from_attempts(attempts) -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -struct RecommitReport { - attempted: usize, - committed: usize, - failed: usize, - harness_errors: usize, - attempts: Vec, -} - -impl RecommitReport { - fn from_attempts(attempts: Vec) -> Self { - let committed = attempts - .iter() - .filter(|attempt| attempt.outcome == Some(OperationOutcome::Ok)) - .count(); - let failed = attempts - .iter() - .filter(|attempt| attempt.is_s3_failure()) - .count(); - let harness_errors = attempts - .iter() - .filter(|attempt| attempt.is_harness_error()) - .count(); - Self { - attempted: attempts.len(), - committed, - failed, - harness_errors, - attempts, - } - } - - fn has_failures(&self) -> bool { - self.failed > 0 || self.harness_errors > 0 - } - - fn failure_classification(&self) -> &'static str { - if self.harness_errors > 0 { - "test_harness" - } else { - "product_or_environment" - } - } - - fn failure_message(&self) -> String { - let sample = self - .attempts - .iter() - .filter_map(RecommitAttempt::failure_sample) - .take(5) - .collect::>() - .join(", "); - format!( - "{} of {} previously unconfirmed PUTs did not commit after recovery; harness_errors={}{}", - self.failed, - self.attempted, - self.harness_errors, - if sample.is_empty() { - String::new() - } else { - format!("; sample: {sample}") - } - ) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -struct RecommitAttempt { - key: String, - size_bytes: usize, - sha256: String, - outcome: Option, - http_status: Option, - error: Option, - harness_error: Option, -} - -impl RecommitAttempt { - fn from_record(object: ObjectSpec, record: OperationRecord) -> Self { - Self { - key: object.key, - size_bytes: object.size_bytes, - sha256: object.sha256, - outcome: Some(record.outcome), - http_status: record.http_status, - error: record.error, - harness_error: None, - } - } - - fn from_harness_error(object: ObjectSpec, error: String) -> Self { - Self { - key: object.key, - size_bytes: object.size_bytes, - sha256: object.sha256, - outcome: None, - http_status: None, - error: None, - harness_error: Some(error), - } - } - - fn is_s3_failure(&self) -> bool { - matches!( - self.outcome, - Some(OperationOutcome::Failed | OperationOutcome::Timeout | OperationOutcome::Unknown) - ) - } - - fn is_harness_error(&self) -> bool { - self.harness_error.is_some() - } - - fn failure_sample(&self) -> Option { - if let Some(error) = &self.harness_error { - return Some(format!("{}=harness_error({error})", self.key)); - } - let outcome = self.outcome?; - if outcome == OperationOutcome::Ok { - return None; - } - let status = self - .http_status - .map(|status| format!(" status={status}")) - .unwrap_or_default(); - let error = self - .error - .as_ref() - .map(|error| format!(" error={error}")) - .unwrap_or_default(); - Some(format!("{}={outcome:?}{status}{error}", self.key)) - } -} - -#[derive(Debug)] -struct MixedTaskResult { - index: usize, - object: ObjectSpec, - put_outcome: OperationOutcome, - get_outcome: OperationOutcome, -} - -#[derive(Debug)] -struct MixedWorkloadResult { - summary: WorkloadSummary, - unconfirmed_puts: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -struct WorkloadSummary { - seed: u64, - object_count: usize, - concurrency: usize, - total_payload_bytes: u64, - puts: OutcomeCounts, - gets: OutcomeCounts, - recommitted_after_recovery: usize, -} - -impl WorkloadSummary { - fn new(plan: &WorkloadPlan) -> Self { - Self { - seed: plan.seed, - object_count: plan.object_count, - concurrency: plan.concurrency, - total_payload_bytes: plan.total_payload_bytes, - puts: OutcomeCounts::default(), - gets: OutcomeCounts::default(), - recommitted_after_recovery: 0, - } - } - - fn require_exercised(&self) -> Result<()> { - ensure!( - self.puts.total() > 0 && self.gets.total() > 0, - "fault workload did not exercise both PUT and GET paths: {self:?}" - ); - Ok(()) - } - - fn require_fault_evidence(&self, require_client_disruption: bool) -> Result<()> { - if require_client_disruption { - ensure!( - self.disrupted() > 0, - "fault was applied but the S3 workload observed no client-visible disrupted operation; increase RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS or RUSTFS_FAULT_TEST_PERCENT, or set RUSTFS_FAULT_TEST_REQUIRE_CLIENT_DISRUPTION=0 if this is expected" - ); - } else if self.disrupted() == 0 { - eprintln!( - "fault was applied, but the S3 workload observed no client-visible disrupted operation" - ); - } - Ok(()) - } - - fn disrupted(&self) -> usize { - self.puts.disrupted() + self.gets.disrupted() - } -} - -#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)] -struct OutcomeCounts { - ok: usize, - failed: usize, - timeout: usize, - unknown: usize, -} - -impl OutcomeCounts { - fn record(&mut self, outcome: OperationOutcome) { - match outcome { - OperationOutcome::Ok => self.ok += 1, - OperationOutcome::Failed => self.failed += 1, - OperationOutcome::Timeout => self.timeout += 1, - OperationOutcome::Unknown => self.unknown += 1, - } - } - - fn total(&self) -> usize { - self.ok + self.failed + self.timeout + self.unknown - } - - fn disrupted(&self) -> usize { - self.failed + self.timeout + self.unknown - } -} - -fn bucket_name(run_id: &str) -> String { - let suffix = run_id - .chars() - .filter(|ch| ch.is_ascii_alphanumeric()) - .take(16) - .collect::() - .to_ascii_lowercase(); - format!("rustfs-fault-{suffix}") -} - -fn generated_seed() -> u64 { - let run = Uuid::new_v4(); - let mut bytes = [0; 8]; - bytes.copy_from_slice(&run.as_bytes()[..8]); - u64::from_le_bytes(bytes) -} - -fn warp_bucket_name(run_id: &str) -> String { - format!("{}-warp", bucket_name(run_id)) -} - -#[cfg(test)] -mod tests { - use super::{ - OutcomeCounts, PodIdentity, PodRuntimeState, RecommitAttempt, RecommitReport, - WorkloadSummary, bucket_name, chaos_manifest_artifact_name, chaos_resource_name_suffix, - pod_deletion_observed, pod_replacement_observed, stable_pod_fingerprint, warp_bucket_name, - }; - use rustfs_operator_e2e::framework::fault_plan::{ - DEFAULT_RUSTFS_DATA_VOLUME, FaultInjection, FaultKind, FaultSelection, FaultTarget, - }; - use rustfs_operator_e2e::framework::fault_scenarios::FaultBackend; - use rustfs_operator_e2e::framework::history::OperationOutcome; - use rustfs_operator_e2e::framework::s3_workload::WorkloadPlan; - - #[test] - fn fault_bucket_name_is_s3_compatible_and_run_scoped() { - assert_eq!( - bucket_name("run-12345678-abcd-efgh"), - "rustfs-fault-run12345678abcde" - ); - assert_eq!( - warp_bucket_name("run-12345678-abcd-efgh"), - "rustfs-fault-run12345678abcde-warp" - ); - } - - #[test] - fn composite_fault_artifacts_and_resource_names_are_indexed() { - let injection = FaultInjection::new( - FaultKind::RustfsVolumeIoError, - FaultBackend::ChaosMeshIoChaos, - FaultTarget::RustfsVolume { - path: DEFAULT_RUSTFS_DATA_VOLUME, - }, - FaultSelection::Percent(20), - std::time::Duration::from_secs(60), - ) - .expect("valid fault"); - - assert_eq!( - chaos_manifest_artifact_name(1, 0, &injection), - "chaos-manifest.yaml" - ); - assert_eq!(chaos_resource_name_suffix(1, 0), ""); - assert_eq!( - chaos_manifest_artifact_name(2, 1, &injection), - "chaos-manifest-01-rustfs_volume_io_error.yaml" - ); - assert_eq!(chaos_resource_name_suffix(2, 1), "-01"); - } - - #[test] - fn workload_summary_counts_disrupted_operations() { - let mut summary = WorkloadSummary::new(&WorkloadPlan::seeded(42, 40000, 80)); - summary.puts.record(OperationOutcome::Ok); - summary.gets.record(OperationOutcome::Timeout); - - assert_eq!(summary.puts.total(), 1); - assert_eq!(summary.gets.total(), 1); - assert_eq!(summary.disrupted(), 1); - assert!(summary.require_exercised().is_ok()); - assert!(summary.require_fault_evidence(true).is_ok()); - } - - #[test] - fn workload_summary_can_require_fault_evidence() { - let summary = WorkloadSummary { - seed: 42, - object_count: 40000, - concurrency: 80, - total_payload_bytes: 20_337_459_200, - puts: OutcomeCounts { - ok: 1, - ..OutcomeCounts::default() - }, - gets: OutcomeCounts { - ok: 1, - ..OutcomeCounts::default() - }, - recommitted_after_recovery: 0, - }; - - assert!(summary.require_fault_evidence(false).is_ok()); - assert!(summary.require_fault_evidence(true).is_err()); - } - - #[test] - fn recommit_report_counts_and_summarizes_failed_attempts() { - let report = RecommitReport::from_attempts(vec![ - RecommitAttempt { - key: "object-a".to_string(), - size_bytes: 4096, - sha256: "sha-a".to_string(), - outcome: Some(OperationOutcome::Ok), - http_status: Some(200), - error: None, - harness_error: None, - }, - RecommitAttempt { - key: "object-b".to_string(), - size_bytes: 4096, - sha256: "sha-b".to_string(), - outcome: Some(OperationOutcome::Failed), - http_status: Some(503), - error: Some("service unavailable".to_string()), - harness_error: None, - }, - ]); - - assert_eq!(report.attempted, 2); - assert_eq!(report.committed, 1); - assert_eq!(report.failed, 1); - assert_eq!(report.harness_errors, 0); - assert!(report.has_failures()); - assert!( - report - .failure_message() - .contains("object-b=Failed status=503") - ); - assert_eq!(report.failure_classification(), "product_or_environment"); - } - - #[test] - fn recommit_report_separates_harness_errors_from_s3_failures() { - let report = RecommitReport::from_attempts(vec![RecommitAttempt { - key: "object-a".to_string(), - size_bytes: 4096, - sha256: "sha-a".to_string(), - outcome: None, - http_status: None, - error: None, - harness_error: Some("record PUT: disk full".to_string()), - }]); - - assert_eq!(report.attempted, 1); - assert_eq!(report.committed, 0); - assert_eq!(report.failed, 0); - assert_eq!(report.harness_errors, 1); - assert!(report.has_failures()); - assert_eq!(report.failure_classification(), "test_harness"); - assert!( - report - .failure_message() - .contains("object-a=harness_error(record PUT: disk full)") - ); - } - - #[test] - fn pod_replacement_requires_old_uid_removed_and_new_uid_added() { - let before = vec![ - PodIdentity { - name: "rustfs-0".to_string(), - uid: "uid-a".to_string(), - }, - PodIdentity { - name: "rustfs-1".to_string(), - uid: "uid-b".to_string(), - }, - ]; - - assert!(!pod_replacement_observed(&before, &before)); - assert!(!pod_replacement_observed(&before, &before[..1])); - assert!(!pod_deletion_observed(&before, &before)); - assert!(pod_deletion_observed(&before, &before[..1])); - assert!(pod_replacement_observed( - &before, - &[ - PodIdentity { - name: "rustfs-0".to_string(), - uid: "uid-c".to_string(), - }, - before[1].clone(), - ], - )); - } - - #[test] - fn stable_pod_fingerprint_requires_four_ready_unchanged_pods() { - let pods = (0..4) - .map(|index| PodRuntimeState { - name: format!("rustfs-{index}"), - uid: format!("uid-{index}"), - phase: "Running".to_string(), - containers_ready: true, - restart_count: index, - terminating: false, - }) - .collect::>(); - - assert_eq!( - stable_pod_fingerprint(&pods), - Some(vec![ - ("uid-0".to_string(), 0), - ("uid-1".to_string(), 1), - ("uid-2".to_string(), 2), - ("uid-3".to_string(), 3), - ]) - ); - assert!(stable_pod_fingerprint(&pods[..3]).is_none()); - - let mut unready = pods; - unready[0].containers_ready = false; - assert!(stable_pod_fingerprint(&unready).is_none()); - } + rustfs_operator_e2e::fault::runner::run_selected_scenario_from_env().await } From 5f0368fea1ad5407a1ed099127fa496559ef7c20 Mon Sep 17 00:00:00 2001 From: GatewayJ <18332154+GatewayJ@users.noreply.github.com> Date: Sun, 21 Jun 2026 19:05:11 +0800 Subject: [PATCH 9/9] test(e2e): expand fault test object model --- e2e/FAULT_TESTING.md | 85 ++--- e2e/Makefile | 10 +- e2e/scripts/fault-test.sh | 132 ++++--- e2e/src/bin/rustfs-e2e.rs | 47 ++- e2e/src/fault/backends/chaos_mesh.rs | 548 ++++++++++++++++++++++++++- e2e/src/fault/backends/host.rs | 22 +- e2e/src/fault/checker.rs | 362 +++++++++++++++--- e2e/src/fault/config.rs | 13 +- e2e/src/fault/history.rs | 8 + e2e/src/fault/plan.rs | 103 ++++- e2e/src/fault/runner.rs | 404 +++++++++++++++++--- e2e/src/fault/scenarios.rs | 261 ++++++++++++- e2e/src/fault/workload.rs | 388 ++++++++++++++++++- 13 files changed, 2113 insertions(+), 270 deletions(-) diff --git a/e2e/FAULT_TESTING.md b/e2e/FAULT_TESTING.md index 2240c22..993cb29 100644 --- a/e2e/FAULT_TESTING.md +++ b/e2e/FAULT_TESTING.md @@ -22,7 +22,7 @@ tests from the `e2e` package. ## Scope Fault tests run only on a dedicated real Kubernetes or K3s cluster. They are -not Kind tests and are not designed for shared application clusters. The suite +not Kind tests and are not designed for shared application clusters. The runner creates and deletes its own namespace, Tenant, PVCs, Pods, Services, StatefulSet, and Chaos Mesh resources. @@ -46,22 +46,22 @@ Run all commands from the repository root. ```bash make -C e2e fault-check +make -C e2e fault-list make -C e2e fault-preflight SCENARIO=io-eio make -C e2e fault-run SCENARIO=io-eio -make -C e2e fault-run-regular make -C e2e fault-run-dm make -C e2e fault-cleanup ``` `fault-check` is local only. It runs Bash syntax, Rust fmt, tests, and clippy. -`fault-run*` prebuilds the ignored `faults` test binary before the fault window, +`fault-run` prebuilds the ignored `faults` test binary before the fault window, then runs that binary directly. The runner reruns preflight before and after the build. ## Required Environment -Only these variables are required for regular fault scenarios: +Only these variables are required for non-static fault scenarios: ```bash export RUSTFS_FAULT_TEST_STORAGE_CLASS= @@ -98,16 +98,15 @@ resource cleanup remain under `e2e/src/framework/`. | `RUSTFS_FAULT_TEST_TENANT` | `fault-test-tenant` | Tenant name. | | `RUSTFS_FAULT_TEST_CHAOS_NAMESPACE` | `chaos-mesh` | Chaos Mesh namespace. | | `RUSTFS_FAULT_TEST_USE_CLUSTER_IP` | `false` | Set to `1` when the runner can reach Service ClusterIPs. | -| `RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS` | `40000` | Total object count; must be at least 4. | +| `RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS` | `40000` | Total object count; must be at least 12. | | `RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY` | `80` | S3 workload concurrency; must be 1 through object count. | | `RUSTFS_FAULT_TEST_DURATION_SECONDS` | `7200` | Maximum fault TTL. Successful runs recover earlier. | | `RUSTFS_FAULT_TEST_REQUEST_TIMEOUT_SECONDS` | `30` | Per S3 request timeout. | | `RUSTFS_FAULT_TEST_TIMEOUT_SECONDS` | `300` | Kubernetes wait timeout. | | `RUSTFS_FAULT_TEST_SEED` | generated | Reuse a workload plan. | -| `RUSTFS_FAULT_TEST_REQUIRE_CLIENT_DISRUPTION` | `false` | Require at least one client-visible failed/timeout/unknown S3 operation. | +| `RUSTFS_FAULT_TEST_REQUIRE_CLIENT_DISRUPTION` | `false` | Force at least one client-visible failed/timeout/unknown S3 operation even when the catalog marks disruption optional. | | `RUSTFS_FAULT_TEST_BUILD_JOBS` | `1` | Cargo prebuild job count. | | `RUSTFS_FAULT_TEST_RUN_ROOT` | timestamped target dir | Artifact root. | -| `RUSTFS_FAULT_TEST_SCENARIOS` | regular scenario list | Space-separated list for `fault-run-regular`. | For a small rehearsal run: @@ -117,11 +116,10 @@ export RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY=8 make -C e2e fault-run SCENARIO=io-eio ``` -`RUSTFS_FAULT_TEST_PERCENT` applies only to percent-based IOChaos scenarios: -`io-eio`, `io-read-mistake`, `disk-full`, and `warp-under-chaos`. Fixed-target -scenarios such as `pod-kill-one`, `network-partition-one`, and `dm-flakey` -reject a percent override. Their run metadata records `percent: null` and -`fault_selection` such as `1 target(s)`. +`RUSTFS_FAULT_TEST_PERCENT` applies only when the Rust scenario catalog marks +the scenario as percent-based. Fixed-target scenarios reject a percent override. +Run `make -C e2e fault-list` or `cargo run --manifest-path e2e/Cargo.toml --bin +rustfs-e2e -- fault-catalog-json` to inspect the current catalog. ## Cluster Preparation @@ -137,9 +135,9 @@ kubectl get storageclass Requirements: - The current context must be a real Kubernetes or K3s cluster, not `kind-*`. -- At least four schedulable Ready nodes are required for the current regular +- At least four schedulable Ready nodes are required for the current default Tenant shape. -- Regular scenarios need a dedicated dynamic StorageClass. +- Non-static scenarios need a dedicated dynamic StorageClass. - `dm-flakey` needs a dedicated static Local PV StorageClass and explicit device-mapper variables. - Other Tenants in the cluster are intentionally ignored by preflight, health @@ -153,7 +151,7 @@ kubectl -n kube-system get configmap local-path-config -o yaml df -h ``` -Chaos Mesh is required for regular scenarios. The validated version is v2.8.3: +Chaos Mesh is required for Chaos Mesh-backed scenarios. The validated version is v2.8.3: ```bash helm repo add chaos-mesh https://charts.chaos-mesh.org @@ -166,7 +164,7 @@ helm upgrade --install chaos-mesh chaos-mesh/chaos-mesh \ --wait --timeout 10m kubectl -n chaos-mesh get deployment,daemonset -kubectl get crd iochaos.chaos-mesh.org podchaos.chaos-mesh.org networkchaos.chaos-mesh.org +kubectl get crd iochaos.chaos-mesh.org podchaos.chaos-mesh.org networkchaos.chaos-mesh.org stresschaos.chaos-mesh.org ``` Use the actual runtime socket for non-K3s clusters. @@ -194,10 +192,11 @@ Use the actual runtime socket for non-K3s clusters. make -C e2e fault-run SCENARIO=io-eio ``` -4. Run the regular suite: +4. List the catalog and run each selected scenario explicitly: ```bash - make -C e2e fault-run-regular + make -C e2e fault-list + make -C e2e fault-run SCENARIO=network-delay ``` 5. Collect artifacts, then clean owned resources: @@ -206,33 +205,21 @@ Use the actual runtime socket for non-K3s clusters. make -C e2e fault-cleanup ``` -## Regular Scenarios +## Scenario Catalog -Current regular scenarios run serially and stop on the first failure: - -```text -io-eio -pod-kill-one -network-partition-one -io-read-mistake -disk-full -warp-under-chaos -``` - -The current catalog maps each scenario to one fault. The internal plan model is -not limited to that: a future scenario can contain multiple `FaultInjection` -entries and can use fixed target counts for multiple Pods or percent selection -for IO faults. - -To run a subset: +The Rust catalog in `e2e/src/fault/scenarios.rs` is the only maintained scenario +source of truth. The shell runner and this guide query that catalog instead of +duplicating scenario names, percent rules, CRDs, tools, or impact policy. ```bash -export RUSTFS_FAULT_TEST_SCENARIOS='pod-kill-one network-partition-one' -make -C e2e fault-run-regular -unset RUSTFS_FAULT_TEST_SCENARIOS +make -C e2e fault-list +cargo run --manifest-path e2e/Cargo.toml --bin rustfs-e2e -- fault-catalog-json ``` -`warp-under-chaos` also requires `warp` in `PATH`. +Each run names exactly one scenario with `SCENARIO=`. SRE-owned scheduling +or automation should live outside this repository and call the same explicit +command for the desired scenario. Tool requirements, such as `warp` for +`warp-under-chaos`, are read from the Rust catalog during preflight. ## Artifacts And Pass Criteria @@ -250,6 +237,7 @@ workload-plan.json history.jsonl workload-summary.json recommit-report.json +checker-pre-recommit-report.json checker-report.json fault-evidence.json chaos-manifest.yaml @@ -266,11 +254,12 @@ A successful run must show: - `fault-evidence.json`: `injected`, `active_during_workload`, and `recovered` are `true`. -- `checker-report.json`: `passed` is `true`, committed object count equals the - selected workload object count, and missing objects, hash mismatches, - successful corrupted reads, and LIST warnings are empty. -- `recommit-report.json`: every previously unconfirmed PUT was recommitted - after recovery. +- `checker-pre-recommit-report.json` and `checker-report.json`: `passed` is + `true`; expected live objects are GET+sha256 verified; missing objects, hash + mismatches, successful corrupted reads, unexpected visible deleted objects, + and LIST warnings are empty. +- `recommit-report.json`: every previously unconfirmed write was recommitted + and GET verified after recovery. - `workload-plan.json`: object count, concurrency, and payload distribution are internally consistent with the selected environment values. @@ -292,7 +281,7 @@ Manual checks: ```bash kubectl -n "${RUSTFS_FAULT_TEST_CHAOS_NAMESPACE:-chaos-mesh}" \ - get iochaos,podchaos,networkchaos \ + get iochaos,podchaos,networkchaos,stresschaos \ -l app.kubernetes.io/managed-by=rustfs-operator-fault-test kubectl get namespace "${RUSTFS_FAULT_TEST_NAMESPACE:-rustfs-fault-test}" @@ -300,8 +289,8 @@ kubectl get namespace "${RUSTFS_FAULT_TEST_NAMESPACE:-rustfs-fault-test}" ## dm-flakey -`dm-flakey` is separate from the regular suite. It needs a dedicated static -Local PV setup and privileged helper access on the fault namespace. +`dm-flakey` is an explicit scenario that needs a dedicated static Local PV setup +and privileged helper access on the fault namespace. There is no Make target that installs this environment. Prepare the host storage and Kubernetes Local PVs first, then use `fault-preflight` to verify them. diff --git a/e2e/Makefile b/e2e/Makefile index 5752d34..9f318ff 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -18,7 +18,7 @@ FAULT_SCRIPT := $(CURDIR)/scripts/fault-test.sh MANIFEST := $(CURDIR)/Cargo.toml FAULT_BUILD_JOBS ?= 1 -.PHONY: help fault-check fault-preflight fault-run fault-run-regular fault-run-dm fault-cleanup +.PHONY: help fault-check fault-list fault-preflight fault-run fault-run-dm fault-cleanup help: @echo "RustFS e2e fault-test package" @@ -26,8 +26,8 @@ help: @echo "Usage:" @echo " make -C e2e fault-check" @echo " make -C e2e fault-preflight [SCENARIO=io-eio]" + @echo " make -C e2e fault-list" @echo " make -C e2e fault-run SCENARIO=io-eio" - @echo " make -C e2e fault-run-regular" @echo " make -C e2e fault-run-dm" @echo " make -C e2e fault-cleanup" @echo "" @@ -45,6 +45,9 @@ fault-check: CARGO_BUILD_JOBS=$(FAULT_BUILD_JOBS) cargo test --manifest-path $(MANIFEST) CARGO_BUILD_JOBS=$(FAULT_BUILD_JOBS) cargo clippy --manifest-path $(MANIFEST) --all-targets -- -D warnings +fault-list: + @bash $(FAULT_SCRIPT) list + fault-preflight: @bash $(FAULT_SCRIPT) preflight "$(or $(SCENARIO),io-eio)" @@ -52,9 +55,6 @@ fault-run: @test -n "$(SCENARIO)" || (echo "SCENARIO is required" >&2; exit 2) @bash $(FAULT_SCRIPT) run "$(SCENARIO)" -fault-run-regular: - @bash $(FAULT_SCRIPT) run-regular - fault-run-dm: @bash $(FAULT_SCRIPT) run dm-flakey diff --git a/e2e/scripts/fault-test.sh b/e2e/scripts/fault-test.sh index 35eeed7..d536a6d 100644 --- a/e2e/scripts/fault-test.sh +++ b/e2e/scripts/fault-test.sh @@ -20,7 +20,6 @@ PACKAGE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" MANIFEST="$PACKAGE_DIR/Cargo.toml" MANAGER="rustfs-operator-fault-test" MANAGER_SELECTOR="app.kubernetes.io/managed-by=$MANAGER" -DEFAULT_SCENARIOS="io-eio pod-kill-one network-partition-one io-read-mistake disk-full warp-under-chaos" WORKLOAD_OBJECTS="${RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS:-40000}" WORKLOAD_CONCURRENCY="${RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY:-80}" BUILD_JOBS="${RUSTFS_FAULT_TEST_BUILD_JOBS:-1}" @@ -32,6 +31,7 @@ CHAOS_NAMESPACE="${RUSTFS_FAULT_TEST_CHAOS_NAMESPACE:-chaos-mesh}" ACTIVE_PID="" ACTIVE_ARTIFACTS="" FAULT_TEST_BINARY="" +FAULT_CATALOG_JSON="" usage() { cat <<'EOF' @@ -40,7 +40,7 @@ Usage: fault-test.sh [scenario] Commands: preflight [scenario] Validate the current real-cluster environment. run Run one destructive scenario with health guards. - run-regular Run the six regular scenarios serially. + list List catalog scenarios. cleanup Remove managed Chaos and the owned fault namespace. RUSTFS_FAULT_TEST_EXPECTED_CONTEXT is optional. When unset, the current @@ -159,35 +159,44 @@ kubectl_cluster() { kubectl --context "$FAULT_CONTEXT" "$@" } +fault_catalog_json() { + if [[ -z "$FAULT_CATALOG_JSON" ]]; then + FAULT_CATALOG_JSON="$(CARGO_BUILD_JOBS="$BUILD_JOBS" cargo run --quiet --manifest-path "$MANIFEST" --bin rustfs-e2e -- fault-catalog-json)" + fi + printf '%s\n' "$FAULT_CATALOG_JSON" +} + +catalog_scenario_query() { + local scenario="$1" + shift + fault_catalog_json | jq -e --arg scenario "$scenario" "$@" +} + is_supported_scenario() { - case "$1" in - io-eio|pod-kill-one|network-partition-one|io-read-mistake|disk-full|warp-under-chaos|dm-flakey) - return 0 - ;; - *) - return 1 - ;; - esac + catalog_scenario_query "$1" 'any(.[]; .scenario == $scenario and .status == "executable")' >/dev/null } -is_percent_scenario() { - case "$1" in - io-eio|io-read-mistake|disk-full|warp-under-chaos) - return 0 - ;; - *) - return 1 - ;; - esac +require_supported_scenario() { + local scenario="$1" + is_supported_scenario "$scenario" || die "unsupported scenario: $scenario" } -scenario_crd() { - case "$1" in - pod-kill-one) echo "podchaos.chaos-mesh.org" ;; - network-partition-one) echo "networkchaos.chaos-mesh.org" ;; - dm-flakey) echo "" ;; - *) echo "iochaos.chaos-mesh.org" ;; - esac +scenario_percent_supported() { + catalog_scenario_query "$1" '.[] | select(.scenario == $scenario) | .percent_supported' >/dev/null +} + +scenario_requires_static_storage() { + catalog_scenario_query "$1" '.[] | select(.scenario == $scenario) | .isolation == "dedicated-linux-block-device"' >/dev/null +} + +scenario_crds() { + local scenario="$1" + fault_catalog_json | jq -r --arg scenario "$scenario" '.[] | select(.scenario == $scenario) | .crds[]?' +} + +scenario_required_tools() { + local scenario="$1" + fault_catalog_json | jq -r --arg scenario "$scenario" '.[] | select(.scenario == $scenario) | .required_tools[]?' } validate_runtime_env_contract() { @@ -198,7 +207,7 @@ validate_runtime_env_contract() { BUILD_JOBS="$(trim_value "$BUILD_JOBS")" require_positive_integer RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS "$WORKLOAD_OBJECTS" - (( 10#$WORKLOAD_OBJECTS >= 4 )) || die "RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS must be at least 4" + (( 10#$WORKLOAD_OBJECTS >= 12 )) || die "RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS must be at least 12" require_positive_integer RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY "$WORKLOAD_CONCURRENCY" (( 10#$WORKLOAD_CONCURRENCY <= 10#$WORKLOAD_OBJECTS )) || die "RUSTFS_FAULT_TEST_WORKLOAD_CONCURRENCY must be <= RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS" require_optional_positive_integer RUSTFS_FAULT_TEST_DURATION_SECONDS @@ -213,7 +222,7 @@ validate_runtime_env_contract() { if [[ -n "$percent" ]]; then require_positive_integer RUSTFS_FAULT_TEST_PERCENT "$percent" (( 10#$percent <= 100 )) || die "RUSTFS_FAULT_TEST_PERCENT must be in 1..=100" - is_percent_scenario "$scenario" || die "RUSTFS_FAULT_TEST_PERCENT only applies to percent-based IOChaos scenarios" + scenario_percent_supported "$scenario" || die "RUSTFS_FAULT_TEST_PERCENT does not apply to scenario $scenario" export RUSTFS_FAULT_TEST_PERCENT="$percent" fi } @@ -322,14 +331,14 @@ require_storage_class() { ')" [[ "$pv_count" -eq 4 ]] || die "dm-flakey requires exactly four Available/Bound 100Gi PVs, found $pv_count" else - [[ "$provisioner" != "kubernetes.io/no-provisioner" ]] || die "regular scenarios require dynamic provisioning" + [[ "$provisioner" != "kubernetes.io/no-provisioner" ]] || die "non-static scenarios require dynamic provisioning" fi } preflight() { local scenario="${1:-io-eio}" - local ready_nodes crd - is_supported_scenario "$scenario" || die "unsupported scenario: $scenario" + local ready_nodes crd tool + require_supported_scenario "$scenario" require_command cargo require_command jq @@ -350,14 +359,15 @@ preflight() { require_storage_class "$scenario" require_namespace_ownership - if [[ "$scenario" != "dm-flakey" ]]; then - crd="$(scenario_crd "$scenario")" - kubectl_cluster get crd "$crd" >/dev/null + if ! scenario_requires_static_storage "$scenario"; then + for crd in $(scenario_crds "$scenario"); do + kubectl_cluster get crd "$crd" >/dev/null + done require_chaos_ready fi - if [[ "$scenario" == "warp-under-chaos" ]]; then - require_command warp - fi + for tool in $(scenario_required_tools "$scenario"); do + require_command "$tool" + done if [[ "$scenario" == "dm-flakey" ]]; then validate_dm_env_contract kubectl_cluster get namespace "$FAULT_NAMESPACE" >/dev/null 2>&1 || die "dm-flakey requires a pre-created owned fault namespace with privileged Pod Security" @@ -375,7 +385,7 @@ preflight_cleanup() { } cleanup_managed_chaos() { - kubectl_ns "$CHAOS_NAMESPACE" delete iochaos,podchaos,networkchaos \ + kubectl_ns "$CHAOS_NAMESPACE" delete iochaos,podchaos,networkchaos,stresschaos \ -l "$MANAGER_SELECTOR" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true } @@ -409,7 +419,7 @@ capture_cluster_snapshot() { kubectl_ns "$FAULT_NAMESPACE" get pods -o wide >"$artifacts/pods-$stage.txt" 2>&1 || true kubectl_ns "$FAULT_NAMESPACE" get pvc -o wide >"$artifacts/pvcs-$stage.txt" 2>&1 || true kubectl_cluster get pv -o wide >"$artifacts/pvs-$stage.txt" 2>&1 || true - kubectl_ns "$CHAOS_NAMESPACE" get iochaos,podchaos,networkchaos -o yaml >"$artifacts/chaos-$stage.yaml" 2>&1 || true + kubectl_ns "$CHAOS_NAMESPACE" get iochaos,podchaos,networkchaos,stresschaos -o yaml >"$artifacts/chaos-$stage.yaml" 2>&1 || true kubectl_ns "$FAULT_NAMESPACE" get events --sort-by=.lastTimestamp >"$artifacts/events-$stage.txt" 2>&1 || true } @@ -437,16 +447,18 @@ find_artifact() { validate_scenario_artifacts() { local scenario="$1" artifacts="$2" run_root="$3" - local metadata plan evidence checker summary recommit seed disruptions recommitted committed + local metadata plan evidence prechecker checker summary recommit seed disruptions recommitted committed metadata="$(find_artifact "$artifacts" run-metadata.json)" plan="$(find_artifact "$artifacts" workload-plan.json)" evidence="$(find_artifact "$artifacts" fault-evidence.json)" + prechecker="$(find_artifact "$artifacts" checker-pre-recommit-report.json)" checker="$(find_artifact "$artifacts" checker-report.json)" summary="$(find_artifact "$artifacts" workload-summary.json)" recommit="$(find_artifact "$artifacts" recommit-report.json)" [[ -f "$metadata" ]] || die "$scenario did not produce run-metadata.json" [[ -f "$plan" ]] || die "$scenario did not produce workload-plan.json" [[ -f "$evidence" ]] || die "$scenario did not produce fault-evidence.json" + [[ -f "$prechecker" ]] || die "$scenario did not produce checker-pre-recommit-report.json" [[ -f "$checker" ]] || die "$scenario did not produce checker-report.json" [[ -f "$summary" ]] || die "$scenario did not produce workload-summary.json" [[ -f "$recommit" ]] || die "$scenario did not produce recommit-report.json" @@ -468,11 +480,20 @@ validate_scenario_artifacts() { ' "$plan" >/dev/null || die "$scenario workload plan does not match the required profile" jq -e '.injected == true and .active_during_workload == true and .recovered == true' "$evidence" >/dev/null || die "$scenario fault evidence is incomplete" jq -e '(.active_snapshots | length) > 0 and (.workload_snapshots | length) > 0' "$evidence" >/dev/null || die "$scenario fault evidence snapshots are missing" - jq -e --argjson objects "$WORKLOAD_OBJECTS" ' - .committed_puts == $objects - and (.missing_committed_objects | length) == 0 + jq -e ' + (.missing_committed_objects | length) == 0 + and (.hash_mismatches | length) == 0 + and (.successful_corrupted_reads | length) == 0 + and (.unexpected_visible_deleted_objects | length) == 0 + and (.list_warnings | length) == 0 + and .tenant_recovered == true + and .passed == true + ' "$prechecker" >/dev/null || die "$scenario pre-recommit checker verdict failed" + jq -e ' + (.missing_committed_objects | length) == 0 and (.hash_mismatches | length) == 0 and (.successful_corrupted_reads | length) == 0 + and (.unexpected_visible_deleted_objects | length) == 0 and (.list_warnings | length) == 0 and .tenant_recovered == true and .passed == true @@ -602,7 +623,7 @@ initialize_summary() { run_one() { local scenario="$1" run_root - is_supported_scenario "$scenario" || die "unsupported scenario: $scenario" + require_supported_scenario "$scenario" run_root="$(new_run_root)" initialize_summary "$run_root" prepare_fault_binary "$scenario" "$run_root" @@ -610,20 +631,8 @@ run_one() { echo "run artifacts: $run_root" } -run_regular() { - local run_root scenario prepared=false - local scenarios="${RUSTFS_FAULT_TEST_SCENARIOS:-$DEFAULT_SCENARIOS}" - run_root="$(new_run_root)" - initialize_summary "$run_root" - for scenario in $scenarios; do - [[ "$scenario" != "dm-flakey" ]] || die "run-regular cannot include dm-flakey" - if [[ "$prepared" == "false" ]]; then - prepare_fault_binary "$scenario" "$run_root" - prepared=true - fi - run_scenario "$scenario" "$run_root" || return $? - done - echo "regular scenario artifacts: $run_root" +list_scenarios() { + fault_catalog_json | jq -r '.[] | .scenario' } cleanup() { @@ -632,7 +641,7 @@ cleanup() { require_namespace_ownership kubectl_cluster delete namespace "$FAULT_NAMESPACE" --wait=true fi - if kubectl_ns "$CHAOS_NAMESPACE" get iochaos,podchaos,networkchaos -l "$MANAGER_SELECTOR" -o name 2>/dev/null | grep -q .; then + if kubectl_ns "$CHAOS_NAMESPACE" get iochaos,podchaos,networkchaos,stresschaos -l "$MANAGER_SELECTOR" -o name 2>/dev/null | grep -q .; then die "managed Chaos resources remain after cleanup" fi echo "managed fault-test resources cleaned; external StorageClasses, PVs, and host devices were not changed" @@ -651,8 +660,9 @@ case "${1:-help}" in [[ -n "${2:-}" ]] || die "scenario is required" run_one "$2" ;; - run-regular) - run_regular + list) + [[ -z "${2:-}" ]] || die "list does not accept arguments; run a named scenario with: fault-test.sh run " + list_scenarios ;; cleanup) preflight_cleanup diff --git a/e2e/src/bin/rustfs-e2e.rs b/e2e/src/bin/rustfs-e2e.rs index b662ac4..dfac2a9 100644 --- a/e2e/src/bin/rustfs-e2e.rs +++ b/e2e/src/bin/rustfs-e2e.rs @@ -13,28 +13,37 @@ // limitations under the License. use anyhow::{Result, bail}; -use rustfs_operator_e2e::framework::{ - cert_manager_tls, command::CommandSpec, config::E2eConfig, deploy, images::ImageSet, - kind::KindCluster, live, resources, storage, +use rustfs_operator_e2e::{ + fault::scenarios::scenario_catalog_json, + framework::{ + cert_manager_tls, command::CommandSpec, config::E2eConfig, deploy, images::ImageSet, + kind::KindCluster, live, resources, storage, + }, }; fn main() -> Result<()> { - let command = std::env::args() - .nth(1) - .unwrap_or_else(|| "help".to_string()); - let config = E2eConfig::from_env(); + let mut args = std::env::args().skip(1); + let command = args.next().unwrap_or_else(|| "help".to_string()); match command.as_str() { "help" | "--help" | "-h" => print_help(), - "assert-context" => assert_context(&config), - "kind-create" => create_kind_cluster(&config), - "kind-delete" => delete_kind_cluster(&config), - "sanitize-live-storage" => sanitize_live_storage(&config), - "reset-live-fixtures" | "reset-live-smoke-fixture" => reset_live_fixtures(&config), - "kind-load-images" => load_images(&config), - "deploy-dev" => deploy_dev(&config), - "rollout-dev" => rollout_dev(&config), - unknown => bail!("unknown rustfs-e2e internal command: {unknown}; run `rustfs-e2e help`"), + "fault-catalog-json" => print_fault_catalog_json(), + _ => { + let config = E2eConfig::from_env(); + match command.as_str() { + "assert-context" => assert_context(&config), + "kind-create" => create_kind_cluster(&config), + "kind-delete" => delete_kind_cluster(&config), + "sanitize-live-storage" => sanitize_live_storage(&config), + "reset-live-fixtures" | "reset-live-smoke-fixture" => reset_live_fixtures(&config), + "kind-load-images" => load_images(&config), + "deploy-dev" => deploy_dev(&config), + "rollout-dev" => rollout_dev(&config), + unknown => { + bail!("unknown rustfs-e2e internal command: {unknown}; run `rustfs-e2e help`") + } + } + } } } @@ -58,6 +67,12 @@ fn print_help() -> Result<()> { ); println!(" deploy-dev Apply operator/console manifests into dedicated Kind"); println!(" rollout-dev Restart and wait for e2e control-plane deployments"); + println!(" fault-catalog-json"); + Ok(()) +} + +fn print_fault_catalog_json() -> Result<()> { + println!("{}", scenario_catalog_json()?); Ok(()) } diff --git a/e2e/src/fault/backends/chaos_mesh.rs b/e2e/src/fault/backends/chaos_mesh.rs index 0cf0700..2da653b 100644 --- a/e2e/src/fault/backends/chaos_mesh.rs +++ b/e2e/src/fault/backends/chaos_mesh.rs @@ -22,6 +22,7 @@ use crate::framework::{config::ClusterTestConfig, kubectl::Kubectl}; const IOCHAOS_CRD: &str = "iochaos.chaos-mesh.org"; const PODCHAOS_CRD: &str = "podchaos.chaos-mesh.org"; const NETWORKCHAOS_CRD: &str = "networkchaos.chaos-mesh.org"; +const STRESSCHAOS_CRD: &str = "stresschaos.chaos-mesh.org"; const RUN_ID_LABEL: &str = "rustfs-fault-test/run-id"; const SCENARIO_LABEL: &str = "rustfs-fault-test/scenario"; const MANAGED_BY_LABEL: &str = "app.kubernetes.io/managed-by"; @@ -32,6 +33,9 @@ pub enum IoChaosAction { Fault { errno: u8, }, + Latency { + delay: String, + }, Mistake { filling: String, max_occurrences: u8, @@ -55,6 +59,12 @@ pub struct IoChaosSpec { pub duration: Duration, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PodChaosAction { + PodKill, + PodFailure { duration: Duration }, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct PodChaosSpec { pub name: String, @@ -63,6 +73,29 @@ pub struct PodChaosSpec { pub scenario: String, pub target_namespace: String, pub tenant_name: String, + pub action: PodChaosAction, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NetworkChaosAction { + Partition, + Delay { + latency: String, + jitter: String, + correlation: String, + }, + Loss { + loss: String, + correlation: String, + }, + Corrupt { + corrupt: String, + correlation: String, + }, + Duplicate { + duplicate: String, + correlation: String, + }, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -73,6 +106,25 @@ pub struct NetworkChaosSpec { pub scenario: String, pub target_namespace: String, pub tenant_name: String, + pub action: NetworkChaosAction, + pub duration: Duration, +} + +#[derive(Debug, Clone)] +pub enum StressChaosAction { + Cpu { workers: u32, load: u32 }, + Memory { workers: u32, size: String }, +} + +#[derive(Debug, Clone)] +pub struct StressChaosSpec { + pub name: String, + pub namespace: String, + pub run_id: String, + pub scenario: String, + pub target_namespace: String, + pub tenant_name: String, + pub action: StressChaosAction, pub duration: Duration, } @@ -166,6 +218,46 @@ impl IoChaosSpec { }) } + pub fn latency_on_rustfs_volume( + config: &ClusterTestConfig, + chaos_namespace: impl Into, + run_id: impl Into, + scenario: impl Into, + volume_path: impl Into, + percent: u8, + duration: Duration, + ) -> Result { + ensure!( + (1..=100).contains(&percent), + "IOChaos percent must be in 1..=100, got {percent}" + ); + ensure!( + duration > Duration::ZERO, + "IOChaos duration must be positive" + ); + + let run_id = run_id.into(); + let short_run_id = run_id.chars().take(12).collect::(); + let scenario = scenario.into(); + + Ok(Self { + name: format!("rustfs-fault-io-latency-{short_run_id}"), + namespace: chaos_namespace.into(), + run_id, + scenario, + target_namespace: config.test_namespace.clone(), + tenant_name: config.tenant_name.clone(), + container_name: "rustfs".to_string(), + volume_path: volume_path.into(), + methods: vec!["READ".to_string(), "WRITE".to_string()], + action: IoChaosAction::Latency { + delay: "250ms".to_string(), + }, + percent, + duration, + }) + } + pub fn enospc_on_rustfs_volume( config: &ClusterTestConfig, chaos_namespace: impl Into, @@ -269,6 +361,9 @@ spec: IoChaosAction::Fault { errno } => { format!(" action: fault\n errno: {errno}") } + IoChaosAction::Latency { delay } => { + format!(" action: latency\n delay: {delay}") + } IoChaosAction::Mistake { filling, max_occurrences, @@ -300,15 +395,42 @@ impl PodChaosSpec { scenario: scenario.into(), target_namespace: config.test_namespace.clone(), tenant_name: config.tenant_name.clone(), + action: PodChaosAction::PodKill, } } + pub fn fail_one_rustfs_pod( + config: &ClusterTestConfig, + chaos_namespace: impl Into, + run_id: impl Into, + scenario: impl Into, + duration: Duration, + ) -> Result { + ensure!( + duration > Duration::ZERO, + "PodChaos duration must be positive" + ); + + let run_id = run_id.into(); + let short_run_id = run_id.chars().take(12).collect::(); + Ok(Self { + name: format!("rustfs-fault-pod-failure-{short_run_id}"), + namespace: chaos_namespace.into(), + run_id, + scenario: scenario.into(), + target_namespace: config.test_namespace.clone(), + tenant_name: config.tenant_name.clone(), + action: PodChaosAction::PodFailure { duration }, + }) + } + pub fn with_name_suffix(mut self, suffix: &str) -> Self { self.name.push_str(suffix); self } pub fn manifest(&self) -> String { + let action = self.action_manifest(); format!( r#"apiVersion: chaos-mesh.org/v1alpha1 kind: PodChaos @@ -320,7 +442,7 @@ metadata: {scenario_label}: "{scenario}" {managed_by_label}: {managed_by_value} spec: - action: pod-kill +{action} mode: one selector: namespaces: @@ -338,8 +460,21 @@ spec: managed_by_value = MANAGED_BY_VALUE, target_namespace = self.target_namespace, tenant_name = self.tenant_name, + action = action, ) } + + fn action_manifest(&self) -> String { + match self.action { + PodChaosAction::PodKill => " action: pod-kill".to_string(), + PodChaosAction::PodFailure { duration } => { + format!( + " action: pod-failure\n duration: \"{}s\"", + duration.as_secs() + ) + } + } + } } impl NetworkChaosSpec { @@ -349,6 +484,111 @@ impl NetworkChaosSpec { run_id: impl Into, scenario: impl Into, duration: Duration, + ) -> Result { + Self::one_rustfs_pod( + config, + chaos_namespace, + run_id, + scenario, + duration, + "net-partition", + NetworkChaosAction::Partition, + ) + } + + pub fn delay_one_rustfs_pod( + config: &ClusterTestConfig, + chaos_namespace: impl Into, + run_id: impl Into, + scenario: impl Into, + duration: Duration, + ) -> Result { + Self::one_rustfs_pod( + config, + chaos_namespace, + run_id, + scenario, + duration, + "net-delay", + NetworkChaosAction::Delay { + latency: "200ms".to_string(), + jitter: "50ms".to_string(), + correlation: "25".to_string(), + }, + ) + } + + pub fn loss_one_rustfs_pod( + config: &ClusterTestConfig, + chaos_namespace: impl Into, + run_id: impl Into, + scenario: impl Into, + duration: Duration, + ) -> Result { + Self::one_rustfs_pod( + config, + chaos_namespace, + run_id, + scenario, + duration, + "net-loss", + NetworkChaosAction::Loss { + loss: "25".to_string(), + correlation: "25".to_string(), + }, + ) + } + + pub fn corrupt_one_rustfs_pod( + config: &ClusterTestConfig, + chaos_namespace: impl Into, + run_id: impl Into, + scenario: impl Into, + duration: Duration, + ) -> Result { + Self::one_rustfs_pod( + config, + chaos_namespace, + run_id, + scenario, + duration, + "net-corrupt", + NetworkChaosAction::Corrupt { + corrupt: "5".to_string(), + correlation: "25".to_string(), + }, + ) + } + + pub fn duplicate_one_rustfs_pod( + config: &ClusterTestConfig, + chaos_namespace: impl Into, + run_id: impl Into, + scenario: impl Into, + duration: Duration, + ) -> Result { + Self::one_rustfs_pod( + config, + chaos_namespace, + run_id, + scenario, + duration, + "net-duplicate", + NetworkChaosAction::Duplicate { + duplicate: "10".to_string(), + correlation: "25".to_string(), + }, + ) + } + + fn one_rustfs_pod( + config: &ClusterTestConfig, + chaos_namespace: impl Into, + run_id: impl Into, + scenario: impl Into, + duration: Duration, + name_action: &str, + action: NetworkChaosAction, ) -> Result { ensure!( duration > Duration::ZERO, @@ -358,12 +598,13 @@ impl NetworkChaosSpec { let run_id = run_id.into(); let short_run_id = run_id.chars().take(12).collect::(); Ok(Self { - name: format!("rustfs-fault-net-partition-{short_run_id}"), + name: format!("rustfs-fault-{name_action}-{short_run_id}"), namespace: chaos_namespace.into(), run_id, scenario: scenario.into(), target_namespace: config.test_namespace.clone(), tenant_name: config.tenant_name.clone(), + action, duration, }) } @@ -375,6 +616,7 @@ impl NetworkChaosSpec { pub fn manifest(&self) -> String { let seconds = self.duration.as_secs(); + let action = self.action_manifest(); format!( r#"apiVersion: chaos-mesh.org/v1alpha1 kind: NetworkChaos @@ -386,7 +628,7 @@ metadata: {scenario_label}: "{scenario}" {managed_by_label}: {managed_by_value} spec: - action: partition +{action} mode: one selector: namespaces: @@ -413,8 +655,180 @@ spec: managed_by_value = MANAGED_BY_VALUE, target_namespace = self.target_namespace, tenant_name = self.tenant_name, + action = action, ) } + + fn action_manifest(&self) -> String { + match &self.action { + NetworkChaosAction::Partition => " action: partition".to_string(), + NetworkChaosAction::Delay { + latency, + jitter, + correlation, + } => format!( + r#" action: delay + delay: + latency: "{latency}" + jitter: "{jitter}" + correlation: "{correlation}""# + ), + NetworkChaosAction::Loss { loss, correlation } => format!( + r#" action: loss + loss: + loss: "{loss}" + correlation: "{correlation}""# + ), + NetworkChaosAction::Corrupt { + corrupt, + correlation, + } => format!( + r#" action: corrupt + corrupt: + corrupt: "{corrupt}" + correlation: "{correlation}""# + ), + NetworkChaosAction::Duplicate { + duplicate, + correlation, + } => format!( + r#" action: duplicate + duplicate: + duplicate: "{duplicate}" + correlation: "{correlation}""# + ), + } + } +} + +impl StressChaosSpec { + pub fn cpu_on_one_rustfs_pod( + config: &ClusterTestConfig, + chaos_namespace: impl Into, + run_id: impl Into, + scenario: impl Into, + duration: Duration, + ) -> Result { + Self::one_rustfs_pod( + config, + chaos_namespace, + run_id, + scenario, + duration, + "stress-cpu", + StressChaosAction::Cpu { + workers: 1, + load: 80, + }, + ) + } + + pub fn memory_on_one_rustfs_pod( + config: &ClusterTestConfig, + chaos_namespace: impl Into, + run_id: impl Into, + scenario: impl Into, + duration: Duration, + ) -> Result { + Self::one_rustfs_pod( + config, + chaos_namespace, + run_id, + scenario, + duration, + "stress-memory", + StressChaosAction::Memory { + workers: 1, + size: "512MiB".to_string(), + }, + ) + } + + fn one_rustfs_pod( + config: &ClusterTestConfig, + chaos_namespace: impl Into, + run_id: impl Into, + scenario: impl Into, + duration: Duration, + name_action: &str, + action: StressChaosAction, + ) -> Result { + ensure!( + duration > Duration::ZERO, + "StressChaos duration must be positive" + ); + + let run_id = run_id.into(); + let short_run_id = run_id.chars().take(12).collect::(); + Ok(Self { + name: format!("rustfs-fault-{name_action}-{short_run_id}"), + namespace: chaos_namespace.into(), + run_id, + scenario: scenario.into(), + target_namespace: config.test_namespace.clone(), + tenant_name: config.tenant_name.clone(), + action, + duration, + }) + } + + pub fn with_name_suffix(mut self, suffix: &str) -> Self { + self.name.push_str(suffix); + self + } + + pub fn manifest(&self) -> String { + let seconds = self.duration.as_secs(); + let stressors = self.stressors_manifest(); + format!( + r#"apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: {name} + namespace: {namespace} + labels: + {run_id_label}: "{run_id}" + {scenario_label}: "{scenario}" + {managed_by_label}: {managed_by_value} +spec: + mode: one + selector: + namespaces: + - {target_namespace} + labelSelectors: + rustfs.tenant: {tenant_name} + stressors: +{stressors} + duration: "{seconds}s" +"#, + name = self.name, + namespace = self.namespace, + run_id_label = RUN_ID_LABEL, + run_id = self.run_id, + scenario_label = SCENARIO_LABEL, + scenario = self.scenario, + managed_by_label = MANAGED_BY_LABEL, + managed_by_value = MANAGED_BY_VALUE, + target_namespace = self.target_namespace, + tenant_name = self.tenant_name, + stressors = stressors, + ) + } + + fn stressors_manifest(&self) -> String { + match &self.action { + StressChaosAction::Cpu { workers, load } => format!( + r#" cpu: + workers: {workers} + load: {load}"# + ), + StressChaosAction::Memory { workers, size } => format!( + r#" memory: + workers: {workers} + size: "{size}""# + ), + } + } } pub fn require_iochaos_crd(config: &ClusterTestConfig) -> Result<()> { @@ -429,6 +843,10 @@ pub fn require_networkchaos_crd(config: &ClusterTestConfig) -> Result<()> { require_crd(config, NETWORKCHAOS_CRD, "Chaos Mesh NetworkChaos") } +pub fn require_stresschaos_crd(config: &ClusterTestConfig) -> Result<()> { + require_crd(config, STRESSCHAOS_CRD, "Chaos Mesh StressChaos") +} + fn require_crd(config: &ClusterTestConfig, crd: &str, description: &str) -> Result<()> { let output = Kubectl::new(config).command(["get", "crd", crd]).run()?; ensure!( @@ -442,7 +860,7 @@ fn require_crd(config: &ClusterTestConfig, crd: &str, description: &str) -> Resu pub fn cleanup_run(config: &ClusterTestConfig, namespace: &str, run_id: &str) -> Result<()> { let selector = format!("{RUN_ID_LABEL}={run_id}"); - for kind in ["iochaos", "podchaos", "networkchaos"] { + for kind in ["iochaos", "podchaos", "networkchaos", "stresschaos"] { Kubectl::new(config) .namespaced(namespace) .command(["delete", kind, "-l", &selector, "--ignore-not-found"]) @@ -466,7 +884,7 @@ pub fn cleanup_run_kind( } pub fn cleanup_managed_chaos(config: &ClusterTestConfig, namespace: &str) -> Result<()> { - for kind in ["iochaos", "podchaos", "networkchaos"] { + for kind in ["iochaos", "podchaos", "networkchaos", "stresschaos"] { cleanup_managed_kind(config, namespace, kind)?; } Ok(()) @@ -484,6 +902,10 @@ pub fn cleanup_managed_networkchaos(config: &ClusterTestConfig, namespace: &str) cleanup_managed_kind(config, namespace, "networkchaos") } +pub fn cleanup_managed_stresschaos(config: &ClusterTestConfig, namespace: &str) -> Result<()> { + cleanup_managed_kind(config, namespace, "stresschaos") +} + fn cleanup_managed_kind(config: &ClusterTestConfig, namespace: &str, kind: &str) -> Result<()> { let selector = format!("{MANAGED_BY_LABEL}={MANAGED_BY_VALUE}"); Kubectl::new(config) @@ -541,6 +963,21 @@ pub fn apply_networkchaos( }) } +pub fn apply_stresschaos(config: &ClusterTestConfig, spec: &StressChaosSpec) -> Result { + Kubectl::new(config) + .namespaced(&spec.namespace) + .apply_yaml_command(spec.manifest()) + .run_checked()?; + + Ok(ChaosGuard { + config: config.clone(), + kind: "stresschaos", + namespace: spec.namespace.clone(), + name: spec.name.clone(), + deleted: false, + }) +} + impl ChaosGuard { pub fn kind(&self) -> &'static str { self.kind @@ -667,7 +1104,9 @@ impl Drop for ChaosGuard { #[cfg(test)] mod tests { - use super::{IoChaosSpec, chaos_experiment_is_active}; + use super::{ + IoChaosSpec, NetworkChaosSpec, PodChaosSpec, StressChaosSpec, chaos_experiment_is_active, + }; use crate::fault::config::FaultTestConfig; use std::time::Duration; @@ -718,6 +1157,103 @@ mod tests { assert!(!manifest.contains(" - READ")); } + #[test] + fn io_latency_manifest_targets_volume_reads_and_writes() { + let config = FaultTestConfig::for_test("real-cluster", "fast-csi"); + let spec = IoChaosSpec::latency_on_rustfs_volume( + &config.cluster, + "chaos-mesh", + "run-1234567890", + "io-latency", + "/data/rustfs0", + 20, + Duration::from_secs(60), + ) + .expect("valid latency chaos"); + let manifest = spec.manifest(); + + assert!(manifest.contains("action: latency")); + assert!(manifest.contains("delay: 250ms")); + assert!(manifest.contains("methods:\n - READ\n - WRITE")); + } + + #[test] + fn pod_failure_manifest_uses_duration() { + let config = FaultTestConfig::for_test("real-cluster", "fast-csi"); + let spec = PodChaosSpec::fail_one_rustfs_pod( + &config.cluster, + "chaos-mesh", + "run-1234567890", + "pod-failure", + Duration::from_secs(60), + ) + .expect("valid pod failure"); + let manifest = spec.manifest(); + + assert!(manifest.contains("kind: PodChaos")); + assert!(manifest.contains("action: pod-failure")); + assert!(manifest.contains("duration: \"60s\"")); + assert!(manifest.contains("rustfs.tenant: fault-test-tenant")); + } + + #[test] + fn network_delay_and_loss_manifests_use_targeted_actions() { + let config = FaultTestConfig::for_test("real-cluster", "fast-csi"); + let delay = NetworkChaosSpec::delay_one_rustfs_pod( + &config.cluster, + "chaos-mesh", + "run-1234567890", + "network-delay", + Duration::from_secs(60), + ) + .expect("valid network delay") + .manifest(); + let loss = NetworkChaosSpec::loss_one_rustfs_pod( + &config.cluster, + "chaos-mesh", + "run-1234567890", + "network-loss", + Duration::from_secs(60), + ) + .expect("valid network loss") + .manifest(); + + assert!(delay.contains("action: delay")); + assert!(delay.contains("latency: \"200ms\"")); + assert!(loss.contains("action: loss")); + assert!(loss.contains("loss: \"25\"")); + } + + #[test] + fn stress_manifests_target_one_rustfs_pod() { + let config = FaultTestConfig::for_test("real-cluster", "fast-csi"); + let cpu = StressChaosSpec::cpu_on_one_rustfs_pod( + &config.cluster, + "chaos-mesh", + "run-1234567890", + "stress-cpu", + Duration::from_secs(60), + ) + .expect("valid cpu stress") + .manifest(); + let memory = StressChaosSpec::memory_on_one_rustfs_pod( + &config.cluster, + "chaos-mesh", + "run-1234567890", + "stress-memory", + Duration::from_secs(60), + ) + .expect("valid memory stress") + .manifest(); + + assert!(cpu.contains("kind: StressChaos")); + assert!(cpu.contains("cpu:")); + assert!(cpu.contains("load: 80")); + assert!(memory.contains("memory:")); + assert!(memory.contains("size: \"512MiB\"")); + assert!(memory.contains("rustfs.tenant: fault-test-tenant")); + } + #[test] fn chaos_name_suffix_keeps_run_label_stable() { let config = FaultTestConfig::for_test("real-cluster", "fast-csi"); diff --git a/e2e/src/fault/backends/host.rs b/e2e/src/fault/backends/host.rs index 1cc25f2..63593d9 100644 --- a/e2e/src/fault/backends/host.rs +++ b/e2e/src/fault/backends/host.rs @@ -117,7 +117,7 @@ pub fn apply_dm_flakey( guard.load_table(spec.fault_table, false)?; let active = guard.snapshot("active")?; ensure!( - active.table.split_whitespace().nth(2) == spec.fault_table.split_whitespace().nth(2), + normalize_dm_table(&active.table) == normalize_dm_table(spec.fault_table), "device-mapper target did not switch to the requested fault table; requested {:?}, active {:?}", spec.fault_table, active.table @@ -181,7 +181,7 @@ impl DmFlakeyGuard { pub fn ensure_active(&self, stage: &str) -> Result { let snapshot = self.snapshot(stage)?; ensure!( - snapshot.table.split_whitespace().nth(2) == self.fault_table.split_whitespace().nth(2), + normalize_dm_table(&snapshot.table) == normalize_dm_table(&self.fault_table), "device-mapper target {:?} is no longer using the requested fault table at {stage}; expected {:?}, active {:?}", self.dm_name, self.fault_table, @@ -510,11 +510,15 @@ spec: ) } +fn normalize_dm_table(table: &str) -> String { + table.split_whitespace().collect::>().join(" ") +} + #[cfg(test)] mod tests { use super::{ DmFlakeySpec, dm_helper_manifest, dm_resume_args, dm_suspend_args, helper_pod_name, - pv_targets_node, validate_dm_spec, + normalize_dm_table, pv_targets_node, validate_dm_spec, }; use crate::fault::config::FaultTestConfig; @@ -555,6 +559,18 @@ mod tests { ); } + #[test] + fn dm_table_comparison_uses_the_full_normalized_table() { + assert_eq!( + normalize_dm_table("0 1024 flakey /dev/loop0 0 1 15\n"), + "0 1024 flakey /dev/loop0 0 1 15" + ); + assert_ne!( + normalize_dm_table("0 1024 flakey /dev/loop0 0 1 15"), + normalize_dm_table("0 1024 flakey /dev/loop1 0 1 15") + ); + } + #[test] fn dm_spec_rejects_unbounded_or_unsafe_targets() { let valid = DmFlakeySpec { diff --git a/e2e/src/fault/checker.rs b/e2e/src/fault/checker.rs index 111e7b9..e9307b4 100644 --- a/e2e/src/fault/checker.rs +++ b/e2e/src/fault/checker.rs @@ -27,9 +27,12 @@ pub struct CheckerReport { pub scenario: String, pub run_id: String, pub committed_puts: usize, + pub expected_live_objects: usize, + pub verified_live_objects: usize, pub missing_committed_objects: Vec, pub hash_mismatches: Vec, pub successful_corrupted_reads: Vec, + pub unexpected_visible_deleted_objects: Vec, pub unknown_writes_materialized: Vec, pub list_warnings: Vec, pub tenant_recovered: bool, @@ -56,40 +59,49 @@ pub async fn check_s3_history( concurrency: usize, ) -> Result { let initial_records = recorder.records(); - let committed = committed_puts(&initial_records); - let unknown_writes = unknown_puts(&initial_records); + let model = object_model(&initial_records); + let read_anomalies = successful_read_anomalies(&initial_records); + let list_warnings = list_history_warnings(&initial_records); let mut report = CheckerReport { scenario: recorder.scenario(), run_id: recorder.run_id(), - committed_puts: committed.len(), + committed_puts: model.committed_writes, + expected_live_objects: model.live.len(), + verified_live_objects: 0, missing_committed_objects: Vec::new(), hash_mismatches: Vec::new(), - successful_corrupted_reads: successful_corrupted_reads(&initial_records, &committed), + successful_corrupted_reads: read_anomalies.corrupted_reads, + unexpected_visible_deleted_objects: read_anomalies.visible_deleted_objects, unknown_writes_materialized: Vec::new(), - list_warnings: Vec::new(), + list_warnings, tenant_recovered, passed: false, }; let mut committed_results = - stream::iter(committed.clone().into_iter().map(|(key, expected_hash)| { + stream::iter(model.live.clone().into_iter().map(|(key, expected)| { let s3 = s3.clone(); let recorder = recorder.clone(); async move { let body = s3.get_object(&key, &recorder).await?; - Ok::<_, anyhow::Error>((key, expected_hash, body)) + Ok::<_, anyhow::Error>((key, expected, body)) } })) .buffer_unordered(concurrency); while let Some(result) = committed_results.next().await { - let (key, expected_hash, body) = result?; + let (key, expected, body) = result?; match body { Some(body) => { let actual_hash = sha256_hex(&body); - if actual_hash != expected_hash { + if actual_hash != expected.sha256 || body.len() != expected.size_bytes { report.hash_mismatches.push(format!( - "{key}: expected {expected_hash}, got {actual_hash}" + "{key}: expected {} ({} bytes), got {actual_hash} ({} bytes)", + expected.sha256, + expected.size_bytes, + body.len() )); + } else { + report.verified_live_objects += 1; } } None => report.missing_committed_objects.push(key), @@ -97,21 +109,22 @@ pub async fn check_s3_history( } let mut unknown_results = - stream::iter(unknown_writes.into_iter().map(|(key, attempted_hash)| { + stream::iter(model.unknown_writes.into_iter().map(|(key, attempted)| { let s3 = s3.clone(); let recorder = recorder.clone(); async move { let body = s3.get_object(&key, &recorder).await?; - Ok::<_, anyhow::Error>((key, attempted_hash, body)) + Ok::<_, anyhow::Error>((key, attempted, body)) } })) .buffer_unordered(concurrency); while let Some(result) = unknown_results.next().await { - let (key, attempted_hash, body) = result?; + let (key, attempted, body) = result?; if let Some(body) = body { let actual_hash = sha256_hex(&body); report.unknown_writes_materialized.push(format!( - "{key}: attempted {attempted_hash}, got {actual_hash}" + "{key}: attempted {}, got {actual_hash}", + attempted.sha256 )); } } @@ -121,13 +134,20 @@ pub async fn check_s3_history( match s3.list_prefix(&prefix, recorder).await? { Some(keys) => { let listed = keys.into_iter().collect::>(); - for key in committed.keys() { + for key in model.live.keys() { if !listed.contains(key) { report.list_warnings.push(format!( - "LIST prefix {prefix} did not include committed key {key}" + "LIST prefix {prefix} did not include expected live key {key}" )); } } + for key in model.deleted { + if listed.contains(&key) { + report + .list_warnings + .push(format!("LIST prefix {prefix} included deleted key {key}")); + } + } } None => report .list_warnings @@ -137,73 +157,189 @@ pub async fn check_s3_history( report.missing_committed_objects.sort(); report.hash_mismatches.sort(); report.unknown_writes_materialized.sort(); + report.unexpected_visible_deleted_objects.sort(); report.list_warnings.sort(); report.passed = report.tenant_recovered && report.missing_committed_objects.is_empty() && report.hash_mismatches.is_empty() && report.successful_corrupted_reads.is_empty() + && report.unexpected_visible_deleted_objects.is_empty() && report.list_warnings.is_empty(); Ok(report) } -fn committed_puts(records: &[OperationRecord]) -> BTreeMap { - records - .iter() - .filter(|record| { - record.kind == OperationKind::Put && record.outcome == OperationOutcome::Ok - }) - .filter_map(|record| Some((record.key.clone()?, record.value_sha256.clone()?))) - .collect() +#[derive(Debug, Clone, PartialEq, Eq)] +struct ExpectedObject { + sha256: String, + size_bytes: usize, } -fn unknown_puts(records: &[OperationRecord]) -> BTreeMap { - records - .iter() - .filter(|record| { - record.kind == OperationKind::Put - && matches!( - record.outcome, - OperationOutcome::Timeout | OperationOutcome::Unknown - ) - }) - .filter_map(|record| Some((record.key.clone()?, record.value_sha256.clone()?))) - .collect() +#[derive(Debug, Default)] +struct ObjectModel { + live: BTreeMap, + deleted: BTreeSet, + unknown_writes: BTreeMap, + committed_writes: usize, +} + +#[derive(Debug, Default)] +struct ReadAnomalies { + corrupted_reads: Vec, + visible_deleted_objects: Vec, +} + +fn object_model(records: &[OperationRecord]) -> ObjectModel { + let mut model = ObjectModel::default(); + for record in records { + apply_record_to_model(&mut model, record); + } + model +} + +fn object_model_before(records: &[OperationRecord], started_at_ms: u64) -> ObjectModel { + let mut model = ObjectModel::default(); + for record in records { + if record.ended_at_ms < started_at_ms { + apply_record_to_model(&mut model, record); + } + } + model +} + +fn apply_record_to_model(model: &mut ObjectModel, record: &OperationRecord) { + match record.kind { + OperationKind::Put | OperationKind::CompleteMultipartUpload + if record.outcome == OperationOutcome::Ok => + { + if let Some((key, object)) = record_object(record) { + model.committed_writes += 1; + model.deleted.remove(&key); + model.live.insert(key, object); + } + } + OperationKind::Put | OperationKind::CompleteMultipartUpload + if matches!( + record.outcome, + OperationOutcome::Timeout | OperationOutcome::Unknown + ) => + { + if let Some((key, object)) = record_object(record) { + model.unknown_writes.insert(key, object); + } + } + OperationKind::Delete if record.outcome == OperationOutcome::Ok => { + if let Some(key) = record.key.clone() { + model.live.remove(&key); + model.deleted.insert(key); + } + } + _ => {} + } +} + +fn list_history_warnings(records: &[OperationRecord]) -> Vec { + let mut warnings = Vec::new(); + for record in records.iter().filter(|record| { + record.kind == OperationKind::List && record.outcome == OperationOutcome::Ok + }) { + let Some(prefix) = record.key.as_deref() else { + continue; + }; + let Some(listed_keys) = record.listed_keys.as_ref() else { + warnings.push(format!("LIST {} did not record returned keys", record.id)); + continue; + }; + let listed = listed_keys + .iter() + .map(String::as_str) + .collect::>(); + let stable = object_model_before(records, record.started_at_ms); + for key in stable.live.keys().filter(|key| key.starts_with(prefix)) { + if !listed.contains(key.as_str()) { + warnings.push(format!( + "LIST {} prefix {prefix} did not include stable live key {key}", + record.id + )); + } + } + for key in stable.deleted.iter().filter(|key| key.starts_with(prefix)) { + if listed.contains(key.as_str()) { + warnings.push(format!( + "LIST {} prefix {prefix} included stable deleted key {key}", + record.id + )); + } + } + } + warnings +} + +fn successful_read_anomalies(records: &[OperationRecord]) -> ReadAnomalies { + let mut live = BTreeMap::::new(); + let mut anomalies = ReadAnomalies::default(); + for record in records { + match record.kind { + OperationKind::Put | OperationKind::CompleteMultipartUpload + if record.outcome == OperationOutcome::Ok => + { + if let Some((key, object)) = record_object(record) { + live.insert(key, object); + } + } + OperationKind::Delete if record.outcome == OperationOutcome::Ok => { + if let Some(key) = record.key.as_ref() { + live.remove(key); + } + } + OperationKind::Get if record.outcome == OperationOutcome::Ok => { + let Some(key) = record.key.as_ref() else { + continue; + }; + let actual_hash = record.value_sha256.as_deref().unwrap_or_default(); + match live.get(key) { + Some(expected) if expected.sha256 != actual_hash => { + anomalies.corrupted_reads.push(format!( + "{key}: expected {}, got {actual_hash}", + expected.sha256 + )); + } + None => anomalies + .visible_deleted_objects + .push(format!("{key}: successful GET had no committed live value")), + _ => {} + } + } + _ => {} + } + } + anomalies } -fn successful_corrupted_reads( - records: &[OperationRecord], - committed: &BTreeMap, -) -> Vec { - records - .iter() - .filter(|record| { - record.kind == OperationKind::Get && record.outcome == OperationOutcome::Ok - }) - .filter_map(|record| { - let key = record.key.as_ref()?; - let expected_hash = committed.get(key)?; - let actual_hash = record.value_sha256.as_ref()?; - (expected_hash != actual_hash) - .then(|| format!("{key}: expected {expected_hash}, got {actual_hash}")) - }) - .collect() +fn record_object(record: &OperationRecord) -> Option<(String, ExpectedObject)> { + Some(( + record.key.clone()?, + ExpectedObject { + sha256: record.value_sha256.clone()?, + size_bytes: record.size_bytes?, + }, + )) } #[cfg(test)] mod tests { - use super::{CheckerReport, successful_corrupted_reads}; + use super::{CheckerReport, list_history_warnings, object_model, successful_read_anomalies}; use crate::fault::history::{OperationKind, OperationOutcome, OperationRecord}; - use std::collections::BTreeMap; fn record( + id: &str, kind: OperationKind, key: &str, hash: &str, outcome: OperationOutcome, ) -> OperationRecord { OperationRecord { - id: "op-1".to_string(), + id: id.to_string(), scenario: "io-eio".to_string(), kind, bucket: "bucket".to_string(), @@ -215,17 +351,120 @@ mod tests { outcome, http_status: Some(200), error: None, + listed_keys: None, + } + } + + fn list_record( + id: &str, + prefix: &str, + started_at_ms: u64, + ended_at_ms: u64, + keys: &[&str], + ) -> OperationRecord { + OperationRecord { + id: id.to_string(), + scenario: "io-eio".to_string(), + kind: OperationKind::List, + bucket: "bucket".to_string(), + key: Some(prefix.to_string()), + value_sha256: None, + size_bytes: Some(keys.len()), + listed_keys: Some(keys.iter().map(|key| key.to_string()).collect()), + started_at_ms, + ended_at_ms, + outcome: OperationOutcome::Ok, + http_status: Some(200), + error: None, } } #[test] fn corrupted_successful_get_is_hard_failure_input() { - let records = vec![record(OperationKind::Get, "k", "bad", OperationOutcome::Ok)]; - let committed = BTreeMap::from([("k".to_string(), "good".to_string())]); + let records = vec![ + record( + "op-1", + OperationKind::Put, + "k", + "good", + OperationOutcome::Ok, + ), + record("op-2", OperationKind::Get, "k", "bad", OperationOutcome::Ok), + ]; + + let anomalies = successful_read_anomalies(&records); + + assert_eq!(anomalies.corrupted_reads, vec!["k: expected good, got bad"]); + } + + #[test] + fn object_model_tracks_overwrite_delete_and_multipart_complete() { + let records = vec![ + record("op-1", OperationKind::Put, "k1", "v1", OperationOutcome::Ok), + record("op-2", OperationKind::Put, "k1", "v2", OperationOutcome::Ok), + record("op-3", OperationKind::Put, "k2", "v1", OperationOutcome::Ok), + record( + "op-4", + OperationKind::Delete, + "k2", + "", + OperationOutcome::Ok, + ), + record( + "op-5", + OperationKind::CompleteMultipartUpload, + "k3", + "mp", + OperationOutcome::Ok, + ), + ]; - let corrupted = successful_corrupted_reads(&records, &committed); + let model = object_model(&records); - assert_eq!(corrupted, vec!["k: expected good, got bad"]); + assert_eq!(model.committed_writes, 4); + assert_eq!(model.live.get("k1").expect("k1").sha256, "v2"); + assert!(!model.live.contains_key("k2")); + assert_eq!(model.live.get("k3").expect("k3").sha256, "mp"); + assert!(model.deleted.contains("k2")); + } + + #[test] + fn list_history_checks_stable_keys_and_ignores_overlapping_changes() { + let records = vec![ + OperationRecord { + started_at_ms: 1, + ended_at_ms: 2, + ..record( + "op-1", + OperationKind::Put, + "fault-test/run-1/stable", + "v1", + OperationOutcome::Ok, + ) + }, + OperationRecord { + started_at_ms: 4, + ended_at_ms: 7, + ..record( + "op-2", + OperationKind::Put, + "fault-test/run-1/overlap", + "v2", + OperationOutcome::Ok, + ) + }, + list_record("op-3", "fault-test/run-1/", 5, 6, &[]), + ]; + + let warnings = list_history_warnings(&records); + + assert_eq!( + warnings, + vec![ + "LIST op-3 prefix fault-test/run-1/ did not include stable live key fault-test/run-1/stable" + ] + ); + assert!(!warnings.iter().any(|warning| warning.contains("overlap"))); } #[test] @@ -234,9 +473,12 @@ mod tests { scenario: "io-eio".to_string(), run_id: "run-1".to_string(), committed_puts: 1, + expected_live_objects: 1, + verified_live_objects: 1, missing_committed_objects: Vec::new(), hash_mismatches: Vec::new(), successful_corrupted_reads: Vec::new(), + unexpected_visible_deleted_objects: Vec::new(), unknown_writes_materialized: Vec::new(), list_warnings: Vec::new(), tenant_recovered: true, diff --git a/e2e/src/fault/config.rs b/e2e/src/fault/config.rs index 77d4aac..5fb2b32 100644 --- a/e2e/src/fault/config.rs +++ b/e2e/src/fault/config.rs @@ -30,6 +30,7 @@ pub const DEFAULT_REQUEST_TIMEOUT_SECONDS: u64 = 30; pub const DEFAULT_CLUSTER_TIMEOUT_SECONDS: u64 = 300; pub const DEFAULT_WARP_DURATION_SECONDS: u64 = 60; pub const DEFAULT_DM_HELPER_IMAGE: &str = "rancher/mirrored-library-busybox:1.37.0"; +pub const MIN_WORKLOAD_OBJECTS: usize = 12; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct FaultWorkloadProfile { @@ -49,8 +50,8 @@ impl FaultWorkloadProfile { pub fn validate(self) -> Result<()> { ensure!( - self.object_count >= 4, - "RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS must be at least 4" + self.object_count >= MIN_WORKLOAD_OBJECTS, + "RUSTFS_FAULT_TEST_WORKLOAD_OBJECTS must be at least {MIN_WORKLOAD_OBJECTS}" ); ensure!( (1..=self.object_count).contains(&self.concurrency), @@ -366,7 +367,7 @@ where #[cfg(test)] mod tests { - use super::{FaultTestConfig, validate_storage_class}; + use super::{FaultTestConfig, FaultWorkloadProfile, validate_storage_class}; #[test] fn real_cluster_fault_defaults_are_isolated() { @@ -475,6 +476,12 @@ mod tests { assert_eq!(config.dm_helper_image, "busybox:test"); } + #[test] + fn workload_object_count_must_cover_all_mixed_operations() { + assert!(FaultWorkloadProfile::new(11, 1).is_err()); + assert!(FaultWorkloadProfile::new(12, 12).is_ok()); + } + #[test] fn kind_context_is_rejected_for_fault_tests() { let result = FaultTestConfig::from_env_with( diff --git a/e2e/src/fault/history.rs b/e2e/src/fault/history.rs index bb6da73..60788f3 100644 --- a/e2e/src/fault/history.rs +++ b/e2e/src/fault/history.rs @@ -29,12 +29,17 @@ pub enum OperationKind { Head, List, Delete, + CreateMultipartUpload, + UploadPart, + CompleteMultipartUpload, + AbortMultipartUpload, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum OperationOutcome { Ok, + NotFound, Failed, Timeout, Unknown, @@ -49,6 +54,8 @@ pub struct OperationRecord { pub key: Option, pub value_sha256: Option, pub size_bytes: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub listed_keys: Option>, pub started_at_ms: u64, pub ended_at_ms: u64, pub outcome: OperationOutcome, @@ -115,6 +122,7 @@ impl Recorder { key, value_sha256, size_bytes, + listed_keys: None, started_at_ms, ended_at_ms: started_at_ms, outcome: OperationOutcome::Unknown, diff --git a/e2e/src/fault/plan.rs b/e2e/src/fault/plan.rs index edf3355..b81add5 100644 --- a/e2e/src/fault/plan.rs +++ b/e2e/src/fault/plan.rs @@ -17,8 +17,10 @@ use std::time::Duration; use crate::fault::scenarios::{ DISK_FULL_SCENARIO, DM_FLAKEY_SCENARIO, FaultBackend, FaultScenario, FaultScenarioSpec, - IO_EIO_SCENARIO, IO_READ_MISTAKE_SCENARIO, NETWORK_PARTITION_ONE_SCENARIO, - POD_KILL_ONE_SCENARIO, WARP_UNDER_CHAOS_SCENARIO, + IO_EIO_SCENARIO, IO_LATENCY_SCENARIO, IO_READ_MISTAKE_SCENARIO, NETWORK_CORRUPT_SCENARIO, + NETWORK_DELAY_SCENARIO, NETWORK_DUPLICATE_SCENARIO, NETWORK_LOSS_SCENARIO, + NETWORK_PARTITION_ONE_SCENARIO, POD_FAILURE_SCENARIO, POD_KILL_ONE_SCENARIO, + STRESS_CPU_SCENARIO, STRESS_MEMORY_SCENARIO, WARP_UNDER_CHAOS_SCENARIO, }; pub const DEFAULT_RUSTFS_DATA_VOLUME: &str = "/data/rustfs0"; @@ -38,10 +40,18 @@ impl FaultWorkloadMode { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FaultKind { RustfsVolumeIoError, + RustfsVolumeLatency, RustfsVolumeReadMistake, RustfsVolumeEnospc, RustfsServerPodKill, + RustfsServerPodFailure, RustfsServerNetworkPartition, + RustfsServerNetworkDelay, + RustfsServerNetworkLoss, + RustfsServerNetworkCorrupt, + RustfsServerNetworkDuplicate, + RustfsServerCpuStress, + RustfsServerMemoryStress, RustfsBlockDeviceFlakey, } @@ -49,10 +59,18 @@ impl FaultKind { pub fn as_str(self) -> &'static str { match self { Self::RustfsVolumeIoError => "rustfs_volume_io_error", + Self::RustfsVolumeLatency => "rustfs_volume_latency", Self::RustfsVolumeReadMistake => "rustfs_volume_read_mistake", Self::RustfsVolumeEnospc => "rustfs_volume_enospc", Self::RustfsServerPodKill => "rustfs_server_pod_kill", + Self::RustfsServerPodFailure => "rustfs_server_pod_failure", Self::RustfsServerNetworkPartition => "rustfs_server_network_partition", + Self::RustfsServerNetworkDelay => "rustfs_server_network_delay", + Self::RustfsServerNetworkLoss => "rustfs_server_network_loss", + Self::RustfsServerNetworkCorrupt => "rustfs_server_network_corrupt", + Self::RustfsServerNetworkDuplicate => "rustfs_server_network_duplicate", + Self::RustfsServerCpuStress => "rustfs_server_cpu_stress", + Self::RustfsServerMemoryStress => "rustfs_server_memory_stress", Self::RustfsBlockDeviceFlakey => "rustfs_block_device_flakey", } } @@ -63,6 +81,7 @@ pub enum FaultTarget { RustfsVolume { path: &'static str }, RustfsServerPod, RustfsServerPeerNetwork, + RustfsServerResource, DedicatedBlockDevice, } @@ -74,6 +93,9 @@ impl FaultTarget { Self::RustfsServerPeerNetwork => { "one RustFS server Pod partitioned from its peers".to_string() } + Self::RustfsServerResource => { + "one RustFS server Pod under resource pressure".to_string() + } Self::DedicatedBlockDevice => "one dedicated block-device-backed PV".to_string(), } } @@ -184,14 +206,23 @@ fn fault_kind_accepts_backend(kind: FaultKind, backend: FaultBackend) -> bool { FaultKind::RustfsVolumeIoError, FaultBackend::ChaosMeshIoChaos | FaultBackend::MinioWarpWithChaos ) | ( - FaultKind::RustfsVolumeReadMistake | FaultKind::RustfsVolumeEnospc, + FaultKind::RustfsVolumeLatency + | FaultKind::RustfsVolumeReadMistake + | FaultKind::RustfsVolumeEnospc, FaultBackend::ChaosMeshIoChaos ) | ( - FaultKind::RustfsServerPodKill, + FaultKind::RustfsServerPodKill | FaultKind::RustfsServerPodFailure, FaultBackend::ChaosMeshPodChaos ) | ( - FaultKind::RustfsServerNetworkPartition, + FaultKind::RustfsServerNetworkPartition + | FaultKind::RustfsServerNetworkDelay + | FaultKind::RustfsServerNetworkLoss + | FaultKind::RustfsServerNetworkCorrupt + | FaultKind::RustfsServerNetworkDuplicate, FaultBackend::ChaosMeshNetworkChaos + ) | ( + FaultKind::RustfsServerCpuStress | FaultKind::RustfsServerMemoryStress, + FaultBackend::ChaosMeshStressChaos ) | ( FaultKind::RustfsBlockDeviceFlakey, FaultBackend::DeviceMapper @@ -202,13 +233,21 @@ fn fault_kind_accepts_backend(kind: FaultKind, backend: FaultBackend) -> bool { fn fault_kind_accepts_selection(kind: FaultKind, selection: FaultSelection) -> bool { match kind { FaultKind::RustfsVolumeIoError + | FaultKind::RustfsVolumeLatency | FaultKind::RustfsVolumeReadMistake | FaultKind::RustfsVolumeEnospc => match selection { FaultSelection::Percent(percent) => (1..=100).contains(&percent), FaultSelection::FixedTargets(_) => false, }, FaultKind::RustfsServerPodKill + | FaultKind::RustfsServerPodFailure | FaultKind::RustfsServerNetworkPartition + | FaultKind::RustfsServerNetworkDelay + | FaultKind::RustfsServerNetworkLoss + | FaultKind::RustfsServerNetworkCorrupt + | FaultKind::RustfsServerNetworkDuplicate + | FaultKind::RustfsServerCpuStress + | FaultKind::RustfsServerMemoryStress | FaultKind::RustfsBlockDeviceFlakey => match selection { FaultSelection::FixedTargets(count) => count > 0, FaultSelection::Percent(_) => false, @@ -266,6 +305,13 @@ impl FaultPlan { FaultSelection::FixedTargets(1), scenario.duration, )?, + POD_FAILURE_SCENARIO => FaultInjection::new( + FaultKind::RustfsServerPodFailure, + spec.backend, + FaultTarget::RustfsServerPod, + FaultSelection::FixedTargets(1), + scenario.duration, + )?, NETWORK_PARTITION_ONE_SCENARIO => FaultInjection::new( FaultKind::RustfsServerNetworkPartition, spec.backend, @@ -273,10 +319,29 @@ impl FaultPlan { FaultSelection::FixedTargets(1), scenario.duration, )?, + NETWORK_DELAY_SCENARIO => { + network_fault(FaultKind::RustfsServerNetworkDelay, spec, scenario)? + } + NETWORK_LOSS_SCENARIO => { + network_fault(FaultKind::RustfsServerNetworkLoss, spec, scenario)? + } + NETWORK_CORRUPT_SCENARIO => { + network_fault(FaultKind::RustfsServerNetworkCorrupt, spec, scenario)? + } + NETWORK_DUPLICATE_SCENARIO => { + network_fault(FaultKind::RustfsServerNetworkDuplicate, spec, scenario)? + } IO_READ_MISTAKE_SCENARIO => { volume_fault(FaultKind::RustfsVolumeReadMistake, spec, scenario)? } + IO_LATENCY_SCENARIO => volume_fault(FaultKind::RustfsVolumeLatency, spec, scenario)?, DISK_FULL_SCENARIO => volume_fault(FaultKind::RustfsVolumeEnospc, spec, scenario)?, + STRESS_CPU_SCENARIO => { + resource_fault(FaultKind::RustfsServerCpuStress, spec, scenario)? + } + STRESS_MEMORY_SCENARIO => { + resource_fault(FaultKind::RustfsServerMemoryStress, spec, scenario)? + } DM_FLAKEY_SCENARIO => FaultInjection::new( FaultKind::RustfsBlockDeviceFlakey, spec.backend, @@ -358,6 +423,34 @@ fn volume_fault( ) } +fn network_fault( + kind: FaultKind, + spec: &FaultScenarioSpec, + scenario: &FaultScenario, +) -> Result { + FaultInjection::new( + kind, + spec.backend, + FaultTarget::RustfsServerPeerNetwork, + FaultSelection::FixedTargets(1), + scenario.duration, + ) +} + +fn resource_fault( + kind: FaultKind, + spec: &FaultScenarioSpec, + scenario: &FaultScenario, +) -> Result { + FaultInjection::new( + kind, + spec.backend, + FaultTarget::RustfsServerResource, + FaultSelection::FixedTargets(1), + scenario.duration, + ) +} + #[cfg(test)] mod tests { use super::{ diff --git a/e2e/src/fault/runner.rs b/e2e/src/fault/runner.rs index 04511c1..97aa511 100644 --- a/e2e/src/fault/runner.rs +++ b/e2e/src/fault/runner.rs @@ -15,7 +15,9 @@ use crate::{ fault::{ backends::{ - chaos_mesh::{self, ChaosGuard, IoChaosSpec, NetworkChaosSpec, PodChaosSpec}, + chaos_mesh::{ + self, ChaosGuard, IoChaosSpec, NetworkChaosSpec, PodChaosSpec, StressChaosSpec, + }, host::{self, DmFlakeyGuard, DmFlakeySpec, DmStatusSnapshot}, }, checker, @@ -448,9 +450,11 @@ async fn run_fault_case( "workload-summary.json", &serde_json::to_string_pretty(&workload.summary)?, )?; + let require_client_disruption = + config.require_client_disruption || spec.impact_policy.requires_client_disruption(); if let Err(error) = workload .summary - .require_fault_evidence(config.require_client_disruption) + .require_fault_evidence(require_client_disruption) { collect_fault_artifacts( collector, @@ -566,6 +570,26 @@ async fn run_fault_case( "fault-evidence.json", &serde_json::to_string_pretty(&recovered_evidence)?, )?; + let pre_recommit_report = + checker::check_s3_history(&s3, &history, true, workload_plan.concurrency).await?; + collector.write_text( + scenario.case_name, + "checker-pre-recommit-report.json", + &serde_json::to_string_pretty(&pre_recommit_report)?, + )?; + if let Err(error) = pre_recommit_report.require_success() { + write_failure_summary( + collector, + scenario.case_name, + FailureSummary::new( + &scenario.name, + "checker-pre-recommit-verdict", + "product_or_environment", + error.to_string(), + ), + )?; + return Err(error); + } let recommit_report = recommit_unconfirmed_objects( &s3, &history, @@ -624,23 +648,6 @@ async fn run_fault_case( "fault-evidence.json", &serde_json::to_string_pretty(&evidence)?, )?; - if report.committed_puts != scenario.object_count { - let message = format!( - "fault scenario {} expected {} committed objects after recovery reconciliation, got {}", - scenario.name, scenario.object_count, report.committed_puts - ); - write_failure_summary( - collector, - scenario.case_name, - FailureSummary::new( - &scenario.name, - "checker-committed-count", - "product_or_environment", - message.clone(), - ), - )?; - bail!("{message}"); - } if let Err(error) = report.require_success() { write_failure_summary( collector, @@ -675,6 +682,7 @@ fn require_fault_backend(config: &FaultTestConfig, backend: FaultBackend) -> Res } FaultBackend::ChaosMeshPodChaos => chaos_mesh::require_podchaos_crd(cluster), FaultBackend::ChaosMeshNetworkChaos => chaos_mesh::require_networkchaos_crd(cluster), + FaultBackend::ChaosMeshStressChaos => chaos_mesh::require_stresschaos_crd(cluster), FaultBackend::DeviceMapper => require_dm_flakey_preflight(config), } } @@ -729,6 +737,9 @@ fn cleanup_fault_backend(config: &FaultTestConfig, backend: FaultBackend) -> Res FaultBackend::ChaosMeshNetworkChaos => { chaos_mesh::cleanup_managed_networkchaos(&config.cluster, &config.chaos_namespace) } + FaultBackend::ChaosMeshStressChaos => { + chaos_mesh::cleanup_managed_stresschaos(&config.cluster, &config.chaos_namespace) + } FaultBackend::DeviceMapper => Ok(()), } } @@ -895,6 +906,23 @@ impl AppliedFault { active_required: true, }) } + FaultKind::RustfsVolumeLatency => { + let chaos = IoChaosSpec::latency_on_rustfs_volume( + cluster, + &config.chaos_namespace, + run_id, + &scenario.name, + injection.rustfs_volume_path()?, + injection.percent()?, + injection.duration(), + )? + .with_name_suffix(resource_name_suffix); + collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; + Ok(Self::Chaos { + guard: Box::new(chaos_mesh::apply_iochaos(cluster, &chaos)?), + active_required: true, + }) + } FaultKind::RustfsVolumeIoError => { let chaos = IoChaosSpec::eio_on_rustfs_volume( cluster, @@ -928,6 +956,21 @@ impl AppliedFault { config: Box::new(cluster.clone()), }) } + FaultKind::RustfsServerPodFailure => { + let chaos = PodChaosSpec::fail_one_rustfs_pod( + cluster, + &config.chaos_namespace, + run_id, + &scenario.name, + injection.duration(), + )? + .with_name_suffix(resource_name_suffix); + collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; + Ok(Self::Chaos { + guard: Box::new(chaos_mesh::apply_podchaos(cluster, &chaos)?), + active_required: true, + }) + } FaultKind::RustfsServerNetworkPartition => { let chaos = NetworkChaosSpec::partition_one_rustfs_pod( cluster, @@ -943,6 +986,79 @@ impl AppliedFault { active_required: true, }) } + FaultKind::RustfsServerNetworkDelay + | FaultKind::RustfsServerNetworkLoss + | FaultKind::RustfsServerNetworkCorrupt + | FaultKind::RustfsServerNetworkDuplicate => { + let chaos = match injection.kind() { + FaultKind::RustfsServerNetworkDelay => NetworkChaosSpec::delay_one_rustfs_pod( + cluster, + &config.chaos_namespace, + run_id, + &scenario.name, + injection.duration(), + )?, + FaultKind::RustfsServerNetworkLoss => NetworkChaosSpec::loss_one_rustfs_pod( + cluster, + &config.chaos_namespace, + run_id, + &scenario.name, + injection.duration(), + )?, + FaultKind::RustfsServerNetworkCorrupt => { + NetworkChaosSpec::corrupt_one_rustfs_pod( + cluster, + &config.chaos_namespace, + run_id, + &scenario.name, + injection.duration(), + )? + } + FaultKind::RustfsServerNetworkDuplicate => { + NetworkChaosSpec::duplicate_one_rustfs_pod( + cluster, + &config.chaos_namespace, + run_id, + &scenario.name, + injection.duration(), + )? + } + _ => unreachable!(), + } + .with_name_suffix(resource_name_suffix); + collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; + Ok(Self::Chaos { + guard: Box::new(chaos_mesh::apply_networkchaos(cluster, &chaos)?), + active_required: true, + }) + } + FaultKind::RustfsServerCpuStress | FaultKind::RustfsServerMemoryStress => { + let chaos = match injection.kind() { + FaultKind::RustfsServerCpuStress => StressChaosSpec::cpu_on_one_rustfs_pod( + cluster, + &config.chaos_namespace, + run_id, + &scenario.name, + injection.duration(), + )?, + FaultKind::RustfsServerMemoryStress => { + StressChaosSpec::memory_on_one_rustfs_pod( + cluster, + &config.chaos_namespace, + run_id, + &scenario.name, + injection.duration(), + )? + } + _ => unreachable!(), + } + .with_name_suffix(resource_name_suffix); + collector.write_text(scenario.case_name, manifest_name, &chaos.manifest())?; + Ok(Self::Chaos { + guard: Box::new(chaos_mesh::apply_stresschaos(cluster, &chaos)?), + active_required: true, + }) + } FaultKind::RustfsBlockDeviceFlakey => { let name = config .dm_name @@ -1629,17 +1745,18 @@ async fn prefill_objects( async move { let object = ObjectSpec::prepare_seeded(&run_id, index, size_bytes, seed); let spec = object.spec.clone(); - let put_outcome = s3.put_object(&object, &history).await?; + let verified = s3.put_and_verify_object(&object, &history).await?; ensure!( - put_outcome == OperationOutcome::Ok, - "prefill PUT failed before fault injection for key {}: {put_outcome:?}", - spec.key + verified.write_outcome == OperationOutcome::Ok, + "prefill PUT failed before fault injection for key {}: {:?}", + spec.key, + verified.write_outcome ); - let head_outcome = s3.head_object(&spec.key, &history).await?; ensure!( - head_outcome == OperationOutcome::Ok, - "prefill HEAD failed before fault injection for key {}: {head_outcome:?}", - spec.key + verified.verified, + "prefill GET verification failed before fault injection for key {}: {:?}", + spec.key, + verified.verify_get_outcome ); Ok::<_, anyhow::Error>((index, spec)) } @@ -1671,16 +1788,78 @@ async fn run_mixed_workload( let seed = plan.seed; let existing = prefilled[offset % prefilled.len()].clone(); async move { - let object = ObjectSpec::prepare_seeded(&run_id, index, size_bytes, seed); - let spec = object.spec.clone(); - let put_outcome = s3.put_object(&object, &history).await?; - let get_outcome = s3.get_object_result(&existing.key, &history).await?.outcome; - Ok::<_, anyhow::Error>(MixedTaskResult { - index, - object: spec, - put_outcome, - get_outcome, - }) + let mut result = MixedTaskResult::new(index); + match offset % 6 { + 0 => { + let object = ObjectSpec::prepare_seeded(&run_id, index, size_bytes, seed); + let spec = object.spec.clone(); + let verified = s3.put_and_verify_object(&object, &history).await?; + result.puts.push(verified.write_outcome); + if let Some(get_outcome) = verified.verify_get_outcome { + result.gets.push(get_outcome); + } + if verified.write_outcome != OperationOutcome::Ok { + result.unconfirmed_puts.push(spec); + } + } + 1 => { + let object = existing.prepare_overwrite(index as u64 + 1); + let spec = object.spec.clone(); + let verified = s3.put_and_verify_object(&object, &history).await?; + result.puts.push(verified.write_outcome); + if let Some(get_outcome) = verified.verify_get_outcome { + result.gets.push(get_outcome); + } + if verified.write_outcome != OperationOutcome::Ok { + result.unconfirmed_puts.push(spec); + } + } + 2 => { + result + .gets + .push(s3.get_object_result(&existing.key, &history).await?.outcome); + } + 3 => { + let prefix = ObjectSpec::key_prefix(&run_id); + let outcome = if s3.list_prefix(&prefix, &history).await?.is_some() { + OperationOutcome::Ok + } else { + OperationOutcome::Unknown + }; + result.lists.push(outcome); + } + 4 => { + let (delete_outcome, verify_get) = + s3.delete_and_verify_absent(&existing.key, &history).await?; + result.deletes.push(delete_outcome); + if let Some(get_outcome) = verify_get { + result.gets.push(get_outcome); + } + } + _ => { + let object = ObjectSpec::prepare_seeded(&run_id, index, size_bytes, seed); + let spec = object.spec.clone(); + let complete_outcome = s3.complete_multipart_object(&object, &history).await?; + result.multipart_completes.push(complete_outcome); + if complete_outcome == OperationOutcome::Ok { + result + .gets + .push(s3.get_object_result(&spec.key, &history).await?.outcome); + } else { + result.unconfirmed_puts.push(spec); + } + let abort_object = ObjectSpec::prepare_seeded( + &run_id, + plan.object_count + index, + 4 * 1024, + seed, + ); + result + .multipart_aborts + .push(s3.abort_multipart_object(&abort_object, &history).await?); + } + } + Ok::<_, anyhow::Error>(result) } }); let results = stream::iter(tasks) @@ -1696,11 +1875,8 @@ async fn run_mixed_workload( let mut summary = WorkloadSummary::new(plan); let mut unconfirmed_puts = Vec::new(); for result in completed { - summary.puts.record(result.put_outcome); - summary.gets.record(result.get_outcome); - if result.put_outcome != OperationOutcome::Ok { - unconfirmed_puts.push(result.object); - } + summary.record_all(&result); + unconfirmed_puts.extend(result.unconfirmed_puts); } summary.require_exercised()?; @@ -1722,7 +1898,17 @@ async fn recommit_unconfirmed_objects( async move { let prepared = object.prepare(); match s3.put_object_record(&prepared, &history).await { - Ok(record) => RecommitAttempt::from_record(object, record), + Ok(record) => { + let verify_get_outcome = if record.outcome == OperationOutcome::Ok { + match s3.get_object_result(&object.key, &history).await { + Ok(get) => Some(get.outcome), + Err(_) => Some(OperationOutcome::Unknown), + } + } else { + None + }; + RecommitAttempt::from_record(object, record, verify_get_outcome) + } Err(error) => { RecommitAttempt::from_harness_error(object, format!("record PUT: {error}")) } @@ -1754,7 +1940,7 @@ impl RecommitReport { .count(); let failed = attempts .iter() - .filter(|attempt| attempt.is_s3_failure()) + .filter(|attempt| attempt.is_s3_failure() || attempt.verify_get_failed()) .count(); let harness_errors = attempts .iter() @@ -1809,18 +1995,24 @@ struct RecommitAttempt { size_bytes: usize, sha256: String, outcome: Option, + verify_get_outcome: Option, http_status: Option, error: Option, harness_error: Option, } impl RecommitAttempt { - fn from_record(object: ObjectSpec, record: OperationRecord) -> Self { + fn from_record( + object: ObjectSpec, + record: OperationRecord, + verify_get_outcome: Option, + ) -> Self { Self { key: object.key, size_bytes: object.size_bytes, sha256: object.sha256, outcome: Some(record.outcome), + verify_get_outcome, http_status: record.http_status, error: record.error, harness_error: None, @@ -1833,6 +2025,7 @@ impl RecommitAttempt { size_bytes: object.size_bytes, sha256: object.sha256, outcome: None, + verify_get_outcome: None, http_status: None, error: None, harness_error: Some(error), @@ -1842,7 +2035,12 @@ impl RecommitAttempt { fn is_s3_failure(&self) -> bool { matches!( self.outcome, - Some(OperationOutcome::Failed | OperationOutcome::Timeout | OperationOutcome::Unknown) + Some( + OperationOutcome::NotFound + | OperationOutcome::Failed + | OperationOutcome::Timeout + | OperationOutcome::Unknown + ) ) } @@ -1850,12 +2048,23 @@ impl RecommitAttempt { self.harness_error.is_some() } + fn verify_get_failed(&self) -> bool { + self.outcome == Some(OperationOutcome::Ok) + && self.verify_get_outcome != Some(OperationOutcome::Ok) + } + fn failure_sample(&self) -> Option { if let Some(error) = &self.harness_error { return Some(format!("{}=harness_error({error})", self.key)); } let outcome = self.outcome?; if outcome == OperationOutcome::Ok { + if self.verify_get_failed() { + return Some(format!( + "{}=verify_get({:?})", + self.key, self.verify_get_outcome + )); + } return None; } let status = self @@ -1874,9 +2083,28 @@ impl RecommitAttempt { #[derive(Debug)] struct MixedTaskResult { index: usize, - object: ObjectSpec, - put_outcome: OperationOutcome, - get_outcome: OperationOutcome, + puts: Vec, + gets: Vec, + deletes: Vec, + lists: Vec, + multipart_completes: Vec, + multipart_aborts: Vec, + unconfirmed_puts: Vec, +} + +impl MixedTaskResult { + fn new(index: usize) -> Self { + Self { + index, + puts: Vec::new(), + gets: Vec::new(), + deletes: Vec::new(), + lists: Vec::new(), + multipart_completes: Vec::new(), + multipart_aborts: Vec::new(), + unconfirmed_puts: Vec::new(), + } + } } #[derive(Debug)] @@ -1893,6 +2121,10 @@ struct WorkloadSummary { total_payload_bytes: u64, puts: OutcomeCounts, gets: OutcomeCounts, + deletes: OutcomeCounts, + lists: OutcomeCounts, + multipart_completes: OutcomeCounts, + multipart_aborts: OutcomeCounts, recommitted_after_recovery: usize, } @@ -1905,14 +2137,44 @@ impl WorkloadSummary { total_payload_bytes: plan.total_payload_bytes, puts: OutcomeCounts::default(), gets: OutcomeCounts::default(), + deletes: OutcomeCounts::default(), + lists: OutcomeCounts::default(), + multipart_completes: OutcomeCounts::default(), + multipart_aborts: OutcomeCounts::default(), recommitted_after_recovery: 0, } } + fn record_all(&mut self, result: &MixedTaskResult) { + for outcome in &result.puts { + self.puts.record(*outcome); + } + for outcome in &result.gets { + self.gets.record(*outcome); + } + for outcome in &result.deletes { + self.deletes.record(*outcome); + } + for outcome in &result.lists { + self.lists.record(*outcome); + } + for outcome in &result.multipart_completes { + self.multipart_completes.record(*outcome); + } + for outcome in &result.multipart_aborts { + self.multipart_aborts.record(*outcome); + } + } + fn require_exercised(&self) -> Result<()> { ensure!( - self.puts.total() > 0 && self.gets.total() > 0, - "fault workload did not exercise both PUT and GET paths: {self:?}" + self.puts.total() > 0 + && self.gets.total() > 0 + && self.deletes.total() > 0 + && self.lists.total() > 0 + && self.multipart_completes.total() > 0 + && self.multipart_aborts.total() > 0, + "fault workload did not exercise every required S3 object path: {self:?}" ); Ok(()) } @@ -1932,13 +2194,19 @@ impl WorkloadSummary { } fn disrupted(&self) -> usize { - self.puts.disrupted() + self.gets.disrupted() + self.puts.disrupted() + + self.gets.disrupted() + + self.deletes.disrupted() + + self.lists.disrupted() + + self.multipart_completes.disrupted() + + self.multipart_aborts.disrupted() } } #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)] struct OutcomeCounts { ok: usize, + not_found: usize, failed: usize, timeout: usize, unknown: usize, @@ -1948,6 +2216,7 @@ impl OutcomeCounts { fn record(&mut self, outcome: OperationOutcome) { match outcome { OperationOutcome::Ok => self.ok += 1, + OperationOutcome::NotFound => self.not_found += 1, OperationOutcome::Failed => self.failed += 1, OperationOutcome::Timeout => self.timeout += 1, OperationOutcome::Unknown => self.unknown += 1, @@ -1955,7 +2224,7 @@ impl OutcomeCounts { } fn total(&self) -> usize { - self.ok + self.failed + self.timeout + self.unknown + self.ok + self.not_found + self.failed + self.timeout + self.unknown } fn disrupted(&self) -> usize { @@ -2040,14 +2309,34 @@ mod tests { let mut summary = WorkloadSummary::new(&WorkloadPlan::seeded(42, 40000, 80)); summary.puts.record(OperationOutcome::Ok); summary.gets.record(OperationOutcome::Timeout); + summary.gets.record(OperationOutcome::NotFound); + summary.deletes.record(OperationOutcome::Ok); + summary.lists.record(OperationOutcome::Ok); + summary.multipart_completes.record(OperationOutcome::Ok); + summary.multipart_aborts.record(OperationOutcome::Ok); assert_eq!(summary.puts.total(), 1); - assert_eq!(summary.gets.total(), 1); + assert_eq!(summary.gets.total(), 2); assert_eq!(summary.disrupted(), 1); assert!(summary.require_exercised().is_ok()); assert!(summary.require_fault_evidence(true).is_ok()); } + #[test] + fn workload_summary_requires_every_object_operation_family() { + let mut summary = WorkloadSummary::new(&WorkloadPlan::seeded(42, 40000, 80)); + summary.puts.record(OperationOutcome::Ok); + summary.gets.record(OperationOutcome::Ok); + summary.deletes.record(OperationOutcome::Ok); + summary.lists.record(OperationOutcome::Ok); + summary.multipart_completes.record(OperationOutcome::Ok); + + assert!(summary.require_exercised().is_err()); + + summary.multipart_aborts.record(OperationOutcome::Ok); + assert!(summary.require_exercised().is_ok()); + } + #[test] fn workload_summary_can_require_fault_evidence() { let summary = WorkloadSummary { @@ -2063,6 +2352,10 @@ mod tests { ok: 1, ..OutcomeCounts::default() }, + deletes: OutcomeCounts::default(), + lists: OutcomeCounts::default(), + multipart_completes: OutcomeCounts::default(), + multipart_aborts: OutcomeCounts::default(), recommitted_after_recovery: 0, }; @@ -2078,6 +2371,7 @@ mod tests { size_bytes: 4096, sha256: "sha-a".to_string(), outcome: Some(OperationOutcome::Ok), + verify_get_outcome: Some(OperationOutcome::Ok), http_status: Some(200), error: None, harness_error: None, @@ -2087,6 +2381,7 @@ mod tests { size_bytes: 4096, sha256: "sha-b".to_string(), outcome: Some(OperationOutcome::Failed), + verify_get_outcome: None, http_status: Some(503), error: Some("service unavailable".to_string()), harness_error: None, @@ -2113,6 +2408,7 @@ mod tests { size_bytes: 4096, sha256: "sha-a".to_string(), outcome: None, + verify_get_outcome: None, http_status: None, error: None, harness_error: Some("record PUT: disk full".to_string()), diff --git a/e2e/src/fault/scenarios.rs b/e2e/src/fault/scenarios.rs index a920d36..747c545 100644 --- a/e2e/src/fault/scenarios.rs +++ b/e2e/src/fault/scenarios.rs @@ -13,6 +13,7 @@ // limitations under the License. use anyhow::{Result, ensure}; +use serde::Serialize; use std::time::Duration; use crate::fault::config::FaultTestConfig; @@ -20,17 +21,32 @@ use crate::fault::config::FaultTestConfig; pub const IO_EIO_SCENARIO: &str = "io-eio"; pub const POD_KILL_ONE_SCENARIO: &str = "pod-kill-one"; pub const NETWORK_PARTITION_ONE_SCENARIO: &str = "network-partition-one"; +pub const NETWORK_DELAY_SCENARIO: &str = "network-delay"; +pub const NETWORK_LOSS_SCENARIO: &str = "network-loss"; +pub const NETWORK_CORRUPT_SCENARIO: &str = "network-corrupt"; +pub const NETWORK_DUPLICATE_SCENARIO: &str = "network-duplicate"; pub const IO_READ_MISTAKE_SCENARIO: &str = "io-read-mistake"; +pub const IO_LATENCY_SCENARIO: &str = "io-latency"; pub const DISK_FULL_SCENARIO: &str = "disk-full"; +pub const POD_FAILURE_SCENARIO: &str = "pod-failure"; +pub const STRESS_CPU_SCENARIO: &str = "stress-cpu"; +pub const STRESS_MEMORY_SCENARIO: &str = "stress-memory"; pub const DM_FLAKEY_SCENARIO: &str = "dm-flakey"; pub const WARP_UNDER_CHAOS_SCENARIO: &str = "warp-under-chaos"; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +const IOCHAOS_CRD: &str = "iochaos.chaos-mesh.org"; +const PODCHAOS_CRD: &str = "podchaos.chaos-mesh.org"; +const NETWORKCHAOS_CRD: &str = "networkchaos.chaos-mesh.org"; +const STRESSCHAOS_CRD: &str = "stresschaos.chaos-mesh.org"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case")] pub enum FaultScenarioStatus { Executable, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case")] pub enum FaultPriority { P0, P1, @@ -38,11 +54,13 @@ pub enum FaultPriority { P3, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case")] pub enum FaultBackend { ChaosMeshIoChaos, ChaosMeshPodChaos, ChaosMeshNetworkChaos, + ChaosMeshStressChaos, DeviceMapper, MinioWarpWithChaos, } @@ -53,14 +71,28 @@ impl FaultBackend { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case")] pub enum FaultIsolation { FreshTenant, ReusableTenant, DedicatedLinuxBlockDevice, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum FaultImpactPolicy { + ClientDisruptionRequired, + ClientDisruptionOptional, +} + +impl FaultImpactPolicy { + pub fn requires_client_disruption(self) -> bool { + matches!(self, Self::ClientDisruptionRequired) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] pub struct FaultScenarioSpec { pub scenario: &'static str, pub case_name: &'static str, @@ -69,6 +101,10 @@ pub struct FaultScenarioSpec { pub backend: FaultBackend, pub status: FaultScenarioStatus, pub isolation: FaultIsolation, + pub crds: &'static [&'static str], + pub required_tools: &'static [&'static str], + pub percent_supported: bool, + pub impact_policy: FaultImpactPolicy, pub boundary: &'static str, pub ci_phase: &'static str, pub target: &'static str, @@ -77,6 +113,16 @@ pub struct FaultScenarioSpec { pub conflict_domain: &'static str, } +impl FaultScenarioSpec { + pub fn requires_static_storage(self) -> bool { + self.isolation == FaultIsolation::DedicatedLinuxBlockDevice + } + + pub fn requires_chaos_mesh(self) -> bool { + !self.crds.is_empty() + } +} + pub const FAULT_SCENARIO_CATALOG: &[FaultScenarioSpec] = &[ FaultScenarioSpec { scenario: IO_EIO_SCENARIO, @@ -86,6 +132,10 @@ pub const FAULT_SCENARIO_CATALOG: &[FaultScenarioSpec] = &[ backend: FaultBackend::ChaosMeshIoChaos, status: FaultScenarioStatus::Executable, isolation: FaultIsolation::FreshTenant, + crds: &[IOCHAOS_CRD], + required_tools: &[], + percent_supported: true, + impact_policy: FaultImpactPolicy::ClientDisruptionRequired, boundary: "rustfs-workload/fault-injection", ci_phase: "faults", target: "one RustFS container data volume selected by tenant label and /data/rustfs0 path", @@ -101,6 +151,10 @@ pub const FAULT_SCENARIO_CATALOG: &[FaultScenarioSpec] = &[ backend: FaultBackend::ChaosMeshPodChaos, status: FaultScenarioStatus::Executable, isolation: FaultIsolation::ReusableTenant, + crds: &[PODCHAOS_CRD], + required_tools: &[], + percent_supported: false, + impact_policy: FaultImpactPolicy::ClientDisruptionRequired, boundary: "rustfs-workload/pod-recovery", ci_phase: "faults", target: "one RustFS Pod selected by tenant label", @@ -116,6 +170,10 @@ pub const FAULT_SCENARIO_CATALOG: &[FaultScenarioSpec] = &[ backend: FaultBackend::ChaosMeshNetworkChaos, status: FaultScenarioStatus::Executable, isolation: FaultIsolation::ReusableTenant, + crds: &[NETWORKCHAOS_CRD], + required_tools: &[], + percent_supported: false, + impact_policy: FaultImpactPolicy::ClientDisruptionRequired, boundary: "rustfs-workload/network-partition", ci_phase: "faults", target: "one RustFS Pod selected by tenant label with peer traffic disrupted inside the e2e namespace", @@ -123,6 +181,82 @@ pub const FAULT_SCENARIO_CATALOG: &[FaultScenarioSpec] = &[ observability: "history.jsonl, workload-summary.json, checker-report.json, networkchaos manifest/describe/yaml, endpoints, events, and RustFS logs", conflict_domain: "run-scoped NetworkChaos resource; must not overlap with PodChaos or IOChaos in the same Tenant", }, + FaultScenarioSpec { + scenario: NETWORK_DELAY_SCENARIO, + case_name: "fault_network_delay_preserves_object_model", + description: "Inject NetworkChaos delay into one RustFS Pod peer path and verify the S3 object model remains explainable.", + priority: FaultPriority::P1, + backend: FaultBackend::ChaosMeshNetworkChaos, + status: FaultScenarioStatus::Executable, + isolation: FaultIsolation::ReusableTenant, + crds: &[NETWORKCHAOS_CRD], + required_tools: &[], + percent_supported: false, + impact_policy: FaultImpactPolicy::ClientDisruptionOptional, + boundary: "rustfs-workload/network-delay", + ci_phase: "faults", + target: "one RustFS Pod selected by tenant label with delayed peer traffic inside the e2e namespace", + validation: "successful reads match a committed value, stable live keys are listed, and recovery preserves the object model", + observability: "history.jsonl, checker reports, networkchaos manifest/describe/yaml, endpoints, events, and RustFS logs", + conflict_domain: "run-scoped NetworkChaos resource; must not overlap with other network faults in the same Tenant", + }, + FaultScenarioSpec { + scenario: NETWORK_LOSS_SCENARIO, + case_name: "fault_network_loss_preserves_object_model", + description: "Inject NetworkChaos packet loss into one RustFS Pod peer path and verify object-model correctness after recovery.", + priority: FaultPriority::P1, + backend: FaultBackend::ChaosMeshNetworkChaos, + status: FaultScenarioStatus::Executable, + isolation: FaultIsolation::ReusableTenant, + crds: &[NETWORKCHAOS_CRD], + required_tools: &[], + percent_supported: false, + impact_policy: FaultImpactPolicy::ClientDisruptionRequired, + boundary: "rustfs-workload/network-loss", + ci_phase: "faults", + target: "one RustFS Pod selected by tenant label with lossy peer traffic inside the e2e namespace", + validation: "successful reads match a committed value, failed operations are explainable, and recovery preserves the object model", + observability: "history.jsonl, checker reports, networkchaos manifest/describe/yaml, endpoints, events, and RustFS logs", + conflict_domain: "run-scoped NetworkChaos resource; must not overlap with other network faults in the same Tenant", + }, + FaultScenarioSpec { + scenario: NETWORK_CORRUPT_SCENARIO, + case_name: "fault_network_corrupt_preserves_object_model", + description: "Inject NetworkChaos packet corruption into one RustFS Pod peer path and verify successful S3 reads never return corrupt bytes.", + priority: FaultPriority::P1, + backend: FaultBackend::ChaosMeshNetworkChaos, + status: FaultScenarioStatus::Executable, + isolation: FaultIsolation::ReusableTenant, + crds: &[NETWORKCHAOS_CRD], + required_tools: &[], + percent_supported: false, + impact_policy: FaultImpactPolicy::ClientDisruptionRequired, + boundary: "rustfs-workload/network-corrupt", + ci_phase: "faults", + target: "one RustFS Pod selected by tenant label with corrupted peer traffic inside the e2e namespace", + validation: "successful reads match a committed value and recovery preserves the object model", + observability: "history.jsonl, checker reports, networkchaos manifest/describe/yaml, endpoints, events, and RustFS logs", + conflict_domain: "run-scoped NetworkChaos resource; must not overlap with other network faults in the same Tenant", + }, + FaultScenarioSpec { + scenario: NETWORK_DUPLICATE_SCENARIO, + case_name: "fault_network_duplicate_preserves_object_model", + description: "Inject NetworkChaos packet duplication into one RustFS Pod peer path and verify object-model correctness after recovery.", + priority: FaultPriority::P1, + backend: FaultBackend::ChaosMeshNetworkChaos, + status: FaultScenarioStatus::Executable, + isolation: FaultIsolation::ReusableTenant, + crds: &[NETWORKCHAOS_CRD], + required_tools: &[], + percent_supported: false, + impact_policy: FaultImpactPolicy::ClientDisruptionOptional, + boundary: "rustfs-workload/network-duplicate", + ci_phase: "faults", + target: "one RustFS Pod selected by tenant label with duplicated peer traffic inside the e2e namespace", + validation: "successful reads match a committed value and recovery preserves the object model", + observability: "history.jsonl, checker reports, networkchaos manifest/describe/yaml, endpoints, events, and RustFS logs", + conflict_domain: "run-scoped NetworkChaos resource; must not overlap with other network faults in the same Tenant", + }, FaultScenarioSpec { scenario: IO_READ_MISTAKE_SCENARIO, case_name: "fault_io_read_mistake_rejects_corrupt_reads", @@ -131,6 +265,10 @@ pub const FAULT_SCENARIO_CATALOG: &[FaultScenarioSpec] = &[ backend: FaultBackend::ChaosMeshIoChaos, status: FaultScenarioStatus::Executable, isolation: FaultIsolation::FreshTenant, + crds: &[IOCHAOS_CRD], + required_tools: &[], + percent_supported: true, + impact_policy: FaultImpactPolicy::ClientDisruptionOptional, boundary: "rustfs-workload/data-integrity", ci_phase: "faults", target: "one RustFS data volume read path selected by tenant label and /data/rustfs0 path", @@ -138,6 +276,25 @@ pub const FAULT_SCENARIO_CATALOG: &[FaultScenarioSpec] = &[ observability: "history.jsonl, checker-report.json with successful_corrupted_reads, iochaos manifest/describe/yaml, RustFS logs, events", conflict_domain: "fresh Tenant/PVC/PV fixture and run-scoped IOChaos mistake resource", }, + FaultScenarioSpec { + scenario: IO_LATENCY_SCENARIO, + case_name: "fault_io_latency_preserves_object_model", + description: "Inject Chaos Mesh IOChaos latency on RustFS data paths and verify delayed storage does not corrupt the S3 object model.", + priority: FaultPriority::P1, + backend: FaultBackend::ChaosMeshIoChaos, + status: FaultScenarioStatus::Executable, + isolation: FaultIsolation::FreshTenant, + crds: &[IOCHAOS_CRD], + required_tools: &[], + percent_supported: true, + impact_policy: FaultImpactPolicy::ClientDisruptionOptional, + boundary: "rustfs-workload/storage-latency", + ci_phase: "faults", + target: "one RustFS data volume selected by tenant label with READ/WRITE operations delayed", + validation: "successful reads match a committed value, timed out operations remain explainable, and recovery preserves the object model", + observability: "history.jsonl, checker reports, iochaos manifest/describe/yaml, RustFS logs, events", + conflict_domain: "fresh Tenant/PVC/PV fixture and run-scoped IOChaos latency resource", + }, FaultScenarioSpec { scenario: DISK_FULL_SCENARIO, case_name: "fault_disk_full_preserves_committed_objects", @@ -146,6 +303,10 @@ pub const FAULT_SCENARIO_CATALOG: &[FaultScenarioSpec] = &[ backend: FaultBackend::ChaosMeshIoChaos, status: FaultScenarioStatus::Executable, isolation: FaultIsolation::FreshTenant, + crds: &[IOCHAOS_CRD], + required_tools: &[], + percent_supported: true, + impact_policy: FaultImpactPolicy::ClientDisruptionRequired, boundary: "rustfs-workload/storage-pressure", ci_phase: "faults", target: "one RustFS data volume selected by tenant label with WRITE operations returning ENOSPC", @@ -153,6 +314,63 @@ pub const FAULT_SCENARIO_CATALOG: &[FaultScenarioSpec] = &[ observability: "history.jsonl, checker-report.json, fault-evidence.json, IOChaos manifest/status, events, RustFS logs", conflict_domain: "fresh Tenant/PVC/PV fixture and run-scoped IOChaos cleanup without consuming node disk capacity", }, + FaultScenarioSpec { + scenario: POD_FAILURE_SCENARIO, + case_name: "fault_pod_failure_preserves_object_model", + description: "Inject Chaos Mesh PodChaos pod-failure against one RustFS Pod and verify object-model correctness after recovery.", + priority: FaultPriority::P1, + backend: FaultBackend::ChaosMeshPodChaos, + status: FaultScenarioStatus::Executable, + isolation: FaultIsolation::ReusableTenant, + crds: &[PODCHAOS_CRD], + required_tools: &[], + percent_supported: false, + impact_policy: FaultImpactPolicy::ClientDisruptionRequired, + boundary: "rustfs-workload/pod-failure", + ci_phase: "faults", + target: "one RustFS Pod selected by tenant label and failed for the scenario duration", + validation: "the failed Pod recovers, Tenant returns Ready, and the S3 object model remains explainable", + observability: "history.jsonl, checker reports, podchaos manifest/describe/yaml, Pod restart counts, current and previous RustFS logs", + conflict_domain: "run-scoped PodChaos resource and one target Pod; can reuse a ready Tenant after the prior scenario has cleaned up", + }, + FaultScenarioSpec { + scenario: STRESS_CPU_SCENARIO, + case_name: "fault_stress_cpu_preserves_object_model", + description: "Inject Chaos Mesh CPU StressChaos into one RustFS Pod and verify object-model correctness under resource pressure.", + priority: FaultPriority::P1, + backend: FaultBackend::ChaosMeshStressChaos, + status: FaultScenarioStatus::Executable, + isolation: FaultIsolation::ReusableTenant, + crds: &[STRESSCHAOS_CRD], + required_tools: &[], + percent_supported: false, + impact_policy: FaultImpactPolicy::ClientDisruptionOptional, + boundary: "rustfs-workload/cpu-pressure", + ci_phase: "faults", + target: "one RustFS Pod selected by tenant label with CPU stressors", + validation: "successful reads match a committed value and recovery preserves the object model", + observability: "history.jsonl, checker reports, stresschaos manifest/describe/yaml, metrics-adjacent Kubernetes snapshots, events, and RustFS logs", + conflict_domain: "run-scoped StressChaos resource; should not overlap with other stress faults in the same Tenant", + }, + FaultScenarioSpec { + scenario: STRESS_MEMORY_SCENARIO, + case_name: "fault_stress_memory_preserves_object_model", + description: "Inject Chaos Mesh memory StressChaos into one RustFS Pod and verify object-model correctness under memory pressure.", + priority: FaultPriority::P1, + backend: FaultBackend::ChaosMeshStressChaos, + status: FaultScenarioStatus::Executable, + isolation: FaultIsolation::ReusableTenant, + crds: &[STRESSCHAOS_CRD], + required_tools: &[], + percent_supported: false, + impact_policy: FaultImpactPolicy::ClientDisruptionOptional, + boundary: "rustfs-workload/memory-pressure", + ci_phase: "faults", + target: "one RustFS Pod selected by tenant label with memory stressors", + validation: "successful reads match a committed value and recovery preserves the object model", + observability: "history.jsonl, checker reports, stresschaos manifest/describe/yaml, metrics-adjacent Kubernetes snapshots, events, and RustFS logs", + conflict_domain: "run-scoped StressChaos resource; should not overlap with other stress faults in the same Tenant", + }, FaultScenarioSpec { scenario: DM_FLAKEY_SCENARIO, case_name: "fault_dm_flakey_preserves_committed_objects", @@ -161,6 +379,10 @@ pub const FAULT_SCENARIO_CATALOG: &[FaultScenarioSpec] = &[ backend: FaultBackend::DeviceMapper, status: FaultScenarioStatus::Executable, isolation: FaultIsolation::DedicatedLinuxBlockDevice, + crds: &[], + required_tools: &[], + percent_supported: false, + impact_policy: FaultImpactPolicy::ClientDisruptionRequired, boundary: "rustfs-workload/block-device-fault", ci_phase: "faults", target: "one dedicated Linux block-device-backed PV used only by the e2e Tenant", @@ -176,6 +398,10 @@ pub const FAULT_SCENARIO_CATALOG: &[FaultScenarioSpec] = &[ backend: FaultBackend::MinioWarpWithChaos, status: FaultScenarioStatus::Executable, isolation: FaultIsolation::FreshTenant, + crds: &[IOCHAOS_CRD], + required_tools: &["warp"], + percent_supported: true, + impact_policy: FaultImpactPolicy::ClientDisruptionOptional, boundary: "rustfs-workload/performance-under-chaos", ci_phase: "faults", target: "RustFS S3 endpoint under an explicitly selected fault backend", @@ -217,7 +443,7 @@ impl FaultScenario { ); config.workload.validate()?; ensure!( - !config.percent_overridden || spec.backend.accepts_percent(), + !config.percent_overridden || spec.percent_supported, "RUSTFS_FAULT_TEST_PERCENT only applies to percent-based IOChaos scenarios; scenario {:?} targets {:?} with a fixed target count", spec.scenario, spec.backend @@ -245,6 +471,10 @@ pub fn scenario_catalog() -> &'static [FaultScenarioSpec] { FAULT_SCENARIO_CATALOG } +pub fn scenario_catalog_json() -> Result { + Ok(serde_json::to_string_pretty(scenario_catalog())?) +} + pub fn scenario_spec(name: &str) -> Result<&'static FaultScenarioSpec> { FAULT_SCENARIO_CATALOG .iter() @@ -263,7 +493,7 @@ pub fn scenario_spec(name: &str) -> Result<&'static FaultScenarioSpec> { mod tests { use super::{ FaultScenario, FaultScenarioStatus, IO_EIO_SCENARIO, POD_KILL_ONE_SCENARIO, - scenario_catalog, + scenario_catalog, scenario_catalog_json, }; use crate::fault::config::{FaultTestConfig, FaultWorkloadProfile}; use std::time::Duration; @@ -328,7 +558,7 @@ mod tests { ); } - assert_eq!(scenario_catalog().len(), 7); + assert_eq!(scenario_catalog().len(), 15); } #[test] @@ -340,6 +570,10 @@ mod tests { assert!(names.insert(scenario.scenario)); assert!(case_names.insert(scenario.case_name)); assert!(!scenario.description.is_empty()); + assert_eq!( + scenario.percent_supported, + scenario.backend.accepts_percent() + ); assert!(!scenario.boundary.is_empty()); assert!(!scenario.ci_phase.is_empty()); assert!(!scenario.target.is_empty()); @@ -348,4 +582,15 @@ mod tests { assert!(!scenario.conflict_domain.is_empty()); } } + + #[test] + fn catalog_exports_machine_readable_json() { + let json = scenario_catalog_json().expect("catalog json"); + let value: serde_json::Value = serde_json::from_str(&json).expect("valid json"); + + assert!(value.as_array().expect("array").len() >= 10); + assert!(json.contains("\"scenario\": \"io-eio\"")); + assert!(json.contains("\"crds\"")); + assert!(json.contains("\"impact_policy\"")); + } } diff --git a/e2e/src/fault/workload.rs b/e2e/src/fault/workload.rs index 6c2bb32..87a0203 100644 --- a/e2e/src/fault/workload.rs +++ b/e2e/src/fault/workload.rs @@ -15,7 +15,13 @@ use anyhow::{Context, Result}; use aws_config::BehaviorVersion; use aws_credential_types::Credentials; -use aws_sdk_s3::{Client, config::Region, error::SdkError, primitives::ByteStream}; +use aws_sdk_s3::{ + Client, + config::Region, + error::SdkError, + primitives::ByteStream, + types::{CompletedMultipartUpload, CompletedPart}, +}; use serde::Serialize; use sha2::{Digest, Sha256}; use std::time::Duration; @@ -69,6 +75,13 @@ pub struct GetObjectResult { pub body: Option>, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VerifiedWriteResult { + pub write_outcome: OperationOutcome, + pub verify_get_outcome: Option, + pub verified: bool, +} + impl ObjectSpec { pub fn key_prefix(run_id: &str) -> String { format!("fault-test/{run_id}/") @@ -104,6 +117,22 @@ impl ObjectSpec { body, } } + + pub fn prepare_overwrite(&self, variant: u64) -> PreparedObject { + let seed = self.seed ^ variant.wrapping_mul(0x9E37_79B9_7F4A_7C15); + let body = seeded_bytes(seed, self.index, self.size_bytes); + let sha256 = sha256_hex(&body); + PreparedObject { + spec: Self { + key: self.key.clone(), + size_bytes: self.size_bytes, + sha256, + seed, + index: self.index, + }, + body, + } + } } impl WorkloadPlan { @@ -370,6 +399,359 @@ impl S3WorkloadClient { } } + pub async fn put_and_verify_object( + &self, + object: &PreparedObject, + recorder: &Recorder, + ) -> Result { + let write_outcome = self.put_object(object, recorder).await?; + if write_outcome != OperationOutcome::Ok { + return Ok(VerifiedWriteResult { + write_outcome, + verify_get_outcome: None, + verified: false, + }); + } + + let get = self.get_object_result(&object.spec.key, recorder).await?; + let verified = get.body.as_deref().is_some_and(|body| { + body.len() == object.spec.size_bytes && sha256_hex(body) == object.spec.sha256 + }); + Ok(VerifiedWriteResult { + write_outcome, + verify_get_outcome: Some(get.outcome), + verified, + }) + } + + pub async fn delete_object(&self, key: &str, recorder: &Recorder) -> Result { + let record = recorder.begin( + OperationKind::Delete, + self.bucket.clone(), + Some(key.to_string()), + None, + None, + ); + let result = timeout( + self.request_timeout, + self.client + .delete_object() + .bucket(&self.bucket) + .key(key) + .send(), + ) + .await; + + match result { + Ok(Ok(_)) => { + recorder.finish(record, OperationOutcome::Ok, Some(204), None)?; + Ok(OperationOutcome::Ok) + } + Ok(Err(error)) => { + let outcome = classify_sdk_error(&error); + recorder.finish( + record, + outcome, + sdk_error_status(&error), + Some(format!("delete object failed: {error}")), + )?; + Ok(outcome) + } + Err(_) => { + recorder.finish( + record, + OperationOutcome::Timeout, + None, + Some("delete object timed out".to_string()), + )?; + Ok(OperationOutcome::Timeout) + } + } + } + + pub async fn delete_and_verify_absent( + &self, + key: &str, + recorder: &Recorder, + ) -> Result<(OperationOutcome, Option)> { + let delete_outcome = self.delete_object(key, recorder).await?; + if delete_outcome != OperationOutcome::Ok { + return Ok((delete_outcome, None)); + } + let get = self.get_object_result(key, recorder).await?; + Ok((delete_outcome, Some(get.outcome))) + } + + pub async fn complete_multipart_object( + &self, + object: &PreparedObject, + recorder: &Recorder, + ) -> Result { + let Some(upload_id) = self + .create_multipart_upload(&object.spec.key, recorder) + .await? + else { + return Ok(OperationOutcome::Unknown); + }; + let mut completed_parts = Vec::new(); + for (index, chunk) in object.body.chunks(5 * 1024 * 1024).enumerate() { + let part_number = (index + 1) as i32; + match self + .upload_part(&object.spec.key, &upload_id, part_number, chunk, recorder) + .await? + { + Some(part) => completed_parts.push(part), + None => { + let _ = self + .abort_multipart_upload(&object.spec.key, &upload_id, recorder) + .await; + return Ok(OperationOutcome::Unknown); + } + } + } + + let record = recorder.begin( + OperationKind::CompleteMultipartUpload, + self.bucket.clone(), + Some(object.spec.key.clone()), + Some(object.spec.sha256.clone()), + Some(object.spec.size_bytes), + ); + let upload = CompletedMultipartUpload::builder() + .set_parts(Some(completed_parts)) + .build(); + let result = timeout( + self.request_timeout, + self.client + .complete_multipart_upload() + .bucket(&self.bucket) + .key(&object.spec.key) + .upload_id(upload_id) + .multipart_upload(upload) + .send(), + ) + .await; + + match result { + Ok(Ok(_)) => { + recorder.finish(record, OperationOutcome::Ok, Some(200), None)?; + Ok(OperationOutcome::Ok) + } + Ok(Err(error)) => { + let outcome = classify_sdk_error(&error); + recorder.finish( + record, + outcome, + sdk_error_status(&error), + Some(format!("complete multipart upload failed: {error}")), + )?; + Ok(outcome) + } + Err(_) => { + recorder.finish( + record, + OperationOutcome::Timeout, + None, + Some("complete multipart upload timed out".to_string()), + )?; + Ok(OperationOutcome::Timeout) + } + } + } + + pub async fn abort_multipart_object( + &self, + object: &PreparedObject, + recorder: &Recorder, + ) -> Result { + let Some(upload_id) = self + .create_multipart_upload(&object.spec.key, recorder) + .await? + else { + return Ok(OperationOutcome::Unknown); + }; + self.abort_multipart_upload(&object.spec.key, &upload_id, recorder) + .await + } + + async fn create_multipart_upload( + &self, + key: &str, + recorder: &Recorder, + ) -> Result> { + let record = recorder.begin( + OperationKind::CreateMultipartUpload, + self.bucket.clone(), + Some(key.to_string()), + None, + None, + ); + let result = timeout( + self.request_timeout, + self.client + .create_multipart_upload() + .bucket(&self.bucket) + .key(key) + .send(), + ) + .await; + + match result { + Ok(Ok(output)) => { + let Some(upload_id) = output.upload_id().map(str::to_string) else { + recorder.finish( + record, + OperationOutcome::Unknown, + Some(200), + Some("create multipart upload omitted upload_id".to_string()), + )?; + return Ok(None); + }; + recorder.finish(record, OperationOutcome::Ok, Some(200), None)?; + Ok(Some(upload_id)) + } + Ok(Err(error)) => { + let outcome = classify_sdk_error(&error); + recorder.finish( + record, + outcome, + sdk_error_status(&error), + Some(format!("create multipart upload failed: {error}")), + )?; + Ok(None) + } + Err(_) => { + recorder.finish( + record, + OperationOutcome::Timeout, + None, + Some("create multipart upload timed out".to_string()), + )?; + Ok(None) + } + } + } + + async fn upload_part( + &self, + key: &str, + upload_id: &str, + part_number: i32, + body: &[u8], + recorder: &Recorder, + ) -> Result> { + let record = recorder.begin( + OperationKind::UploadPart, + self.bucket.clone(), + Some(key.to_string()), + Some(sha256_hex(body)), + Some(body.len()), + ); + let result = timeout( + self.request_timeout, + self.client + .upload_part() + .bucket(&self.bucket) + .key(key) + .upload_id(upload_id) + .part_number(part_number) + .body(ByteStream::from(body.to_vec())) + .send(), + ) + .await; + + match result { + Ok(Ok(output)) => { + let Some(e_tag) = output.e_tag().map(str::to_string) else { + recorder.finish( + record, + OperationOutcome::Unknown, + Some(200), + Some(format!("upload part {part_number} omitted ETag")), + )?; + return Ok(None); + }; + recorder.finish(record, OperationOutcome::Ok, Some(200), None)?; + Ok(Some( + CompletedPart::builder() + .part_number(part_number) + .e_tag(e_tag) + .build(), + )) + } + Ok(Err(error)) => { + let outcome = classify_sdk_error(&error); + recorder.finish( + record, + outcome, + sdk_error_status(&error), + Some(format!("upload part {part_number} failed: {error}")), + )?; + Ok(None) + } + Err(_) => { + recorder.finish( + record, + OperationOutcome::Timeout, + None, + Some(format!("upload part {part_number} timed out")), + )?; + Ok(None) + } + } + } + + async fn abort_multipart_upload( + &self, + key: &str, + upload_id: &str, + recorder: &Recorder, + ) -> Result { + let record = recorder.begin( + OperationKind::AbortMultipartUpload, + self.bucket.clone(), + Some(key.to_string()), + None, + None, + ); + let result = timeout( + self.request_timeout, + self.client + .abort_multipart_upload() + .bucket(&self.bucket) + .key(key) + .upload_id(upload_id) + .send(), + ) + .await; + + match result { + Ok(Ok(_)) => { + recorder.finish(record, OperationOutcome::Ok, Some(204), None)?; + Ok(OperationOutcome::Ok) + } + Ok(Err(error)) => { + let outcome = classify_sdk_error(&error); + recorder.finish( + record, + outcome, + sdk_error_status(&error), + Some(format!("abort multipart upload failed: {error}")), + )?; + Ok(outcome) + } + Err(_) => { + recorder.finish( + record, + OperationOutcome::Timeout, + None, + Some("abort multipart upload timed out".to_string()), + )?; + Ok(OperationOutcome::Timeout) + } + } + } + pub async fn head_object(&self, key: &str, recorder: &Recorder) -> Result { let record = recorder.begin( OperationKind::Head, @@ -484,6 +866,7 @@ impl S3WorkloadClient { let mut record = record; record.size_bytes = Some(keys.len()); + record.listed_keys = Some(keys.clone()); recorder.finish(record, OperationOutcome::Ok, Some(200), None)?; Ok(Some(keys)) } @@ -553,6 +936,9 @@ fn classify_sdk_error(error: &SdkError) -> OperationOutcome { match error { SdkError::TimeoutError(_) => OperationOutcome::Timeout, SdkError::DispatchFailure(_) | SdkError::ResponseError(_) => OperationOutcome::Unknown, + SdkError::ServiceError(context) if context.raw().status().as_u16() == 404 => { + OperationOutcome::NotFound + } SdkError::ConstructionFailure(_) | SdkError::ServiceError(_) => OperationOutcome::Failed, _ => OperationOutcome::Unknown, }