diff --git a/crates/xtask/src/tmt.rs b/crates/xtask/src/tmt.rs index 795830343..cd6ea815f 100644 --- a/crates/xtask/src/tmt.rs +++ b/crates/xtask/src/tmt.rs @@ -1,3 +1,7 @@ +use std::sync::{Arc, Mutex, mpsc}; +use std::time::Duration; +use std::usize; + use anyhow::{Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; use fn_error_context::context; @@ -25,6 +29,10 @@ const FIELD_ADJUST: &str = "adjust"; const FIELD_FIXME_SKIP_IF_COMPOSEFS: &str = "fixme_skip_if_composefs"; const FIELD_FIXME_SKIP_IF_UKI: &str = "fixme_skip_if_uki"; +/// For tests that should only run for composefs systems +/// Ex. composefs-gc +const FIELD_SKIP_IF_OSTREE: &str = "skip_if_ostree"; + // bcvk options const BCVK_OPT_BIND_STORAGE_RO: &str = "--bind-storage-ro"; const ENV_BOOTC_UPGRADE_IMAGE: &str = "BOOTC_upgrade_image"; @@ -32,6 +40,36 @@ const ENV_BOOTC_UPGRADE_IMAGE: &str = "BOOTC_upgrade_image"; // Distro identifiers const DISTRO_CENTOS_9: &str = "centos-9"; +// Tests sorted by time taken (descending) +const TESTS_SORTED_BY_TIME: [&str; 23] = [ + // 10+ mins + "multi-device-esp", + "composefs-gc-uki", + "composefs-gc", + // 5+ mins + "loader-entries-source", + "download-only-upgrade", + "bib-build", + "rollback", + "logically-bound-switch", + "soft-reboot", + "switch-to-unified", + "image-pushpull-upgrade", + "install-no-boot-dir", + "upgrade-tag", + "custom-selinux-policy", + "factory-reset", + // 3+ mins + "upgrade-check-status", + "soft-reboot-selinux-policy", + "install-bootloader-none", + "install-outside-container", + "install-unified-flag", + "usroverlay", + "image-upgrade-reboot", + "install-karg-delete", +]; + // Import the argument types from xtask.rs use crate::bcvk::BcvkInstallOpts; use crate::{RunTmtArgs, SealState, TmtProvisionArgs, out_of_sync_error}; @@ -207,10 +245,11 @@ fn verify_ssh_connectivity(sh: &Shell, port: u16, key_path: &Utf8Path) -> Result ) } -#[derive(Debug)] +#[derive(Debug, Default)] struct PlanMetadata { try_bind_storage: bool, skip_if_composefs: bool, + skip_if_ostree: bool, skip_if_uki: bool, } @@ -252,8 +291,7 @@ fn parse_plan_metadata( .and_modify(|m| m.try_bind_storage = b) .or_insert(PlanMetadata { try_bind_storage: b, - skip_if_uki: false, - skip_if_composefs: false, + ..Default::default() }); } } @@ -268,8 +306,7 @@ fn parse_plan_metadata( .and_modify(|m| m.skip_if_composefs = b) .or_insert(PlanMetadata { skip_if_composefs: b, - skip_if_uki: false, - try_bind_storage: false, + ..Default::default() }); } } @@ -284,8 +321,22 @@ fn parse_plan_metadata( .and_modify(|m| m.skip_if_uki = b) .or_insert(PlanMetadata { skip_if_uki: b, - skip_if_composefs: false, - try_bind_storage: false, + ..Default::default() + }); + } + } + + if let Some(skip_if_ostree) = plan_data.get(&serde_yaml::Value::String(format!( + "extra-{}", + FIELD_SKIP_IF_OSTREE + ))) { + if let Some(b) = skip_if_ostree.as_bool() { + plan_metadata + .entry(plan_name.to_string()) + .and_modify(|m| m.skip_if_ostree = b) + .or_insert(PlanMetadata { + skip_if_ostree: b, + ..Default::default() }); } } @@ -294,6 +345,372 @@ fn parse_plan_metadata( Ok(plan_metadata) } +struct RunPlanResult { + plan_name: String, + passed: bool, + time_taken: Option, + run_id: Option, +} + +struct VmConfig { + cpu: String, + mem: String, + parallel_count: usize, + preserve: bool, +} + +struct TestContext<'a> { + random_suffix: &'a str, + distro: &'a str, + image: &'a str, + upgrade_image: Option<&'a str>, +} + +struct TmtConfig<'a> { + context: &'a [String], + env_vars: &'a [String], + user_env: &'a [String], +} + +impl RunPlanResult { + fn new( + plan_name: String, + passed: bool, + time_taken: Option, + run_id: Option, + ) -> Self { + Self { + plan_name, + passed, + time_taken, + run_id, + } + } +} + +fn run_plan( + plan: &str, + vm_name: &str, + image: &str, + plan_bcvk_opts: Vec, + firmware_args: Vec, + context: Vec, + tmt_env_vars: Vec, + arg_env: Vec, + preserve_vm: bool, + vm_cpu: &str, + vm_mem_mb: &str, + libvirt_lock: Arc>, +) -> RunPlanResult { + let sh = match Shell::new() { + Ok(sh) => sh, + Err(err) => { + eprintln!("Failed to create new shell instance: {err:?}"); + return RunPlanResult::new(plan.to_string(), false, None, None); + } + }; + + // Launch VM with bcvk + let firmware_args_slice = firmware_args.as_slice(); + + let guard = match libvirt_lock.lock() { + Ok(g) => g, + Err(e) => { + eprintln!("Mutex lock failed for plan {plan}: {e:#}"); + return RunPlanResult::new(plan.to_string(), false, None, None); + } + }; + + let launch_result = cmd!( + sh, + "bcvk libvirt run --name {vm_name} --memory {vm_mem_mb} --cpus {vm_cpu} --detach {firmware_args_slice...} {COMMON_INST_ARGS...} {plan_bcvk_opts...} {image}" + ) + .run() + .context("Launching VM with bcvk"); + + drop(guard); + + if let Err(e) = launch_result { + eprintln!("Failed to launch VM for plan {}: {:#}", plan, e); + return RunPlanResult::new(plan.to_string(), false, None, None); + } + + // Ensure VM cleanup happens even on error (unless --preserve-vm is set) + let cleanup_vm = || { + if preserve_vm { + return; + } + if let Err(e) = cmd!(sh, "bcvk libvirt rm --stop --force {vm_name}") + .ignore_stderr() + .ignore_status() + .run() + { + eprintln!("Warning: Failed to cleanup VM {}: {}", vm_name, e); + } + }; + + // Wait for VM to be ready and get SSH info + let vm_info = wait_for_vm_ready(&sh, &vm_name); + let (ssh_port, ssh_key) = match vm_info { + Ok((port, key)) => (port, key), + Err(e) => { + eprintln!("Failed to get VM info for plan {}: {:#}", plan, e); + cleanup_vm(); + return RunPlanResult::new(plan.to_string(), false, None, None); + } + }; + + println!("VM ready, SSH port: {}", ssh_port); + + // Save SSH private key to a temporary file + let key_file = tempfile::NamedTempFile::new().context("Creating temporary SSH key file"); + + let key_file = match key_file { + Ok(f) => f, + Err(e) => { + eprintln!("Failed to create SSH key file for plan {}: {:#}", plan, e); + cleanup_vm(); + return RunPlanResult::new(plan.to_string(), false, None, None); + } + }; + + let key_path = Utf8PathBuf::try_from(key_file.path().to_path_buf()) + .context("Converting key path to UTF-8"); + + let key_path = match key_path { + Ok(p) => p, + Err(e) => { + eprintln!("Failed to convert key path for plan {}: {:#}", plan, e); + cleanup_vm(); + return RunPlanResult::new(plan.to_string(), false, None, None); + } + }; + + if let Err(e) = std::fs::write(&key_path, ssh_key) { + eprintln!("Failed to write SSH key for plan {}: {:#}", plan, e); + cleanup_vm(); + return RunPlanResult::new(plan.to_string(), false, None, None); + } + + // Set proper permissions on the key file (SSH requires 0600) + { + use std::os::unix::fs::PermissionsExt; + let perms = std::fs::Permissions::from_mode(0o600); + if let Err(e) = std::fs::set_permissions(&key_path, perms) { + eprintln!("Failed to set key permissions for plan {}: {:#}", plan, e); + cleanup_vm(); + return RunPlanResult::new(plan.to_string(), false, None, None); + } + } + + // Verify SSH connectivity + println!("Verifying SSH connectivity..."); + if let Err(e) = verify_ssh_connectivity(&sh, ssh_port, &key_path) { + eprintln!("SSH verification failed for plan {}: {:#}", plan, e); + cleanup_vm(); + return RunPlanResult::new(plan.to_string(), false, None, None); + } + + println!("SSH connectivity verified"); + + let ssh_port_str = ssh_port.to_string(); + + let time_start = std::time::Instant::now(); + + // Run tmt for this specific plan using connect provisioner + println!("Running tmt tests for plan {}...", plan); + + // Generate a unique run ID for this test + // Use the VM name which already contains a random suffix for uniqueness + let run_id = vm_name; + + // Run tmt for this specific plan + // Note: provision must come before plan for connect to work properly + let how = ["--how=connect", "--guest=localhost", "--user=root"]; + let env = ["TMT_SCRIPTS_DIR=/var/lib/tmt/scripts", "BCVK_EXPORT=1"] + .into_iter() + .chain(arg_env.iter().map(|v| v.as_str())) + .chain(tmt_env_vars.iter().map(|v| v.as_str())) + .flat_map(|v| ["--environment", v]); + let test_result = cmd!( + sh, + "tmt {context...} run --id {run_id} --all {env...} provision {how...} --port {ssh_port_str} --key {key_path} plan --name {plan}" + ) + .run(); + + let elapsed = time_start.elapsed(); + + // Log disk usage after each test run to help diagnose "no space left on device" failures + println!("Disk usage after plan {}:", plan); + let _ = cmd!(sh, "df -h").run(); + + // Clean up VM regardless of test result (unless --preserve-vm is set) + cleanup_vm(); + + let plan_result = match test_result { + Ok(_) => { + println!("Plan {} completed successfully", plan); + RunPlanResult::new( + plan.to_string(), + true, + Some(elapsed), + Some(run_id.to_string()), + ) + } + Err(e) => { + eprintln!("Plan {} failed: {:#}", plan, e); + RunPlanResult::new( + plan.to_string(), + false, + Some(elapsed), + Some(run_id.to_string()), + ) + } + }; + + // Print VM connection details if preserving + if preserve_vm { + // Copy SSH key to a persistent location + let persistent_key_path = Utf8Path::new("target").join(format!("{}.ssh-key", vm_name)); + if let Err(e) = std::fs::copy(&key_path, &persistent_key_path) { + eprintln!("Warning: Failed to save persistent SSH key: {}", e); + } else { + println!("\n========================================"); + println!("VM preserved for debugging:"); + println!("========================================"); + println!("VM name: {}", vm_name); + println!("SSH port: {}", ssh_port_str); + println!("SSH key: {}", persistent_key_path); + println!("\nTo connect via SSH:"); + println!( + " ssh -i {} -p {} -o IdentitiesOnly=yes root@localhost", + persistent_key_path, ssh_port_str + ); + println!("\nTo cleanup:"); + println!(" bcvk libvirt rm --stop --force {}", vm_name); + println!("========================================\n"); + } + } + + plan_result +} + +fn run_plans( + plans: Vec<&str>, + plan_metadata: &std::collections::HashMap, + bcvk_opts: &crate::bcvk::BcvkInstallOpts, + firmware_args: &[String], + vm_config: VmConfig, + test_context: TestContext, + tmt_config: &TmtConfig, + libvirt_lock: &Arc>, +) -> Result> { + let mut test_results: Vec = Vec::new(); + let mut active_threads = 0; + let (tx, rx) = mpsc::channel::(); + + for plan in plans { + let plan_name = sanitize_plan_name(plan); + let vm_name = format!("bootc-tmt-{}-{}", test_context.random_suffix, plan_name); + + println!("\n========================================"); + println!("Running plan: {}", plan); + println!("VM name: {}", vm_name); + println!("========================================\n"); + + // Get bcvk-opts based on plan metadata and distro support + let plan_bcvk_opts = { + let supports_bind_storage_ro = distro_supports_bind_storage_ro(test_context.distro); + let try_bind_storage = plan_metadata + .iter() + .find(|(key, _)| plan.ends_with(key.as_str())) + .map(|(_, v)| v.try_bind_storage) + .unwrap_or(false); + + let mut opts = Vec::new(); + let mut env_vars = tmt_config.env_vars.to_vec(); + + if try_bind_storage && supports_bind_storage_ro { + opts.push(BCVK_OPT_BIND_STORAGE_RO.to_string()); + if let Some(upgrade_img) = test_context.upgrade_image { + env_vars.push(format!("{}={}", ENV_BOOTC_UPGRADE_IMAGE, upgrade_img)); + } + } else if try_bind_storage && !supports_bind_storage_ro { + println!( + "Note: Test wants bind storage but skipping on {} (missing systemd.extra-unit.* support)", + test_context.distro + ); + } + + opts.extend(bcvk_opts.install_args()); + opts + }; + + let firmware_args = firmware_args.to_vec(); + let context = tmt_config.context.to_vec(); + let env_vars = tmt_config.env_vars.to_vec(); + let user_env = tmt_config.user_env.to_vec(); + let cloned_plan = plan.to_string(); + let cloned_vm_name = vm_name.to_string(); + let image = test_context.image.to_string(); + let vm_mem = vm_config.mem.clone(); + let vm_cpu = vm_config.cpu.clone(); + let libvirt_lock = libvirt_lock.clone(); + + let tx_clone = tx.clone(); + std::thread::spawn(move || { + let result = run_plan( + &cloned_plan, + &cloned_vm_name, + &image, + plan_bcvk_opts, + firmware_args, + context, + env_vars, + user_env, + vm_config.preserve, + &vm_cpu, + &vm_mem, + libvirt_lock, + ); + + if let Err(e) = tx_clone.send(result) { + eprintln!("Failed to send result through channel: {}", e); + } + }); + + active_threads += 1; + + if active_threads >= vm_config.parallel_count { + match rx.recv() { + Ok(plan_result) => { + test_results.push(plan_result); + active_threads -= 1; + } + Err(e) => { + eprintln!("Failed to receive result from channel: {}", e); + active_threads -= 1; + } + } + } + } + + drop(tx); + + for _ in 0..active_threads { + match rx.recv() { + Ok(plan_result) => { + test_results.push(plan_result); + } + Err(e) => { + eprintln!("Failed to receive remaining result from channel: {}", e); + } + } + } + + Ok(test_results) +} + /// Run TMT tests using bcvk for VM management /// This spawns a separate VM per test plan to avoid state leakage between tests. #[context("Running TMT tests")] @@ -391,6 +808,14 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { .map(|(_, v)| v.skip_if_composefs) .unwrap_or(false) }); + } else { + plans.retain(|plan| { + !plan_metadata + .iter() + .find(|(key, _)| plan.ends_with(key.as_str())) + .map(|(_, v)| v.skip_if_ostree) + .unwrap_or(false) + }); } if matches!(args.boot_type, crate::BootType::Uki) { @@ -417,273 +842,146 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { return Ok(()); } - println!("Found {} test plan(s): {:?}", plans.len(), plans); - - // Generate a random suffix for VM names - let random_suffix = generate_random_suffix(); - - // Track overall success/failure - let mut all_passed = true; - let mut test_results: Vec<(String, bool, Option)> = Vec::new(); - - // Environment variables to pass to tmt (in addition to args.env) - let mut tmt_env_vars = Vec::new(); - - // Run each plan in its own VM - for plan in plans { - let plan_name = sanitize_plan_name(plan); - let vm_name = format!("bootc-tmt-{}-{}", random_suffix, plan_name); - - println!("\n========================================"); - println!("Running plan: {}", plan); - println!("VM name: {}", vm_name); - println!("========================================\n"); - - // Reset plan-specific environment variables - tmt_env_vars.clear(); - - // Get bcvk-opts based on plan metadata and distro support - let plan_bcvk_opts = { - let supports_bind_storage_ro = distro_supports_bind_storage_ro(&distro); - - // Plan names from tmt are like /tmt/plans/integration/plan-01-readonly - // but metadata keys are like /plan-01-readonly, so match on suffix - let try_bind_storage = plan_metadata - .iter() - .find(|(key, _)| plan.ends_with(key.as_str())) - .map(|(_, v)| v.try_bind_storage) - .unwrap_or(false); - - let mut opts = Vec::new(); - - // If test wants bind storage and distro supports it, add --bind-storage-ro - if try_bind_storage && supports_bind_storage_ro { - opts.push(BCVK_OPT_BIND_STORAGE_RO.to_string()); - - // If upgrade image is provided, set it as an environment variable for tmt - // (not bcvk, as bcvk doesn't support --env) - if let Some(ref upgrade_img) = args.upgrade_image { - tmt_env_vars.push(format!("{}={}", ENV_BOOTC_UPGRADE_IMAGE, upgrade_img)); - } - } else if try_bind_storage && !supports_bind_storage_ro { - println!( - "Note: Test wants bind storage but skipping on {} (missing systemd.extra-unit.* support)", - distro - ); - } - // Add --filesystem=xfs by default on fedora-coreos - if variant_id == "coreos" { - if distro.starts_with("fedora") { - opts.push("--filesystem=xfs".to_string()); - } - } - - opts.extend(bcvk_opts.install_args()); + plans.sort_by_key(|full_plan_name| { + TESTS_SORTED_BY_TIME + .iter() + .position(|test_time| full_plan_name.contains(test_time)) + .unwrap_or(usize::MAX) + }); - opts - }; + println!("Found {} test plan(s): {:?}", plans.len(), plans); - // Launch VM with bcvk - let firmware_args_slice = firmware_args.as_slice(); - let launch_result = cmd!( - sh, - "bcvk libvirt run --name {vm_name} --detach {firmware_args_slice...} {COMMON_INST_ARGS...} {plan_bcvk_opts...} {image}" - ) - .run() - .context("Launching VM with bcvk"); + let mut install_opts = Vec::new(); - if let Err(e) = launch_result { - eprintln!("Failed to launch VM for plan {}: {:#}", plan, e); - all_passed = false; - test_results.push((plan.to_string(), false, None)); - continue; + // Add --filesystem=xfs by default on fedora-coreos + if variant_id == "coreos" { + if distro.starts_with("fedora") { + install_opts.push("--filesystem=xfs".to_string()); } + } - // Ensure VM cleanup happens even on error (unless --preserve-vm is set) - let cleanup_vm = || { - if preserve_vm { - return; - } - if let Err(e) = cmd!(sh, "bcvk libvirt rm --stop --force {vm_name}") - .ignore_stderr() - .ignore_status() - .run() - { - eprintln!("Warning: Failed to cleanup VM {}: {}", vm_name, e); - } - }; - - // Wait for VM to be ready and get SSH info - let vm_info = wait_for_vm_ready(sh, &vm_name); - let (ssh_port, ssh_key) = match vm_info { - Ok((port, key)) => (port, key), - Err(e) => { - eprintln!("Failed to get VM info for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false, None)); - continue; - } - }; - - println!("VM ready, SSH port: {}", ssh_port); - - // Save SSH private key to a temporary file - let key_file = tempfile::NamedTempFile::new().context("Creating temporary SSH key file"); + if args.composefs_backend { + let filesystem = args.filesystem.as_deref().unwrap_or("ext4"); + install_opts.push(format!("--filesystem={}", filesystem)); + install_opts.push("--composefs-backend".into()); - let key_file = match key_file { - Ok(f) => f, - Err(e) => { - eprintln!("Failed to create SSH key file for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false, None)); - continue; - } - }; + if let Some(b) = &args.bootloader { + install_opts.push(format!("--bootloader={b}")); + } + } - let key_path = Utf8PathBuf::try_from(key_file.path().to_path_buf()) - .context("Converting key path to UTF-8"); + for k in &args.karg { + install_opts.push(format!("--karg={k}")); + } - let key_path = match key_path { - Ok(p) => p, - Err(e) => { - eprintln!("Failed to convert key path for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false, None)); - continue; - } - }; + let start = std::time::Instant::now(); - if let Err(e) = std::fs::write(&key_path, ssh_key) { - eprintln!("Failed to write SSH key for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false, None)); - continue; - } + println!("Creating base disk..."); + let opts = install_opts.clone(); + cmd!(sh, "bcvk libvirt to-base-disk {opts...} localhost/bootc").run()?; - // Set proper permissions on the key file (SSH requires 0600) - { - use std::os::unix::fs::PermissionsExt; - let perms = std::fs::Permissions::from_mode(0o600); - if let Err(e) = std::fs::set_permissions(&key_path, perms) { - eprintln!("Failed to set key permissions for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false, None)); - continue; - } - } + println!("Creating base disk took: {:#?}", start.elapsed()); - // Verify SSH connectivity - println!("Verifying SSH connectivity..."); - if let Err(e) = verify_ssh_connectivity(sh, ssh_port, &key_path) { - eprintln!("SSH verification failed for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false, None)); - continue; - } + // Generate a random suffix for VM names + let random_suffix = generate_random_suffix(); - println!("SSH connectivity verified"); + let num_cpu = std::thread::available_parallelism() + .map(|c| c.get()) + .unwrap_or(1); - let ssh_port_str = ssh_port.to_string(); + println!("num_cpu: {num_cpu}"); - // Run tmt for this specific plan using connect provisioner - println!("Running tmt tests for plan {}...", plan); + let (vm_cpu, vm_mem) = (1, 2048); + let avail_cpu = (num_cpu - 1).max(1); + let parallel_vms = (avail_cpu / vm_cpu).min(6); - // Generate a unique run ID for this test - // Use the VM name which already contains a random suffix for uniqueness - let run_id = vm_name.clone(); + println!("parallel_vms: {parallel_vms}"); - // Run tmt for this specific plan - // Note: provision must come before plan for connect to work properly - let context = context.clone(); - let how = ["--how=connect", "--guest=localhost", "--user=root"]; - let env = ["TMT_SCRIPTS_DIR=/var/lib/tmt/scripts", "BCVK_EXPORT=1"] - .into_iter() - .chain(args.env.iter().map(|v| v.as_str())) - .chain(tmt_env_vars.iter().map(|v| v.as_str())) - .flat_map(|v| ["--environment", v]); - let test_result = cmd!( - sh, - "tmt {context...} run --id {run_id} --all {env...} provision {how...} --port {ssh_port_str} --key {key_path} plan --name {plan}" - ) - .run(); + let libvirt_lock = Arc::new(Mutex::new(())); - // Log disk usage after each test run to help diagnose "no space left on device" failures - println!("Disk usage after plan {}:", plan); - let _ = cmd!(sh, "df -h").run(); + let vm_config = VmConfig { + cpu: vm_cpu.to_string(), + mem: vm_mem.to_string(), + parallel_count: parallel_vms, + preserve: preserve_vm, + }; - // Clean up VM regardless of test result (unless --preserve-vm is set) - cleanup_vm(); + let test_context = TestContext { + random_suffix: &random_suffix, + distro: &distro, + image, + upgrade_image: args.upgrade_image.as_deref(), + }; - match test_result { - Ok(_) => { - println!("Plan {} completed successfully", plan); - test_results.push((plan.to_string(), true, Some(run_id))); - } - Err(e) => { - eprintln!("Plan {} failed: {:#}", plan, e); - all_passed = false; - test_results.push((plan.to_string(), false, Some(run_id))); - } - } + let tmt_config = TmtConfig { + context: &context, + env_vars: &Vec::new(), // Will be populated per plan + user_env: &args.env, + }; - // Print VM connection details if preserving - if preserve_vm { - // Copy SSH key to a persistent location - let persistent_key_path = Utf8Path::new("target").join(format!("{}.ssh-key", vm_name)); - if let Err(e) = std::fs::copy(&key_path, &persistent_key_path) { - eprintln!("Warning: Failed to save persistent SSH key: {}", e); - } else { - println!("\n========================================"); - println!("VM preserved for debugging:"); - println!("========================================"); - println!("VM name: {}", vm_name); - println!("SSH port: {}", ssh_port_str); - println!("SSH key: {}", persistent_key_path); - println!("\nTo connect via SSH:"); - println!( - " ssh -i {} -p {} -o IdentitiesOnly=yes root@localhost", - persistent_key_path, ssh_port_str - ); - println!("\nTo cleanup:"); - println!(" bcvk libvirt rm --stop --force {}", vm_name); - println!("========================================\n"); - } - } - } + // Run all plans + let mut test_results = run_plans( + plans, + &plan_metadata, + &bcvk_opts, + &firmware_args, + vm_config, + test_context, + &tmt_config, + &libvirt_lock, + )?; // Print summary println!("\n========================================"); println!("Test Summary"); println!("========================================"); - for (plan, passed, _) in &test_results { + + test_results.sort_by(|a, b| b.time_taken.cmp(&a.time_taken)); + + for RunPlanResult { + plan_name: plan, + passed, + time_taken, + .. + } in &test_results + { let status = if *passed { "PASSED" } else { "FAILED" }; - println!("{}: {}", plan, status); + println!( + "{}: {} ({:?})", + plan, + status, + time_taken.unwrap_or(Duration::from_secs(0)) + ); } println!("========================================\n"); // Print detailed error reports for failed tests let failed_tests: Vec<_> = test_results .iter() - .filter(|(_, passed, _)| !passed) + .filter(|plan_res| !plan_res.passed) .collect(); - if !failed_tests.is_empty() { - println!("\n========================================"); - println!("Detailed Error Reports"); - println!("========================================\n"); + if failed_tests.is_empty() { + println!("All tests passed"); + return Ok(()); + } - for (plan, _, run_id) in failed_tests { - println!("----------------------------------------"); - println!("Plan: {}", plan); - println!("----------------------------------------"); + println!("\n========================================"); + println!("Detailed Error Reports"); + println!("========================================\n"); - if let Some(id) = run_id { + for RunPlanResult { + plan_name: plan, + run_id, + .. + } in &failed_tests + { + println!("----------------------------------------"); + println!("Plan: {}", plan); + println!("----------------------------------------"); + + match run_id { + Some(id) => { println!("Run ID: {}\n", id); // Run tmt with the specific run ID and generate verbose report @@ -700,18 +998,82 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { ); } } - } else { - println!("Run ID not available - cannot generate detailed report"); } - println!("\n"); + None => { + println!("Run ID not available - cannot generate detailed report"); + } } - println!("========================================\n"); + println!("\n"); } - if !all_passed { - anyhow::bail!("Some test plans failed"); + println!("========================================\n"); + + // Rerun failed plans + println!("Rerunning failed plans...\n"); + + let failed_plan_names: Vec<&str> = failed_tests + .iter() + .map(|result| result.plan_name.as_str()) + .collect(); + + let rerun_suffix = format!("{}-rerun", random_suffix); + let test_context_rerun = TestContext { + random_suffix: &rerun_suffix, + distro: &distro, + image, + upgrade_image: args.upgrade_image.as_deref(), + }; + + let vm_config_rerun = VmConfig { + cpu: vm_cpu.to_string(), + mem: vm_mem.to_string(), + parallel_count: parallel_vms, + preserve: preserve_vm, + }; + + let rerun_results = run_plans( + failed_plan_names, + &plan_metadata, + &bcvk_opts, + &firmware_args, + vm_config_rerun, + test_context_rerun, + &tmt_config, + &libvirt_lock, + )?; + + // Print rerun summary + println!("\n========================================"); + println!("Rerun Summary"); + println!("========================================"); + + let mut rerun_all_passed = true; + for RunPlanResult { + plan_name: plan, + passed, + time_taken, + .. + } in &rerun_results + { + let status = if *passed { + "PASSED" + } else { + rerun_all_passed = false; + "FAILED" + }; + println!( + "{}: {} ({:?})", + plan, + status, + time_taken.unwrap_or(Duration::from_secs(0)) + ); + } + println!("========================================\n"); + + if !rerun_all_passed { + anyhow::bail!("Some test plans still failed after rerun"); } Ok(()) @@ -906,6 +1268,8 @@ struct TestDef { try_bind_storage: bool, /// Whether to skip this test for composefs backend skip_if_composefs: bool, + /// Whether to skip this test for ostree backend + skip_if_ostree: bool, /// Whether to skip this test for images with UKI skip_if_uki: bool, /// TMT fmf attributes to pass through (summary, duration, adjust, etc.) @@ -1071,6 +1435,13 @@ fn generate_integration() -> Result<(String, String)> { .and_then(|v| v.as_bool()) .unwrap_or(false); + let skip_if_ostree = metadata + .extra + .as_mapping() + .and_then(|m| m.get(&serde_yaml::Value::String(FIELD_SKIP_IF_OSTREE.to_string()))) + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let skip_if_uki = metadata .extra .as_mapping() @@ -1088,6 +1459,7 @@ fn generate_integration() -> Result<(String, String)> { test_command, try_bind_storage, skip_if_composefs, + skip_if_ostree, skip_if_uki, tmt: metadata.tmt, }); @@ -1211,6 +1583,13 @@ fn generate_integration() -> Result<(String, String)> { ); } + if test.skip_if_ostree { + plan_value.insert( + serde_yaml::Value::String(format!("extra-{}", FIELD_SKIP_IF_OSTREE)), + serde_yaml::Value::Bool(true), + ); + } + if test.skip_if_uki { plan_value.insert( serde_yaml::Value::String(format!("extra-{}", FIELD_FIXME_SKIP_IF_UKI)), diff --git a/tmt/plans/integration.fmf b/tmt/plans/integration.fmf index b3366648b..d27a77058 100644 --- a/tmt/plans/integration.fmf +++ b/tmt/plans/integration.fmf @@ -196,6 +196,8 @@ execute: how: fmf test: - /tmt/tests/tests/test-35-composefs-gc + extra-skip_if_ostree: true + extra-fixme_skip_if_uki: true /plan-35-upgrade-preflight-disk-check: summary: Verify pre-flight disk space check rejects images with inflated layer sizes @@ -254,6 +256,7 @@ execute: how: fmf test: - /tmt/tests/tests/test-41-composefs-gc-uki + extra-skip_if_ostree: true /plan-42-loader-entries-source: summary: Test bootc loader-entries set-options-for-source @@ -269,6 +272,7 @@ execute: how: fmf test: - /tmt/tests/tests/test-43-switch-same-digest + extra-skip_if_ostree: true /plan-44-shadow-fixup: summary: Test bootc-sysusers-shadow-sync removes orphaned gshadow entries before sysusers diff --git a/tmt/tests/booted/tap.nu b/tmt/tests/booted/tap.nu index b8b0b3f7e..63076d3c6 100644 --- a/tmt/tests/booted/tap.nu +++ b/tmt/tests/booted/tap.nu @@ -112,3 +112,15 @@ export def make_uki_containerfile [containerfile: string] { return $"($containerfile)\n($uki_stuff)" } + +# We share host container storage in the VM, so we usually already have localhost/bootc +# as an image in the VM. If we don't find it, only then do we copy from ostree/composefs +# storage +export def img_cp_to_store_smart [] { + let podman_digest = (podman image inspect localhost/bootc) | jq -r '.[0].Digest' + let bootc_digest = (bootc status --json | from json).status.booted.image.imageDigest + + if ($podman_digest != $bootc_digest) { + bootc image copy-to-storage + } +} diff --git a/tmt/tests/booted/test-bib-build.nu b/tmt/tests/booted/test-bib-build.nu index 765698204..ac26ce7eb 100644 --- a/tmt/tests/booted/test-bib-build.nu +++ b/tmt/tests/booted/test-bib-build.nu @@ -26,12 +26,14 @@ const BIB_IMAGE = "quay.io/centos-bootc/bootc-image-builder:latest" def main [] { tap begin "bootc-image-builder qcow2 build test" + print ">>>>>>>>>>>>>>>>>> the current working directory <<<<<<<<<<<<<<<<<" ($env.PWD) + let td = mktemp -d cd $td # Copy the currently booted image to podman storage print "=== Copying booted image to containers-storage ===" - bootc image copy-to-storage + tap img_cp_to_store_smart # Verify the image is in storage let images = podman images --format json | from json @@ -87,8 +89,10 @@ DISKEOF ' | save Dockerfile podman build -t localhost/bootc-bib-test . + let output_dir = "/var/output" + # Create output directory for bib - mkdir output + mkdir $output_dir # Run bootc-image-builder to create a qcow2 # We use --local to pull from local containers-storage @@ -97,11 +101,11 @@ DISKEOF let bib_image = $BIB_IMAGE # Note: we disable SELinux labeling since we're running in a test VM # and use unconfined_t to avoid permission issues - podman run --rm --privileged -v /var/lib/containers/storage:/var/lib/containers/storage --security-opt label=type:unconfined_t -v ./output:/output $bib_image --type qcow2 --rootfs xfs localhost/bootc-bib-test + podman run --rm --privileged -v /var/lib/containers/storage:/var/lib/containers/storage --security-opt label=type:unconfined_t -v $"($output_dir):/output" $bib_image --type qcow2 --rootfs xfs localhost/bootc-bib-test # Verify output was created print "=== Verifying output ===" - let disk_path = "output/qcow2/disk.qcow2" + let disk_path = $"($output_dir)/qcow2/disk.qcow2" assert ($disk_path | path exists) $"Expected disk image at ($disk_path)" # Check the disk has reasonable virtual size (at least 4GB as per disk.yaml) diff --git a/tmt/tests/booted/test-composefs-gc-uki.nu b/tmt/tests/booted/test-composefs-gc-uki.nu index e2efd561c..dbcb4d63e 100644 --- a/tmt/tests/booted/test-composefs-gc-uki.nu +++ b/tmt/tests/booted/test-composefs-gc-uki.nu @@ -2,6 +2,8 @@ # tmt: # summary: Test composefs garbage collection for UKI # duration: 30m +# extra: +# skip_if_ostree: true use std assert use tap.nu @@ -24,7 +26,7 @@ if not $is_uki { # Create a large file in a new container image, then bootc switch to the image def first_boot [] { - bootc image copy-to-storage + tap img_cp_to_store_smart mut containerfile = $" FROM localhost/bootc as base diff --git a/tmt/tests/booted/test-composefs-gc.nu b/tmt/tests/booted/test-composefs-gc.nu index d19a01efe..a7fd4a8ce 100644 --- a/tmt/tests/booted/test-composefs-gc.nu +++ b/tmt/tests/booted/test-composefs-gc.nu @@ -2,6 +2,9 @@ # tmt: # summary: Test composefs garbage collection with same and different kernel+initrd # duration: 30m +# extra: +# skip_if_ostree: true +# fixme_skip_if_uki: true use std assert use tap.nu @@ -22,7 +25,7 @@ if ($st.status.booted.composefs.bootType | str downcase) == "uki" { # Create a large file in a new container image, then bootc switch to the image def first_boot [] { - bootc image copy-to-storage + tap img_cp_to_store_smart echo $" FROM localhost/bootc diff --git a/tmt/tests/booted/test-custom-selinux-policy.nu b/tmt/tests/booted/test-custom-selinux-policy.nu index 80f34be30..52edc4a32 100644 --- a/tmt/tests/booted/test-custom-selinux-policy.nu +++ b/tmt/tests/booted/test-custom-selinux-policy.nu @@ -24,7 +24,7 @@ def initial_build [] { let td = mktemp -d cd $td - bootc image copy-to-storage + tap img_cp_to_store_smart # A simple derived container that customizes selinux policy for random dir "FROM localhost/bootc diff --git a/tmt/tests/booted/test-download-only-upgrade.nu b/tmt/tests/booted/test-download-only-upgrade.nu index 59dcdc3f5..331027348 100644 --- a/tmt/tests/booted/test-download-only-upgrade.nu +++ b/tmt/tests/booted/test-download-only-upgrade.nu @@ -40,7 +40,7 @@ def initial_build [] { # This test only works in local mode assert ($imgsrc | str ends-with "-local") "This test requires local mode" - bootc image copy-to-storage + tap img_cp_to_store_smart # Create test file v1 on host "v1" | save testing-bootc-upgrade-apply diff --git a/tmt/tests/booted/test-image-pushpull-upgrade.nu b/tmt/tests/booted/test-image-pushpull-upgrade.nu index 708b868ec..d51260d67 100644 --- a/tmt/tests/booted/test-image-pushpull-upgrade.nu +++ b/tmt/tests/booted/test-image-pushpull-upgrade.nu @@ -39,7 +39,7 @@ def initial_build [] { let td = mktemp -d cd $td - bootc image copy-to-storage + tap img_cp_to_store_smart let img = podman image inspect localhost/bootc | from json mkdir usr/lib/bootc/kargs.d diff --git a/tmt/tests/booted/test-image-upgrade-reboot.nu b/tmt/tests/booted/test-image-upgrade-reboot.nu index f28cff1da..7fcab4283 100644 --- a/tmt/tests/booted/test-image-upgrade-reboot.nu +++ b/tmt/tests/booted/test-image-upgrade-reboot.nu @@ -46,7 +46,7 @@ def initial_build [] { let imgsrc = imgsrc # For the packit case, we build locally right now if ($imgsrc | str ends-with "-local") { - bootc image copy-to-storage + tap img_cp_to_store_smart # A simple derived container that adds a file ( diff --git a/tmt/tests/booted/test-install-bootloader-none.nu b/tmt/tests/booted/test-install-bootloader-none.nu index ef89deae7..2c0e52ed1 100644 --- a/tmt/tests/booted/test-install-bootloader-none.nu +++ b/tmt/tests/booted/test-install-bootloader-none.nu @@ -10,7 +10,7 @@ def main [] { tap begin "install with --bootloader=none" # Copy the booted image to container storage for use as install source - bootc image copy-to-storage + tap img_cp_to_store_smart let target_image = "containers-storage:localhost/bootc" truncate -s 10G disk.img diff --git a/tmt/tests/booted/test-install-no-boot-dir.nu b/tmt/tests/booted/test-install-no-boot-dir.nu index bc012fd66..a8859cd9b 100644 --- a/tmt/tests/booted/test-install-no-boot-dir.nu +++ b/tmt/tests/booted/test-install-no-boot-dir.nu @@ -10,7 +10,7 @@ def main [] { tap begin "install to-filesystem without /boot" # Copy the booted image to container storage for use as install source - bootc image copy-to-storage + tap img_cp_to_store_smart let target_image = "containers-storage:localhost/bootc" mkdir /var/mnt diff --git a/tmt/tests/booted/test-install-outside-container.nu b/tmt/tests/booted/test-install-outside-container.nu index 8c88cf17a..eccb2c1e3 100644 --- a/tmt/tests/booted/test-install-outside-container.nu +++ b/tmt/tests/booted/test-install-outside-container.nu @@ -10,7 +10,7 @@ use tap.nu # EFI update metadata. Export to OCI layout on a writable path since # containers-storage: transport can't work when the root fs is read-only # (composefs), and install-outside-container tests run directly on the host. -bootc image copy-to-storage +tap img_cp_to_store_smart skopeo copy containers-storage:localhost/bootc oci:/var/tmp/bootc-oci let target_image = "oci:/var/tmp/bootc-oci" diff --git a/tmt/tests/booted/test-loader-entries-source.nu b/tmt/tests/booted/test-loader-entries-source.nu index b52656798..164834b92 100644 --- a/tmt/tests/booted/test-loader-entries-source.nu +++ b/tmt/tests/booted/test-loader-entries-source.nu @@ -206,7 +206,7 @@ def fifth_boot [] { # Build a derived image and switch to it (this stages a deployment). # Then call set-options-for-source on top. The staged deployment should # be replaced with one that has the new image AND the source kargs. - bootc image copy-to-storage + tap img_cp_to_store_smart let td = mktemp -d $"FROM localhost/bootc diff --git a/tmt/tests/booted/test-logically-bound-switch.nu b/tmt/tests/booted/test-logically-bound-switch.nu index 298d7ff86..7da50dc2d 100644 --- a/tmt/tests/booted/test-logically-bound-switch.nu +++ b/tmt/tests/booted/test-logically-bound-switch.nu @@ -29,7 +29,7 @@ let booted = $st.status.booted.image echo '{}' | save -f /run/ostree/auth.json def initial_setup [] { - bootc image copy-to-storage + tap img_cp_to_store_smart podman images podman image inspect localhost/bootc | from json } diff --git a/tmt/tests/booted/test-multi-device-esp.nu b/tmt/tests/booted/test-multi-device-esp.nu index b3f69fcf3..d0d704311 100644 --- a/tmt/tests/booted/test-multi-device-esp.nu +++ b/tmt/tests/booted/test-multi-device-esp.nu @@ -168,7 +168,7 @@ def run_install [mountpoint: string] { def test_single_esp [] { tap begin "multi-device ESP detection tests" - bootc image copy-to-storage + tap img_cp_to_store_smart print "Starting single ESP test" diff --git a/tmt/tests/booted/test-rollback.nu b/tmt/tests/booted/test-rollback.nu index 0afe377ac..03dee51c4 100644 --- a/tmt/tests/booted/test-rollback.nu +++ b/tmt/tests/booted/test-rollback.nu @@ -39,7 +39,7 @@ def initial_switch [] { let imgsrc = imgsrc if ($imgsrc | str ends-with "-local") { - bootc image copy-to-storage + tap img_cp_to_store_smart print "Building derived container" let dockerfile = $"FROM localhost/bootc as base diff --git a/tmt/tests/booted/test-soft-reboot-selinux-policy.nu b/tmt/tests/booted/test-soft-reboot-selinux-policy.nu index 4e2706804..c3ca145f4 100644 --- a/tmt/tests/booted/test-soft-reboot-selinux-policy.nu +++ b/tmt/tests/booted/test-soft-reboot-selinux-policy.nu @@ -33,7 +33,7 @@ def initial_build [] { let td = mktemp -d cd $td - bootc image copy-to-storage + tap img_cp_to_store_smart # copy-to-storage does not copy repo file # but OSCI gating test needs repo to install package diff --git a/tmt/tests/booted/test-soft-reboot.nu b/tmt/tests/booted/test-soft-reboot.nu index 9be3fbcd5..3274161cc 100644 --- a/tmt/tests/booted/test-soft-reboot.nu +++ b/tmt/tests/booted/test-soft-reboot.nu @@ -24,7 +24,7 @@ def initial_build [] { let td = mktemp -d cd $td - bootc image copy-to-storage + tap img_cp_to_store_smart # A simple derived container that adds a file, but also injects some kargs let dockerfile = $"FROM localhost/bootc as base diff --git a/tmt/tests/booted/test-switch-same-digest.nu b/tmt/tests/booted/test-switch-same-digest.nu index eda56d467..de1dcd6d2 100644 --- a/tmt/tests/booted/test-switch-same-digest.nu +++ b/tmt/tests/booted/test-switch-same-digest.nu @@ -2,6 +2,8 @@ # tmt: # summary: Error on bootc switch to image with identical fs-verity digest # duration: 10m +# extra: +# skip_if_ostree: true # # Verify that `bootc switch` errors out when the target image produces the # same composefs fs-verity digest as an existing deployment. The simplest @@ -17,7 +19,7 @@ if not (tap is_composefs) { tap begin "bootc switch to same-digest image must error" # Copy the booted image into podman storage so we can retag it. -bootc image copy-to-storage +tap img_cp_to_store_smart # Tag the same image under a second name — identical bits, so the composefs # EROFS digest will be the same as the currently booted deployment. diff --git a/tmt/tests/booted/test-switch-to-unified.nu b/tmt/tests/booted/test-switch-to-unified.nu index 4e8b61442..9b7f9d497 100644 --- a/tmt/tests/booted/test-switch-to-unified.nu +++ b/tmt/tests/booted/test-switch-to-unified.nu @@ -29,7 +29,7 @@ def first_boot [] { tap begin "copy image to podman storage, switch, then onboard to unified storage" # Copy the currently booted image to podman storage - bootc image copy-to-storage + tap img_cp_to_store_smart # Switch to the base image using containers-storage transport bootc switch --transport containers-storage localhost/bootc diff --git a/tmt/tests/booted/test-upgrade-check-status.nu b/tmt/tests/booted/test-upgrade-check-status.nu index bde2813c2..731ffe149 100644 --- a/tmt/tests/booted/test-upgrade-check-status.nu +++ b/tmt/tests/booted/test-upgrade-check-status.nu @@ -30,7 +30,7 @@ def imgsrc [] { def initial_build [] { tap begin "upgrade --check cached update in status" - bootc image copy-to-storage + tap img_cp_to_store_smart # A simple derived container that adds a file with a version label "FROM localhost/bootc diff --git a/tmt/tests/booted/test-upgrade-tag.nu b/tmt/tests/booted/test-upgrade-tag.nu index 2125762c8..941170f59 100644 --- a/tmt/tests/booted/test-upgrade-tag.nu +++ b/tmt/tests/booted/test-upgrade-tag.nu @@ -23,7 +23,7 @@ def initial_build [] { cd $td # Copy bootc image to local storage - bootc image copy-to-storage + tap img_cp_to_store_smart # Build v1 image let dockerfile = $"FROM localhost/bootc as base