diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..b1c8f87b3 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.'cfg(target_os = "macos")'] +runner = "dev/macos-sign-and-run.sh" diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e634063b..d2e3c314b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Changed +Certain fixed guest addresses were changed on AArch64 to more easily +accommodate 16k pages without wasting memory. Snapshots taken from +sandboxes using the old addresses will not be loadable by new +hyperlight versions. + ### Removed ### Fixed diff --git a/Cargo.lock b/Cargo.lock index 7db5b5dd0..56fc157d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1663,10 +1663,12 @@ name = "hyperlight-host" version = "0.16.0" dependencies = [ "anyhow", + "bindgen", "bitflags 2.13.1", "blake3", "built", "bytemuck", + "cc", "cfg-if", "cfg_aliases", "chrono", @@ -1704,6 +1706,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "opentelemetry_sdk", "page_size", + "parking_lot", "proc-maps", "proptest", "rand 0.10.2", diff --git a/dev/macos-entitlements.plist b/dev/macos-entitlements.plist new file mode 100644 index 000000000..c2ef1a38b --- /dev/null +++ b/dev/macos-entitlements.plist @@ -0,0 +1,8 @@ + + + + + com.apple.security.hypervisor + + + diff --git a/dev/macos-sign-and-run.sh b/dev/macos-sign-and-run.sh new file mode 100755 index 000000000..cc9bcef93 --- /dev/null +++ b/dev/macos-sign-and-run.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + + +codesign -f -s - --entitlements "$(dirname "$0")/macos-entitlements.plist" "$1" +exec "$@" diff --git a/flake.lock b/flake.lock index 487dd3b5c..faf4c21f8 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1770115704, - "narHash": "sha256-KHFT9UWOF2yRPlAnSXQJh6uVcgNcWlFqqiAZ7OVlHNc=", + "lastModified": 1782723713, + "narHash": "sha256-oPXCU/SSUokcGaJREHibG1CBX3+s/W7orDWQOZDsEeQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e6eae2ee2110f3d31110d5c222cd395303343b08", + "rev": "b5aa0fbd538984f6e3d201be0005b4463d8b09f8", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index ae01e57b9..f8a0f5ce8 100644 --- a/flake.nix +++ b/flake.nix @@ -11,8 +11,8 @@ orig = super.rustChannelOf args; patchRustPkg = pkg: (pkg.overrideAttrs (oA: { buildCommand = (builtins.replaceStrings - [ "rustc,rustdoc" ] - [ "rustc,rustdoc,clippy-driver,cargo-clippy,miri,cargo-miri" ] + [ "rustc,rustdoc" "librustc_driver-*.so" ] + [ "rustc,rustdoc,clippy-driver,cargo-clippy,miri,cargo-miri" "librustc_driver-*.{so,dylib}" ] oA.buildCommand) + (let wrapperPath = self.path + "/pkgs/build-support/bintools-wrapper/ld-wrapper.sh"; baseOut = self.clangStdenv.cc.bintools.out; @@ -51,7 +51,7 @@ toolchainVersionAttrs = args; }; })) // { - targetPlatforms = [ "aarch64-linux" "x86_64-linux" ]; + targetPlatforms = [ "aarch64-linux" "x86_64-linux" "aarch64-darwin" ]; badTargetPlatforms = [ ]; }; overrideRustPkg = pkg: self.lib.makeOverridable (origArgs: @@ -82,7 +82,7 @@ "x86_64-unknown-linux-gnu" "x86_64-pc-windows-msvc" "x86_64-unknown-none" "wasm32-wasip1" "wasm32-wasip2" "wasm32-unknown-unknown" - "aarch64-unknown-none" + "aarch64-unknown-none" "aarch64-apple-darwin" ]; extensions = [ "rust-src" ] ++ (if args.channel == "nightly" then [ "miri-preview" ] else []); }); @@ -166,13 +166,14 @@ esac if [ -f ''${root}/flake.nix ]; then - mkdir -p $root/$.cargo - cat >$root/.cargo/config.toml <>$root/.cargo/config.toml < Result<()> { // re-run the build if this script is changed (or deleted!), // even if the rust code is completely unchanged. println!("cargo:rerun-if-changed=build.rs"); + let out_dir = std::env::var("OUT_DIR")?; + let out_path = std::path::PathBuf::from(&out_dir); + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR")?; // Windows requires the hyperlight_surrogate.exe binary to be next to the executable running // hyperlight. We are using rust-embed to include the binary in the hyperlight-host library @@ -41,9 +44,7 @@ fn main() -> Result<()> { // We need to copy/rename the source for hyperlight surrogate into a // temp directory because we cannot include a file name `Cargo.toml` // inside this package. - let out_dir = std::env::var("OUT_DIR")?; std::fs::create_dir_all(format!("{out_dir}/hyperlight_surrogate/src"))?; - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR")?; std::fs::copy( format!("{manifest_dir}/src/hyperlight_surrogate/src/main.rs"), format!("{out_dir}/hyperlight_surrogate/src/main.rs"), @@ -62,7 +63,7 @@ fn main() -> Result<()> { // be the same as the CARGO_TARGET_DIR for the hyperlight-host otherwise // the build script will hang. Using a sub directory works tho! // xref - https://github.com/rust-lang/cargo/issues/6412 - let target_dir = std::path::PathBuf::from(&out_dir).join("../../hls"); + let target_dir = out_path.join("../../hls"); let profile = std::env::var("PROFILE")?; let build_profile = if profile.to_lowercase() == "debug" { @@ -101,6 +102,56 @@ fn main() -> Result<()> { ); } + if std::env::var("CARGO_CFG_TARGET_OS")? == "macos" + && std::env::var("CARGO_FEATURE_HVF").is_ok() + { + println!("cargo:rerun-if-env-changed=BINDGEN_EXTRA_CLANG_ARGS"); + println!( + "cargo:rerun-if-changed=src/hyperlight_host/src/hypervisor/virtual_machine/hvf/bindings.h" + ); + println!( + "cargo:rerun-if-changed=src/hyperlight_host/src/hypervisor/virtual_machine/hvf/fp_abi.c" + ); + println!("cargo:rustc-link-lib=framework=Hypervisor"); + bindgen::Builder::default() + .clang_args(&["-framework", "Hypervisor"]) + .clang_args(&[ + "-I", + &format!("{manifest_dir}/src/hypervisor/virtual_machine/hvf"), + ]) + // The hvf simd register functions use C simd vector + // parameters/returns for the register value, which Rust + // does not presently stably support for `extern "C"` + // code. So, we don't generate native bindings for these + // functions, and instead generate C stubs (see `fp_abi.c`, + // which is built below) + .blocklist_type("hv_simd_fp_uchar16_t") + .blocklist_function("hv_vcpu_get_simd_fp_reg") + .blocklist_function("hv_vcpu_set_simd_fp_reg") + .default_alias_style(bindgen::AliasVariation::NewType) + .type_alias("hv_memory_flags_t") + .newtype_enum("hv_reg_t") + .newtype_enum("hv_simd_fp_reg_t") + .newtype_enum("hv_sys_reg_t") + .newtype_enum("hv_exit_reason_t") + .no_debug("hv_return_t") + .formatter(bindgen::Formatter::Prettyplease) + .header(format!( + "{manifest_dir}/src/hypervisor/virtual_machine/hvf/bindings.h" + )) + .generate() + .context("Unable to generate hvf bindings")? + .write_to_file(out_path.join("hvf_bindings.rs")) + .context("Couldn't write hvf bindings")?; + + cc::Build::new() + .opt_level(3) + .file(format!( + "{manifest_dir}/src/hypervisor/virtual_machine/hvf/fp_abi.c" + )) + .compile("hyperlight_host_hvf_abi_wrapper"); + } + // Makes #[cfg(kvm)] == #[cfg(all(feature = "kvm", target_os = "linux"))] // Essentially the kvm and mshv3 features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")]. // You should never use #[cfg(feature = "kvm")] or #[cfg(feature = "mshv3")] in the codebase. @@ -108,6 +159,7 @@ fn main() -> Result<()> { gdb: { all(feature = "gdb", debug_assertions, target_arch = "x86_64") }, kvm: { all(feature = "kvm", target_os = "linux") }, mshv3: { all(feature = "mshv3", target_os = "linux") }, + hvf: { all(feature = "hvf", target_os = "macos") }, crashdump: { all(feature = "crashdump", target_arch = "x86_64") }, // print_debug feature is aliased with debug_assertions to make it only available in debug-builds. print_debug: { all(feature = "print_debug", debug_assertions) }, diff --git a/src/hyperlight_host/src/hypervisor/hyperlight_vm/aarch64.rs b/src/hyperlight_host/src/hypervisor/hyperlight_vm/aarch64.rs index 770959c14..15b196173 100644 --- a/src/hyperlight_host/src/hypervisor/hyperlight_vm/aarch64.rs +++ b/src/hyperlight_host/src/hypervisor/hyperlight_vm/aarch64.rs @@ -17,24 +17,28 @@ limitations under the License. // TODO(aarch64): implement arch-specific HyperlightVm methods use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64}; use super::{ AccessPageTableError, CreateHyperlightVmError, DispatchGuestCallError, HyperlightVm, InitializeError, }; +#[cfg(hvf)] +use crate::hypervisor::HvfInterruptHandle; +use crate::hypervisor::InterruptHandleImpl; +#[cfg(any(kvm, mshv3))] +use crate::hypervisor::LinuxInterruptHandle; #[cfg(gdb)] use crate::hypervisor::gdb::{DebugCommChannel, DebugMsg, DebugResponse}; use crate::hypervisor::hyperlight_vm::get_guest_log_filter; use crate::hypervisor::regs::{CommonFpu, CommonRegisters, CommonSpecialRegisters}; +#[cfg(hvf)] +use crate::hypervisor::virtual_machine::hvf::HvfVm; #[cfg(kvm)] use crate::hypervisor::virtual_machine::kvm::KvmVm; -#[cfg(kvm)] -use crate::hypervisor::virtual_machine::{HypervisorType, VmError}; use crate::hypervisor::virtual_machine::{ - RegisterError, ResetVcpuError, VirtualMachine, get_available_hypervisor, + HypervisorType, RegisterError, ResetVcpuError, VirtualMachine, VmError, + get_available_hypervisor, }; -use crate::hypervisor::{InterruptHandleImpl, LinuxInterruptHandle}; use crate::mem::mgr::{SandboxMemoryManager, SnapshotSharedMemory}; use crate::mem::shared_mem::{GuestSharedMemory, HostSharedMemory}; use crate::sandbox::SandboxConfiguration; @@ -47,6 +51,7 @@ use crate::sandbox::uninitialized::SandboxRuntimeConfig; impl HyperlightVm { #[allow(clippy::too_many_arguments)] + #[cfg_attr(target_os = "macos", allow(unused))] pub(crate) fn new( snapshot_mem: SnapshotSharedMemory, scratch_mem: GuestSharedMemory, @@ -61,23 +66,26 @@ impl HyperlightVm { ) -> std::result::Result { // TODO: support gdb on aarch64 type VmType = Box; - let vm: VmType = match get_available_hypervisor() { + #[cfg(hvf)] + let interrupt_handle: Arc = + Arc::new(HvfInterruptHandle::new(config.get_interrupt_retry_delay())); + let mut vm: VmType = match get_available_hypervisor() { #[cfg(kvm)] Some(HypervisorType::Kvm) => Box::new(KvmVm::new().map_err(VmError::CreateVm)?), // TODO: mshv support #[cfg(mshv3)] Some(HypervisorType::Mshv) => return Err(CreateHyperlightVmError::NoHypervisorFound), + #[cfg(hvf)] + Some(HypervisorType::Hvf) => { + Box::new(HvfVm::new(interrupt_handle.clone()).map_err(VmError::CreateVm)?) + } None => return Err(CreateHyperlightVmError::NoHypervisorFound), }; vm.set_sregs(&CommonSpecialRegisters::defaults(root_pt_addr)) .map_err(VmError::Register)?; - let interrupt_handle: Arc = Arc::new(LinuxInterruptHandle { - state: AtomicU8::new(0), - tid: AtomicU64::new(unsafe { libc::pthread_self() as u64 }), - retry_delay: config.get_interrupt_retry_delay(), - sig_rt_min_offset: config.get_interrupt_vcpu_sigrtmin_offset(), - dropped: AtomicBool::new(false), - }); + #[cfg(any(kvm, mshv3))] + let interrupt_handle: Arc = + Arc::new(LinuxInterruptHandle::new(config)); let snapshot_slot = 0u32; let scratch_slot = 1u32; diff --git a/src/hyperlight_host/src/hypervisor/hyperlight_vm/mod.rs b/src/hyperlight_host/src/hypervisor/hyperlight_vm/mod.rs index 2e545aba7..b8a9087ef 100644 --- a/src/hyperlight_host/src/hypervisor/hyperlight_vm/mod.rs +++ b/src/hyperlight_host/src/hypervisor/hyperlight_vm/mod.rs @@ -587,7 +587,7 @@ impl HyperlightVm { } pub(crate) fn clear_cancel(&self) { - self.interrupt_handle.clear_cancel(); + self.interrupt_handle.state().clear_cancel(); } pub(super) fn run( @@ -607,12 +607,12 @@ impl HyperlightVm { // without sending any signals/WHV api calls #[cfg(any(kvm, mshv3))] self.interrupt_handle.set_tid(); - self.interrupt_handle.set_running(); + self.interrupt_handle.state().set_running(); // NOTE: `set_running()`` must be called before checking `is_cancelled()` // otherwise we risk missing a call to `kill()` because the vcpu would not be marked as running yet so signals won't be sent - let exit_reason = if self.interrupt_handle.is_cancelled() - || self.interrupt_handle.is_debug_interrupted() + let exit_reason = if self.interrupt_handle.state().is_cancelled() + || self.interrupt_handle.state().is_debug_interrupted() { Ok(VmExit::Cancelled()) } else { @@ -652,14 +652,14 @@ impl HyperlightVm { // If kill() is called and ran to completion BEFORE this line executes: // - CANCEL_BIT will be set. Cancellation is deferred to the next iteration. // - Signals will be sent until `clear_running()` is called, which is ok - self.interrupt_handle.clear_running(); + self.interrupt_handle.state().clear_running(); // ===== KILL() TIMING POINT 5: Before capturing cancel_requested ===== // If kill() is called and ran to completion BEFORE this line executes: // - CANCEL_BIT will be set. Cancellation is deferred to the next iteration. // - Signals will not be sent - let cancel_requested = self.interrupt_handle.is_cancelled(); - let debug_interrupted = self.interrupt_handle.is_debug_interrupted(); + let cancel_requested = self.interrupt_handle.state().is_cancelled(); + let debug_interrupted = self.interrupt_handle.state().is_debug_interrupted(); // ===== KILL() TIMING POINT 6: Before checking exit_reason ===== // If kill() is called and ran to completion BEFORE this line executes: @@ -754,7 +754,7 @@ impl HyperlightVm { // If the vcpu was interrupted by a debugger, we need to handle it #[cfg(gdb)] { - self.interrupt_handle.clear_debug_interrupt(); + self.interrupt_handle.state().clear_debug_interrupt(); if let Err(e) = self.handle_debug(dbg_mem_access_fn.clone(), VcpuStopReason::Interrupt) { diff --git a/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs b/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs index ae17e100b..efefff921 100644 --- a/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs +++ b/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs @@ -18,11 +18,6 @@ limitations under the License. use std::collections::HashMap; #[cfg(crashdump)] use std::path::Path; -#[cfg(any(kvm, mshv3))] -use std::sync::atomic::AtomicBool; -use std::sync::atomic::AtomicU8; -#[cfg(any(kvm, mshv3))] -use std::sync::atomic::AtomicU64; use std::sync::{Arc, Mutex}; use tracing::{Span, instrument}; @@ -32,6 +27,8 @@ use super::*; use crate::hypervisor::InterruptHandleImpl; #[cfg(any(kvm, mshv3))] use crate::hypervisor::LinuxInterruptHandle; +#[cfg(target_os = "windows")] +use crate::hypervisor::WindowsInterruptHandle; #[cfg(crashdump)] use crate::hypervisor::crashdump; #[cfg(gdb)] @@ -54,8 +51,6 @@ use crate::hypervisor::virtual_machine::whp::WhpVm; use crate::hypervisor::virtual_machine::{ HypervisorType, RegisterError, VmError, get_available_hypervisor, }; -#[cfg(target_os = "windows")] -use crate::hypervisor::{PartitionState, WindowsInterruptHandle}; #[cfg(crashdump)] use crate::mem::memory_region::MemoryRegion; use crate::mem::mgr::SandboxMemoryManager; @@ -90,7 +85,7 @@ impl HyperlightVm { #[cfg(not(gdb))] type VmType = Box; - let vm: VmType = match get_available_hypervisor() { + let mut vm: VmType = match get_available_hypervisor() { #[cfg(kvm)] Some(HypervisorType::Kvm) => Box::new(KvmVm::new().map_err(VmError::CreateVm)?), #[cfg(mshv3)] @@ -106,35 +101,12 @@ impl HyperlightVm { .map_err(VmError::Register)?; #[cfg(any(kvm, mshv3))] - let interrupt_handle: Arc = Arc::new(LinuxInterruptHandle { - state: AtomicU8::new(0), - #[cfg(all( - target_arch = "x86_64", - target_vendor = "unknown", - target_os = "linux", - target_env = "musl" - ))] - tid: AtomicU64::new(unsafe { libc::pthread_self() as u64 }), - #[cfg(not(all( - target_arch = "x86_64", - target_vendor = "unknown", - target_os = "linux", - target_env = "musl" - )))] - tid: AtomicU64::new(unsafe { libc::pthread_self() }), - retry_delay: config.get_interrupt_retry_delay(), - sig_rt_min_offset: config.get_interrupt_vcpu_sigrtmin_offset(), - dropped: AtomicBool::new(false), - }); + let interrupt_handle: Arc = + Arc::new(LinuxInterruptHandle::new(config)); #[cfg(target_os = "windows")] - let interrupt_handle: Arc = Arc::new(WindowsInterruptHandle { - state: AtomicU8::new(0), - partition_state: std::sync::RwLock::new(PartitionState { - handle: vm.partition_handle(), - dropped: false, - }), - }); + let interrupt_handle: Arc = + Arc::new(WindowsInterruptHandle::new(vm.partition_handle())); let snapshot_slot = 0u32; let scratch_slot = 1u32; @@ -598,8 +570,6 @@ impl HyperlightVm { #[cfg(gdb)] pub(super) mod debug { - use hyperlight_common::mem::PAGE_SIZE; - use super::HyperlightVm; use crate::hypervisor::gdb::arch::{SW_BP, SW_BP_SIZE}; use crate::hypervisor::gdb::{ @@ -789,7 +759,7 @@ pub(super) mod debug { let read_len = std::cmp::min( data.len(), - (PAGE_SIZE - (gpa & (PAGE_SIZE - 1))).try_into().unwrap(), + page_size::get() - (gpa as usize & (page_size::get() - 1)), ); mem_access.read(&mut data[..read_len], gpa)?; @@ -817,7 +787,7 @@ pub(super) mod debug { let write_len = std::cmp::min( data.len(), - (PAGE_SIZE - (gpa & (PAGE_SIZE - 1))).try_into().unwrap(), + page_size::get() - (gpa as usize & (page_size::get() - 1)), ); // Use the memory access to write to guest memory diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index 732f08563..54ec16e8f 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -40,43 +40,122 @@ pub(crate) mod hyperlight_vm; use std::fmt::Debug; #[cfg(any(kvm, mshv3))] -use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}; -#[cfg(target_os = "windows")] +use std::sync::atomic::{AtomicBool, AtomicU64}; use std::sync::atomic::{AtomicU8, Ordering}; -#[cfg(any(kvm, mshv3))] +#[cfg(any(kvm, mshv3, hvf))] use std::time::Duration; -/// A trait for platform-specific interrupt handle implementation details -pub(crate) trait InterruptHandleImpl: InterruptHandle { - /// Set the thread ID for the vcpu thread - #[cfg(any(kvm, mshv3))] - fn set_tid(&self); +#[derive(Debug)] +pub(crate) struct InterruptHandleStateMachine(AtomicU8); +impl InterruptHandleStateMachine { + const RUNNING_BIT: u8 = 1 << 1; + const CANCEL_BIT: u8 = 1 << 0; + #[cfg(gdb)] + const DEBUG_INTERRUPT_BIT: u8 = 1 << 2; + + fn new() -> Self { + Self(AtomicU8::new(0)) + } /// Set the running state - fn set_running(&self); + pub(crate) fn set_running(&self) { + // Release ordering to ensure that the tid store (which uses Release) + // is visible to any thread that observes running=true via Acquire ordering. + // This prevents the interrupt thread from reading a stale tid value. + self.0.fetch_or(Self::RUNNING_BIT, Ordering::Release); + } /// Clear the running state - fn clear_running(&self); - - /// Mark the handle as dropped - fn set_dropped(&self); + pub(crate) fn clear_running(&self) { + // Release ordering to ensure all vcpu operations are visible before clearing running + self.0.fetch_and(!Self::RUNNING_BIT, Ordering::Release); + } /// Check if cancellation was requested - fn is_cancelled(&self) -> bool; + pub(crate) fn is_cancelled(&self) -> bool { + self.get_running_cancel_debug().1 + } + + /// Set the cancellation request flag + fn set_cancel(&self) { + // Release ordering ensures that any writes before kill() are visible to the vcpu thread + // when it checks is_cancelled() with Acquire ordering + self.0.fetch_or(Self::CANCEL_BIT, Ordering::Release); + } /// Clear the cancellation request flag - fn clear_cancel(&self); + fn clear_cancel(&self) { + // Release ordering to ensure that any operations from the previous run() + // are visible to other threads. While this is typically called by the vcpu thread + // at the start of run(), the VM itself can move between threads across guest calls. + self.0.fetch_and(!Self::CANCEL_BIT, Ordering::Release); + } /// Check if debug interrupt was requested (always returns false when gdb feature is disabled) - fn is_debug_interrupted(&self) -> bool; + pub(crate) fn is_debug_interrupted(&self) -> bool { + #[cfg(gdb)] + { + self.get_running_cancel_debug().2 + } + #[cfg(not(gdb))] + { + false + } + } + + /// Clear the debug interrupt request flag + #[cfg(gdb)] + fn set_debug_interrupt(&self) { + self.0 + .fetch_or(Self::DEBUG_INTERRUPT_BIT, Ordering::Release); + } - // Clear the debug interrupt request flag + /// Clear the debug interrupt request flag #[cfg(gdb)] - fn clear_debug_interrupt(&self); + fn clear_debug_interrupt(&self) { + self.0 + .fetch_and(!Self::DEBUG_INTERRUPT_BIT, Ordering::Release); + } + + /// Get the running, cancel and debug flags atomically. + fn get_running_cancel_debug(&self) -> (bool, bool, bool) { + let state = self.0.load(Ordering::Acquire); + let running = state & Self::RUNNING_BIT != 0; + let cancel = state & Self::CANCEL_BIT != 0; + #[cfg(gdb)] + let debug = state & Self::DEBUG_INTERRUPT_BIT != 0; + #[cfg(not(gdb))] + let debug = false; + (running, cancel, debug) + } +} + +/// A trait for platform-specific interrupt handle implementation details +pub(crate) trait InterruptHandleImpl: InterruptHandle { + /// Set the thread ID for the vcpu thread + #[cfg(any(kvm, mshv3))] + fn set_tid(&self); + + /// Set the currently-executing vcpu id + #[cfg(hvf)] + fn set_vcpu(&self, vcpu: hv_vcpu_t); + + /// Mark the handle as dropped + fn set_dropped(&self); +} + +pub(crate) trait InterruptHandleInternal { + /// Local access the shared state, which does not perform any + /// operations other than updating the state machine + fn state(&self) -> &InterruptHandleStateMachine; + /// Trigger the actual kill-like operation without + /// modifying the state + fn common_kill(&self) -> bool; } /// A trait for handling interrupts to a sandbox's vcpu -pub trait InterruptHandle: Send + Sync + Debug { +#[allow(private_bounds)] +pub trait InterruptHandle: Send + Sync + Debug + InterruptHandleInternal { /// Interrupt the corresponding sandbox from running. /// /// - If this is called while the the sandbox currently executing a guest function call, it will interrupt the sandbox and return `true`. @@ -84,7 +163,10 @@ pub trait InterruptHandle: Send + Sync + Debug { /// /// # Note /// This function will block for the duration of the time it takes for the vcpu thread to be interrupted. - fn kill(&self) -> bool; + fn kill(&self) -> bool { + self.state().set_cancel(); + self.common_kill() + } /// Used by a debugger to interrupt the corresponding sandbox from running. /// @@ -96,25 +178,71 @@ pub trait InterruptHandle: Send + Sync + Debug { /// # Note /// This function will block for the duration of the time it takes for the vcpu thread to be interrupted. #[cfg(gdb)] - fn kill_from_debugger(&self) -> bool; + fn kill_from_debugger(&self) -> bool { + self.state().set_debug_interrupt(); + self.common_kill() + } /// Returns true if the corresponding sandbox has been dropped fn dropped(&self) -> bool; } +#[cfg(any(kvm, mshv3, hvf))] +#[derive(Debug)] +pub(super) struct RetryingInterruptHandle { + retry_delay: Duration, + inner: T, +} + +#[cfg(any(kvm, mshv3, hvf))] +impl InterruptHandleImpl for RetryingInterruptHandle { + #[cfg(any(kvm, mshv3))] + fn set_tid(&self) { + self.inner.set_tid(); + } + + #[cfg(hvf)] + fn set_vcpu(&self, vcpu: hv_vcpu_t) { + self.inner.set_vcpu(vcpu); + } + + fn set_dropped(&self) { + self.inner.set_dropped(); + } +} +#[cfg(any(kvm, mshv3, hvf))] +impl InterruptHandle for RetryingInterruptHandle { + fn dropped(&self) -> bool { + self.inner.dropped() + } +} +#[cfg(any(kvm, mshv3, hvf))] +impl InterruptHandleInternal for RetryingInterruptHandle { + fn state(&self) -> &InterruptHandleStateMachine { + self.inner.state() + } + fn common_kill(&self) -> bool { + let mut succeeded = false; + loop { + let (running, cancel, debug) = self.state().get_running_cancel_debug(); + // Check if we should continue sending signals + // Exit if not running OR if neither cancel nor debug_interrupt is set + let should_continue = running && (cancel || debug); + if !should_continue { + break; + } + tracing::info!("Trying to kill vcpu thread..."); + succeeded |= self.inner.common_kill(); + std::thread::sleep(self.retry_delay); + } + succeeded + } +} + #[cfg(any(kvm, mshv3))] #[derive(Debug)] -pub(super) struct LinuxInterruptHandle { - /// Atomic value packing vcpu execution state. - /// - /// Bit layout: - /// - Bit 2: DEBUG_INTERRUPT_BIT - set when debugger interrupt is requested - /// - Bit 1: RUNNING_BIT - set when vcpu is actively running - /// - Bit 0: CANCEL_BIT - set when cancellation has been requested - /// - /// CANCEL_BIT persists across vcpu exits/re-entries within a single `VirtualCPU::run()` call - /// (e.g., during host function calls), but is cleared at the start of each new `VirtualCPU::run()` call. - state: AtomicU8, +pub(super) struct LinuxInterruptHandleState { + state: InterruptHandleStateMachine, /// Thread ID where the vcpu is running. /// @@ -125,67 +253,42 @@ pub(super) struct LinuxInterruptHandle { /// Whether the corresponding VM has been dropped. dropped: AtomicBool, - /// Delay between retry attempts when sending signals to interrupt the vcpu. - retry_delay: Duration, - /// Offset from SIGRTMIN for the signal used to interrupt the vcpu thread. sig_rt_min_offset: u8, } +#[cfg(any(kvm, mshv3))] +pub(super) type LinuxInterruptHandle = RetryingInterruptHandle; #[cfg(any(kvm, mshv3))] impl LinuxInterruptHandle { - const RUNNING_BIT: u8 = 1 << 1; - const CANCEL_BIT: u8 = 1 << 0; - #[cfg(gdb)] - const DEBUG_INTERRUPT_BIT: u8 = 1 << 2; - - /// Get the running, cancel and debug flags atomically. - /// - /// # Memory Ordering - /// Uses `Acquire` ordering to synchronize with the `Release` in `set_running()` and `kill()`. - /// This ensures that when we observe running=true, we also see the correct `tid` value. - fn get_running_cancel_debug(&self) -> (bool, bool, bool) { - let state = self.state.load(Ordering::Acquire); - let running = state & Self::RUNNING_BIT != 0; - let cancel = state & Self::CANCEL_BIT != 0; - #[cfg(gdb)] - let debug = state & Self::DEBUG_INTERRUPT_BIT != 0; - #[cfg(not(gdb))] - let debug = false; - (running, cancel, debug) - } - - fn send_signal(&self) -> bool { - let signal_number = libc::SIGRTMIN() + self.sig_rt_min_offset as libc::c_int; - let mut sent_signal = false; - - loop { - let (running, cancel, debug) = self.get_running_cancel_debug(); - - // Check if we should continue sending signals - // Exit if not running OR if neither cancel nor debug_interrupt is set - let should_continue = running && (cancel || debug); - - if !should_continue { - break; - } - - tracing::info!("Sending signal to kill vcpu thread..."); - sent_signal = true; - // Acquire ordering to synchronize with the Release store in set_tid() - // This ensures we see the correct tid value for the currently running vcpu - unsafe { - libc::pthread_kill(self.tid.load(Ordering::Acquire) as _, signal_number); - } - std::thread::sleep(self.retry_delay); + fn new(config: &crate::sandbox::SandboxConfiguration) -> Self { + RetryingInterruptHandle { + retry_delay: config.get_interrupt_retry_delay(), + inner: LinuxInterruptHandleState { + state: InterruptHandleStateMachine::new(), + #[cfg(all( + target_arch = "x86_64", + target_vendor = "unknown", + target_os = "linux", + target_env = "musl" + ))] + tid: AtomicU64::new(unsafe { libc::pthread_self() as u64 }), + #[cfg(not(all( + target_arch = "x86_64", + target_vendor = "unknown", + target_os = "linux", + target_env = "musl" + )))] + tid: AtomicU64::new(unsafe { libc::pthread_self() as u64 }), + sig_rt_min_offset: config.get_interrupt_vcpu_sigrtmin_offset(), + dropped: AtomicBool::new(false), + }, } - - sent_signal } } #[cfg(any(kvm, mshv3))] -impl InterruptHandleImpl for LinuxInterruptHandle { +impl InterruptHandleImpl for LinuxInterruptHandleState { fn set_tid(&self) { // Release ordering to synchronize with the Acquire load of `running` in send_signal() // This ensures that when send_signal() observes RUNNING_BIT=true (via Acquire), @@ -194,48 +297,6 @@ impl InterruptHandleImpl for LinuxInterruptHandle { .store(unsafe { libc::pthread_self() as u64 }, Ordering::Release); } - fn set_running(&self) { - // Release ordering to ensure that the tid store (which uses Release) - // is visible to any thread that observes running=true via Acquire ordering. - // This prevents the interrupt thread from reading a stale tid value. - self.state.fetch_or(Self::RUNNING_BIT, Ordering::Release); - } - - fn is_cancelled(&self) -> bool { - // Acquire ordering to synchronize with the Release in kill() - // This ensures we see the cancel flag set by the interrupt thread - self.state.load(Ordering::Acquire) & Self::CANCEL_BIT != 0 - } - - fn clear_cancel(&self) { - // Release ordering to ensure that any operations from the previous run() - // are visible to other threads. While this is typically called by the vcpu thread - // at the start of run(), the VM itself can move between threads across guest calls. - self.state.fetch_and(!Self::CANCEL_BIT, Ordering::Release); - } - - fn clear_running(&self) { - // Release ordering to ensure all vcpu operations are visible before clearing running - self.state.fetch_and(!Self::RUNNING_BIT, Ordering::Release); - } - - fn is_debug_interrupted(&self) -> bool { - #[cfg(gdb)] - { - self.state.load(Ordering::Acquire) & Self::DEBUG_INTERRUPT_BIT != 0 - } - #[cfg(not(gdb))] - { - false - } - } - - #[cfg(gdb)] - fn clear_debug_interrupt(&self) { - self.state - .fetch_and(!Self::DEBUG_INTERRUPT_BIT, Ordering::Release); - } - fn set_dropped(&self) { // Release ordering to ensure all VM cleanup operations are visible // to any thread that checks dropped() via Acquire @@ -244,22 +305,7 @@ impl InterruptHandleImpl for LinuxInterruptHandle { } #[cfg(any(kvm, mshv3))] -impl InterruptHandle for LinuxInterruptHandle { - fn kill(&self) -> bool { - // Release ordering ensures that any writes before kill() are visible to the vcpu thread - // when it checks is_cancelled() with Acquire ordering - self.state.fetch_or(Self::CANCEL_BIT, Ordering::Release); - - // Send signals to interrupt the vcpu if it's currently running - self.send_signal() - } - - #[cfg(gdb)] - fn kill_from_debugger(&self) -> bool { - self.state - .fetch_or(Self::DEBUG_INTERRUPT_BIT, Ordering::Release); - self.send_signal() - } +impl InterruptHandle for LinuxInterruptHandleState { fn dropped(&self) -> bool { // Acquire ordering to synchronize with the Release in set_dropped() // This ensures we see all VM cleanup operations that happened before drop @@ -267,101 +313,63 @@ impl InterruptHandle for LinuxInterruptHandle { } } -#[cfg(target_os = "windows")] -#[derive(Debug)] -pub(super) struct WindowsInterruptHandle { - /// Atomic value packing vcpu execution state. - /// - /// Bit layout: - /// - Bit 2: DEBUG_INTERRUPT_BIT - set when debugger interrupt is requested - /// - Bit 1: RUNNING_BIT - set when vcpu is actively running - /// - Bit 0: CANCEL_BIT - set when cancellation has been requested - /// - /// `WHvCancelRunVirtualProcessor()` will return Ok even if the vcpu is not running, - /// which is why we need the RUNNING_BIT. - /// - /// CANCEL_BIT persists across vcpu exits/re-entries within a single `VirtualCPU::run()` call - /// (e.g., during host function calls), but is cleared at the start of each new `VirtualCPU::run()` call. - state: AtomicU8, +#[cfg(any(kvm, mshv3))] +impl InterruptHandleInternal for LinuxInterruptHandleState { + fn state(&self) -> &InterruptHandleStateMachine { + &self.state + } + fn common_kill(&self) -> bool { + let signal_number = libc::SIGRTMIN() + self.sig_rt_min_offset as libc::c_int; + unsafe { + libc::pthread_kill(self.tid.load(Ordering::Acquire) as _, signal_number); + } + true + } +} +#[cfg(any(target_os = "windows", hvf))] +#[derive(Debug)] +/// An interrupt handle that captures the pattern that requests to +/// cancel need to be mutually exclusive with partition destruction +#[allow(private_bounds)] +pub(super) struct SynchronousInterruptHandle { + state: InterruptHandleStateMachine, /// RwLock protecting the partition handle and dropped state. /// - /// This lock prevents a race condition between `kill()` calling `WHvCancelRunVirtualProcessor` - /// and `WhpVm::drop()` calling `WHvDeletePartition`. These two Windows Hypervisor Platform APIs - /// must not execute concurrently - if `WHvDeletePartition` frees the partition while - /// `WHvCancelRunVirtualProcessor` is still accessing it, the result is a use-after-free - /// causing STATUS_ACCESS_VIOLATION or STATUS_HEAP_CORRUPTION. + /// Fox example, on Windows, this lock prevents a race condition + /// between `kill()` calling `WHvCancelRunVirtualProcessor` and + /// `WhpVm::drop()` calling `WHvDeletePartition`. These two + /// Windows Hypervisor Platform APIs must not execute + /// concurrently---if `WHvDeletePartition` frees the partition + /// while `WHvCancelRunVirtualProcessor` is still accessing it, + /// the result is a use-after-free causing STATUS_ACCESS_VIOLATION + /// or STATUS_HEAP_CORRUPTION. /// /// The synchronization works as follows: /// - `kill()` takes a read lock before calling `WHvCancelRunVirtualProcessor` /// - `set_dropped()` takes a write lock, which blocks until all in-flight `kill()` calls complete, /// then sets `dropped = true`. This is called from `HyperlightVm::drop()` before `WhpVm::drop()` /// runs, ensuring no `kill()` is accessing the partition when `WHvDeletePartition` is called. - partition_state: std::sync::RwLock, + dropped_state: std::sync::RwLock<(bool, T)>, } - -/// State protected by the RwLock in `WindowsInterruptHandle`. -/// -/// Contains a copy of the partition handle from `WhpVm` (not an owning reference). -/// The RwLock and `dropped` flag ensure this handle is never used after `WhpVm` -/// deletes the partition. -#[cfg(target_os = "windows")] -#[derive(Debug)] -pub(super) struct PartitionState { - /// Copy of partition handle from `WhpVm`. Only valid while `dropped` is false. - pub(super) handle: windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE, - /// Set true before partition deletion; prevents further use of `handle`. - pub(super) dropped: bool, -} - -#[cfg(target_os = "windows")] -impl WindowsInterruptHandle { - const RUNNING_BIT: u8 = 1 << 1; - const CANCEL_BIT: u8 = 1 << 0; - #[cfg(gdb)] - const DEBUG_INTERRUPT_BIT: u8 = 1 << 2; +#[cfg(any(target_os = "windows", hvf))] +trait SynchronousInterruptState: Debug + Send + Sync { + /// The inside-the-lock part of the common part of both kill() + /// and kill_from_debugger() + fn actually_cancel(&self) -> bool; + + #[cfg(hvf)] + fn set_vcpu(&mut self, vcpu: hv_vcpu_t); } -#[cfg(target_os = "windows")] -impl InterruptHandleImpl for WindowsInterruptHandle { - fn set_running(&self) { - // Release ordering to ensure prior memory operations are visible when another thread observes running=true - self.state.fetch_or(Self::RUNNING_BIT, Ordering::Release); - } - - fn is_cancelled(&self) -> bool { - // Acquire ordering to synchronize with the Release in kill() - // This ensures we see the CANCEL_BIT set by the interrupt thread - self.state.load(Ordering::Acquire) & Self::CANCEL_BIT != 0 - } - - fn clear_cancel(&self) { - // Release ordering to ensure that any operations from the previous run() - // are visible to other threads. While this is typically called by the vcpu thread - // at the start of run(), the VM itself can move between threads across guest calls. - self.state.fetch_and(!Self::CANCEL_BIT, Ordering::Release); - } - - fn clear_running(&self) { - // Release ordering to ensure all vcpu operations are visible before clearing running - self.state.fetch_and(!Self::RUNNING_BIT, Ordering::Release); - } - - fn is_debug_interrupted(&self) -> bool { - #[cfg(gdb)] - { - self.state.load(Ordering::Acquire) & Self::DEBUG_INTERRUPT_BIT != 0 - } - #[cfg(not(gdb))] - { - false - } - } - - #[cfg(gdb)] - fn clear_debug_interrupt(&self) { - self.state - .fetch_and(!Self::DEBUG_INTERRUPT_BIT, Ordering::Release); +#[cfg(any(target_os = "windows", hvf))] +impl InterruptHandleImpl for SynchronousInterruptHandle { + #[cfg(hvf)] + fn set_vcpu(&self, vcpu: hv_vcpu_t) { + let Ok(mut guard) = self.dropped_state.write() else { + return; + }; + guard.1.set_vcpu(vcpu); } fn set_dropped(&self) { @@ -371,9 +379,9 @@ impl InterruptHandleImpl for WindowsInterruptHandle { // 3. Set dropped=true so no future kill() calls will use the handle // After this returns, no WHvCancelRunVirtualProcessor calls are in progress // or will ever be made, so WHvDeletePartition can safely be called. - match self.partition_state.write() { + match self.dropped_state.write() { Ok(mut guard) => { - guard.dropped = true; + guard.0 = true; } Err(e) => { tracing::error!("Failed to acquire partition_state write lock: {}", e); @@ -382,26 +390,33 @@ impl InterruptHandleImpl for WindowsInterruptHandle { } } -#[cfg(target_os = "windows")] -impl InterruptHandle for WindowsInterruptHandle { - fn kill(&self) -> bool { - use windows::Win32::System::Hypervisor::WHvCancelRunVirtualProcessor; - - // Release ordering ensures that any writes before kill() are visible to the vcpu thread - // when it checks is_cancelled() with Acquire ordering - self.state.fetch_or(Self::CANCEL_BIT, Ordering::Release); - - // Acquire ordering to synchronize with the Release in set_running() - // This ensures we see the running state set by the vcpu thread - let state = self.state.load(Ordering::Acquire); - if state & Self::RUNNING_BIT == 0 { +#[cfg(any(target_os = "windows", hvf))] +impl InterruptHandle for SynchronousInterruptHandle { + fn dropped(&self) -> bool { + // Take read lock to check dropped state consistently + match self.dropped_state.read() { + Ok(guard) => guard.0, + Err(e) => { + tracing::error!("Failed to acquire partition_state read lock: {}", e); + true // Assume dropped if we can't acquire lock + } + } + } +} +#[cfg(any(target_os = "windows", hvf))] +impl InterruptHandleInternal for SynchronousInterruptHandle { + fn state(&self) -> &InterruptHandleStateMachine { + &self.state + } + fn common_kill(&self) -> bool { + if !self.state.get_running_cancel_debug().0 { return false; } // Take read lock to prevent race with WHvDeletePartition in set_dropped(). // Multiple kill() calls can proceed concurrently (read locks don't block each other), // but set_dropped() will wait for all kill() calls to complete before proceeding. - let guard = match self.partition_state.read() { + let guard = match self.dropped_state.read() { Ok(guard) => guard, Err(e) => { tracing::error!("Failed to acquire partition_state read lock: {}", e); @@ -409,49 +424,67 @@ impl InterruptHandle for WindowsInterruptHandle { } }; - if guard.dropped { + if guard.0 { return false; } - unsafe { WHvCancelRunVirtualProcessor(guard.handle, 0, 0).is_ok() } + guard.1.actually_cancel() } - #[cfg(gdb)] - fn kill_from_debugger(&self) -> bool { - use windows::Win32::System::Hypervisor::WHvCancelRunVirtualProcessor; - - self.state - .fetch_or(Self::DEBUG_INTERRUPT_BIT, Ordering::Release); +} - // Acquire ordering to synchronize with the Release in set_running() - let state = self.state.load(Ordering::Acquire); - if state & Self::RUNNING_BIT == 0 { - return false; +#[cfg(target_os = "windows")] +use windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE; +#[cfg(target_os = "windows")] +pub(super) type WindowsInterruptHandle = SynchronousInterruptHandle; +#[cfg(target_os = "windows")] +impl WindowsInterruptHandle { + fn new(hdl: WHV_PARTITION_HANDLE) -> Self { + SynchronousInterruptHandle { + state: InterruptHandleStateMachine::new(), + dropped_state: std::sync::RwLock::new((false, hdl)), } + } +} +#[cfg(target_os = "windows")] +impl SynchronousInterruptState for WHV_PARTITION_HANDLE { + fn actually_cancel(&self) -> bool { + use windows::Win32::System::Hypervisor::WHvCancelRunVirtualProcessor; + unsafe { WHvCancelRunVirtualProcessor(*self, 0, 0).is_ok() } + } +} - // Take read lock to prevent race with WHvDeletePartition in set_dropped() - let guard = match self.partition_state.read() { - Ok(guard) => guard, - Err(e) => { - tracing::error!("Failed to acquire partition_state read lock: {}", e); - return false; - } - }; - - if guard.dropped { +#[cfg(hvf)] +use crate::hypervisor::virtual_machine::hvf::bindings::hv_vcpu_t; +#[cfg(hvf)] +pub(super) type HvfInterruptHandle = + RetryingInterruptHandle>>; +#[cfg(hvf)] +impl SynchronousInterruptState for Option { + fn actually_cancel(&self) -> bool { + use crate::hypervisor::virtual_machine::hvf::bindings::{HV_SUCCESS, hv_vcpus_exit}; + let Some(vcpu) = self else { return false; + }; + unsafe { + // bindgen automatically uses *mut, but actually this will + // not be written to. + hv_vcpus_exit(&raw const *vcpu as *mut hv_vcpu_t, 1).0.0.0 == HV_SUCCESS } - - unsafe { WHvCancelRunVirtualProcessor(guard.handle, 0, 0).is_ok() } } - fn dropped(&self) -> bool { - // Take read lock to check dropped state consistently - match self.partition_state.read() { - Ok(guard) => guard.dropped, - Err(e) => { - tracing::error!("Failed to acquire partition_state read lock: {}", e); - true // Assume dropped if we can't acquire lock - } + fn set_vcpu(&mut self, vcpu: hv_vcpu_t) { + *self = Some(vcpu); + } +} +#[cfg(hvf)] +impl HvfInterruptHandle { + pub(super) fn new(retry_delay: Duration) -> Self { + RetryingInterruptHandle { + retry_delay, + inner: SynchronousInterruptHandle { + state: InterruptHandleStateMachine::new(), + dropped_state: std::sync::RwLock::new((false, None)), + }, } } } diff --git a/src/hyperlight_host/src/hypervisor/surrogate_process.rs b/src/hyperlight_host/src/hypervisor/surrogate_process.rs index 481aa44e8..0189c6530 100644 --- a/src/hyperlight_host/src/hypervisor/surrogate_process.rs +++ b/src/hyperlight_host/src/hypervisor/surrogate_process.rs @@ -18,7 +18,6 @@ use core::ffi::c_void; use std::collections::HashMap; use std::collections::hash_map::Entry; -use hyperlight_common::mem::PAGE_SIZE_USIZE; use tracing::{Span, instrument}; use windows::Win32::Foundation::HANDLE; use windows::Win32::System::Memory::{ @@ -150,7 +149,7 @@ impl SurrogateProcess { VirtualProtectEx( self.process_handle.into(), first_guard_page_start, - PAGE_SIZE_USIZE, + page_size::get(), PAGE_NOACCESS, &mut unused_out_old_prot_flags, ) @@ -161,12 +160,12 @@ impl SurrogateProcess { // the last page of the raw_size is the guard page let last_guard_page_start = - unsafe { first_guard_page_start.add(host_size - PAGE_SIZE_USIZE) }; + unsafe { first_guard_page_start.add(host_size - page_size::get()) }; if let Err(e) = unsafe { VirtualProtectEx( self.process_handle.into(), last_guard_page_start, - PAGE_SIZE_USIZE, + page_size::get(), PAGE_NOACCESS, &mut unused_out_old_prot_flags, ) diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/hvf/bindings.h b/src/hyperlight_host/src/hypervisor/virtual_machine/hvf/bindings.h new file mode 100644 index 000000000..7d0dd17d8 --- /dev/null +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/hvf/bindings.h @@ -0,0 +1,22 @@ +/* +Copyright 2025 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include + +hv_return_t +hv_vcpu_get_simd_fp_reg_rsabi(hv_vcpu_t vcpu, hv_simd_fp_reg_t reg, char *val); +hv_return_t +hv_vcpu_set_simd_fp_reg_rsabi(hv_vcpu_t vcpu, hv_simd_fp_reg_t reg, const char *val); diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/hvf/fp_abi.c b/src/hyperlight_host/src/hypervisor/virtual_machine/hvf/fp_abi.c new file mode 100644 index 000000000..a6fa3edac --- /dev/null +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/hvf/fp_abi.c @@ -0,0 +1,36 @@ +/* +Copyright 2025 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include "bindings.h" + +hv_return_t +hv_vcpu_get_simd_fp_reg_rsabi(hv_vcpu_t vcpu, hv_simd_fp_reg_t reg, char *val) { + hv_simd_fp_uchar16_t simd = {0}; + hv_return_t ret = hv_vcpu_get_simd_fp_reg(vcpu, reg, &simd); + for (int i = 0; i < 16; ++i) { + val[i] = simd[i]; + } + return ret; +} + +hv_return_t +hv_vcpu_set_simd_fp_reg_rsabi(hv_vcpu_t vcpu, hv_simd_fp_reg_t reg, const char *val) { + hv_simd_fp_uchar16_t simd; + for (int i = 0; i < 16; ++i) { + simd[i] = val[i]; + } + return hv_vcpu_set_simd_fp_reg(vcpu, reg, simd); +} diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/hvf/mod.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/hvf/mod.rs new file mode 100644 index 000000000..b83cdeaef --- /dev/null +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/hvf/mod.rs @@ -0,0 +1,938 @@ +/* +Copyright 2025 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +//! # Bridging Hyperlight's assumptions with Hypervisor.framework +//! +//! Hypervisor.framework has some constraints that run counter to the +//! flexibility provided by the usual Hyperlight API. In particular, +//! hvf assumes that there is only <= 1 VM per process, and <= 1 VCPU +//! per thread. +//! +//! ## Supporting more than one Sandbox per process +//! +//! This is not yet implemented, but we plan to support >1 sandbox per +//! process by using nested virtualisation on platforms where it is +//! available, making each sandbox a nested guest VM. This does, +//! unfortunately, of course have some performance implications. +//! +//! ## Supporting [`core::marker::Send`] on Sandboxes +//! +//! The Hyperlight public API constraints sandboxes (and, by +//! extension, hypervisor API implementations) to implement +//! [`core::marker::Send`]. Other hypervisors have one vCPU but allow +//! the vCPU handle to be mgirated across threads, although this comes +//! with severe performance impact in some cases (e.g. KVM). +//! +//! Unfortunately, this does not work on hvf, since cross-thread +//! access is prohibited (most `hv_vcpu_` functions note that "This +//! function must be called by the owning thread") rather than merely +//! unperformant. +//! +//! There are, largely, two approaches that we could use to work +//! around this. We could either: +//! +//! 1. Create a dedicated vcpu thread, and implement sandbox +//! operations as RPCs to that thread +//! 2. Create one vcpu per thread-from-which-a-Sandbox-is-used, and +//! reset that vcpu's state to match the correct sandbox whenever a +//! sandbox operation is called. +//! +//! A long time ago, Hyperlight briefly unconditionally used approach +//! (1) on all hypervisors, but it had unacceptable performance impact +//! on most of them. +//! +//! Although an implementation of (1) scoped just to hvf could make +//! sense, this module presently implements (2), on the rationale that +//! it ought to be /possible/ for a host application to get decent +//! performance out of (2) (if the host application exercises some +//! discipline and uses a 1:1 mapping between sandboxes and threads; +//! although there is some unavoidable overhead due to needing to sync +//! registers on every VM exit, unfortunately), but an implementation +//! based on (1) would have to create the thread up-front (not knowing +//! if the sandbox would in fact be Sent to another thread) and so +//! would impose unavoidable overhead on all consumers. +//! +//! Applications which wish to use multiple sandboxes per thread, or +//! multiple threads per sandbox, and to have decent performance on +//! Hypervisor.framework, should consider (and benchmark!) the +//! alternative architecture of keeping the sandbox itself on a single +//! thread and pass data/make RPCs to/from that thread. + +use core::cell::RefCell; +use core::ffi; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use hyperlight_common::outb::VmAction; +use parking_lot::{Mutex, MutexGuard}; + +use super::{ + CreateVmError, HvfSyncError, HypervisorError, MapMemoryError, RegisterError, ResetVcpuError, + RunVcpuError, UnmapMemoryError, VirtualMachine, VmExit, +}; +use crate::hypervisor::InterruptHandleImpl; +use crate::hypervisor::regs::{ + CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters, +}; +use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; + +#[allow( + dead_code, + non_snake_case, + non_upper_case_globals, + non_camel_case_types +)] // bindgen +pub(crate) mod bindings { + include!(concat!(env!("OUT_DIR"), "/hvf_bindings.rs")); + impl hv_return_t { + pub(super) fn is_success(&self) -> Result<(), super::HypervisorError> { + if self.0.0.0 == HV_SUCCESS { + Ok(()) + } else { + Err(super::HypervisorError::HvfError(*self)) + } + } + pub(super) fn unless_success( + &self, + f: impl Fn(super::HypervisorError) -> T, + ) -> Result<(), T> { + self.is_success().map_err(f) + } + } + impl core::fmt::Display for hv_return_t { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { + write!(f, "0x{:x}", self.0.0.0) + } + } + // the default base-10 signed printout is totally useless + impl core::fmt::Debug for hv_return_t { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { + write!(f, "0x{:x}", self.0.0.0) + } + } +} + +pub(super) fn is_hypervisor_present() -> bool { + let mut val: ffi::c_int = 0; + let mut len: usize = core::mem::size_of::(); + let ret = unsafe { + libc::sysctlbyname( + c"kern.hv_support".as_ptr(), + &raw mut val as *mut ffi::c_void, + &raw mut len, + std::ptr::null_mut(), + 0, + ) + }; + ret == 0 && val == 1 +} + +// Used to figure out when registers have been updated since the last +// time that the sandbox ran on this vcpu & need to be sync'd back. +#[derive(Clone, Copy, Default, Debug)] +struct EpochStamped { + value: T, + epoch: u64, +} + +#[derive(Clone, Copy, PartialEq, Debug)] +struct SandboxId(u64); + +struct HvfCpu { + id: bindings::hv_vcpu_t, + exit: *mut bindings::hv_vcpu_exit_t, + current_loaded: Option>, + + // reset_vcpu() needs to destroy the current vcpu before it can + // create the new one, but when it creates the new one and assigns + // it to the thread-local, the HvfCpu for the old one will be + // dropped. This lets the Drop implementation know that that has + // happened and avoid destroying the cpu a second time. + destroyed_in_reset_vcpu: bool, +} + +impl HvfCpu { + /// Must be called only once per thread. + fn new() -> Result { + use core::mem::MaybeUninit; + let mut vcpu: MaybeUninit = MaybeUninit::zeroed(); + let mut exit: *mut bindings::hv_vcpu_exit_t = core::ptr::null_mut(); + unsafe { + let config = bindings::hv_vcpu_config_create(); + bindings::hv_vcpu_create(vcpu.as_mut_ptr(), &raw mut exit, config) + } + .is_success()?; + Ok(Self { + id: unsafe { vcpu.assume_init() }, + exit, + current_loaded: None, + destroyed_in_reset_vcpu: false, + }) + } + + fn reset_vcpu(&mut self) -> Result<(), HypervisorError> { + // TODO: figure out if there is a more efficient way to clear + // all the state + unsafe { bindings::hv_vcpu_destroy(self.id) }.is_success()?; + self.destroyed_in_reset_vcpu = true; + *self = HvfCpu::new()?; + Ok(()) + } + + fn hv_vcpu_get_reg(&self, reg: bindings::hv_reg_t) -> Result { + let mut value: u64 = 0; + unsafe { bindings::hv_vcpu_get_reg(self.id, reg, &raw mut value) }.is_success()?; + Ok(value) + } + + fn hv_vcpu_set_reg( + &mut self, + reg: bindings::hv_reg_t, + value: u64, + ) -> Result<(), HypervisorError> { + unsafe { bindings::hv_vcpu_set_reg(self.id, reg, value) }.is_success() + } + + fn hv_vcpu_get_simd_fp_reg( + &self, + reg: bindings::hv_simd_fp_reg_t, + ) -> Result { + let mut value: [u8; 16] = [0; 16]; + unsafe { + bindings::hv_vcpu_get_simd_fp_reg_rsabi(self.id, reg, value.as_mut_ptr() as *mut i8) + } + .is_success()?; + Ok(u128::from_ne_bytes(value)) + } + + fn hv_vcpu_set_simd_fp_reg( + &mut self, + reg: bindings::hv_simd_fp_reg_t, + value: u128, + ) -> Result<(), HypervisorError> { + let bytes: [u8; 16] = value.to_ne_bytes(); + unsafe { + bindings::hv_vcpu_set_simd_fp_reg_rsabi(self.id, reg, bytes.as_ptr() as *const i8) + } + .is_success() + } + + fn hv_vcpu_get_sys_reg(&self, reg: bindings::hv_sys_reg_t) -> Result { + let mut value: u64 = 0; + unsafe { bindings::hv_vcpu_get_sys_reg(self.id, reg, &raw mut value) }.is_success()?; + Ok(value) + } + + fn hv_vcpu_set_sys_reg( + &mut self, + reg: bindings::hv_sys_reg_t, + value: u64, + ) -> Result<(), HypervisorError> { + unsafe { bindings::hv_vcpu_set_sys_reg(self.id, reg, value) }.is_success() + } + + fn set_vcpu_regs(&mut self, regs: &CommonRegisters) -> Result<(), HypervisorError> { + use bindings::{hv_reg_t, hv_sys_reg_t}; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X0, regs.x[0])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X1, regs.x[1])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X2, regs.x[2])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X3, regs.x[3])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X4, regs.x[4])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X5, regs.x[5])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X6, regs.x[6])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X7, regs.x[7])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X8, regs.x[8])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X9, regs.x[9])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X10, regs.x[10])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X11, regs.x[11])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X12, regs.x[12])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X13, regs.x[13])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X14, regs.x[14])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X15, regs.x[15])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X16, regs.x[16])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X17, regs.x[17])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X18, regs.x[18])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X19, regs.x[19])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X20, regs.x[20])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X21, regs.x[21])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X22, regs.x[22])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X23, regs.x[23])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X24, regs.x[24])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X25, regs.x[25])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X26, regs.x[26])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X27, regs.x[27])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X28, regs.x[28])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X29, regs.x[29])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_X30, regs.x[30])?; + // set SP_EL0 or SP_EL1 depending on SPSel + if regs.pstate & 0x1 == 0x1 { + self.hv_vcpu_set_sys_reg(hv_sys_reg_t::HV_SYS_REG_SP_EL1, regs.sp)?; + } else { + self.hv_vcpu_set_sys_reg(hv_sys_reg_t::HV_SYS_REG_SP_EL0, regs.sp)?; + } + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_PC, regs.pc)?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_CPSR, regs.pstate)?; + Ok(()) + } + + fn get_vcpu_regs(&self) -> Result { + use bindings::{hv_reg_t, hv_sys_reg_t}; + let pstate = self.hv_vcpu_get_reg(hv_reg_t::HV_REG_CPSR)?; + Ok(CommonRegisters { + x: [ + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X0)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X1)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X2)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X3)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X4)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X5)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X6)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X7)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X8)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X9)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X10)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X11)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X12)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X13)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X14)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X15)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X16)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X17)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X18)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X19)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X20)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X21)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X22)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X23)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X24)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X25)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X26)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X27)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X28)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X29)?, + self.hv_vcpu_get_reg(hv_reg_t::HV_REG_X30)?, + ], + sp: if pstate & 0x1 == 0x1 { + self.hv_vcpu_get_sys_reg(hv_sys_reg_t::HV_SYS_REG_SP_EL1)? + } else { + self.hv_vcpu_get_sys_reg(hv_sys_reg_t::HV_SYS_REG_SP_EL0)? + }, + pc: self.hv_vcpu_get_reg(hv_reg_t::HV_REG_PC)?, + pstate, + }) + } + + fn set_vcpu_fpregs(&mut self, regs: &CommonFpu) -> Result<(), HypervisorError> { + use bindings::{hv_reg_t, hv_simd_fp_reg_t}; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q0, regs.v[0])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q1, regs.v[1])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q2, regs.v[2])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q3, regs.v[3])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q4, regs.v[4])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q5, regs.v[5])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q6, regs.v[6])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q7, regs.v[7])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q8, regs.v[8])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q9, regs.v[9])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q10, regs.v[10])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q11, regs.v[11])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q12, regs.v[12])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q13, regs.v[13])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q14, regs.v[14])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q15, regs.v[15])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q16, regs.v[16])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q17, regs.v[17])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q18, regs.v[18])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q19, regs.v[19])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q20, regs.v[20])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q21, regs.v[21])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q22, regs.v[22])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q23, regs.v[23])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q24, regs.v[24])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q25, regs.v[25])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q26, regs.v[26])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q27, regs.v[27])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q28, regs.v[28])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q29, regs.v[29])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q30, regs.v[30])?; + self.hv_vcpu_set_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q31, regs.v[31])?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_FPSR, regs.fpsr as u64)?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_FPCR, regs.fpcr as u64)?; + Ok(()) + } + + fn get_vcpu_fpregs(&mut self) -> Result { + use bindings::{hv_reg_t, hv_simd_fp_reg_t}; + Ok(CommonFpu { + v: [ + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q0)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q1)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q2)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q3)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q4)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q5)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q6)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q7)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q8)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q9)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q10)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q11)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q12)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q13)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q14)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q15)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q16)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q17)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q18)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q19)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q20)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q21)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q22)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q23)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q24)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q25)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q26)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q27)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q28)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q29)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q30)?, + self.hv_vcpu_get_simd_fp_reg(hv_simd_fp_reg_t::HV_SIMD_FP_REG_Q31)?, + ], + fpsr: self.hv_vcpu_get_reg(hv_reg_t::HV_REG_FPSR)? as u32, + fpcr: self.hv_vcpu_get_reg(hv_reg_t::HV_REG_FPCR)? as u32, + }) + } + + fn set_vcpu_sregs(&mut self, regs: &CommonSpecialRegisters) -> Result<(), HypervisorError> { + use bindings::hv_sys_reg_t; + self.hv_vcpu_set_sys_reg(hv_sys_reg_t::HV_SYS_REG_TTBR0_EL1, regs.ttbr0_el1)?; + self.hv_vcpu_set_sys_reg(hv_sys_reg_t::HV_SYS_REG_TCR_EL1, regs.tcr_el1)?; + self.hv_vcpu_set_sys_reg(hv_sys_reg_t::HV_SYS_REG_MAIR_EL1, regs.mair_el1)?; + self.hv_vcpu_set_sys_reg(hv_sys_reg_t::HV_SYS_REG_SCTLR_EL1, regs.sctlr_el1)?; + self.hv_vcpu_set_sys_reg(hv_sys_reg_t::HV_SYS_REG_CPACR_EL1, regs.cpacr_el1)?; + self.hv_vcpu_set_sys_reg(hv_sys_reg_t::HV_SYS_REG_VBAR_EL1, regs.vbar_el1)?; + self.hv_vcpu_set_sys_reg(hv_sys_reg_t::HV_SYS_REG_SP_EL1, regs.sp_el1)?; + Ok(()) + } + + fn get_vcpu_sregs(&self) -> Result { + use bindings::hv_sys_reg_t; + Ok(CommonSpecialRegisters { + ttbr0_el1: self.hv_vcpu_get_sys_reg(hv_sys_reg_t::HV_SYS_REG_TTBR0_EL1)?, + tcr_el1: self.hv_vcpu_get_sys_reg(hv_sys_reg_t::HV_SYS_REG_TCR_EL1)?, + mair_el1: self.hv_vcpu_get_sys_reg(hv_sys_reg_t::HV_SYS_REG_MAIR_EL1)?, + sctlr_el1: self.hv_vcpu_get_sys_reg(hv_sys_reg_t::HV_SYS_REG_SCTLR_EL1)?, + cpacr_el1: self.hv_vcpu_get_sys_reg(hv_sys_reg_t::HV_SYS_REG_CPACR_EL1)?, + vbar_el1: self.hv_vcpu_get_sys_reg(hv_sys_reg_t::HV_SYS_REG_VBAR_EL1)?, + sp_el1: self.hv_vcpu_get_sys_reg(hv_sys_reg_t::HV_SYS_REG_SP_EL1)?, + }) + } + + fn sync_state_from( + &mut self, + sandbox_id: EpochStamped, + regs: &EpochStamped, + sregs: &EpochStamped, + fpregs: &EpochStamped, + ) -> Result { + let (regs_dirty, fpregs_dirty, sregs_dirty) = match self.current_loaded { + None => (true, true, true), + Some(s) => { + if sandbox_id.value != s.value || sandbox_id.epoch > s.epoch { + self.reset_vcpu().map_err(HvfSyncError::ResetVcpu)?; + (true, true, true) + } else { + ( + regs.epoch > s.epoch, + fpregs.epoch > s.epoch, + sregs.epoch > s.epoch, + ) + } + } + }; + if regs_dirty { + self.set_vcpu_regs(®s.value) + .map_err(RegisterError::SetRegs)?; + } + if fpregs_dirty { + self.set_vcpu_fpregs(&fpregs.value) + .map_err(RegisterError::SetFpu)?; + } + if sregs_dirty { + self.set_vcpu_sregs(&sregs.value) + .map_err(RegisterError::SetSregs)?; + } + let sync_epoch = core::cmp::max(sandbox_id.epoch, core::cmp::max(regs.epoch, sregs.epoch)); + self.current_loaded = Some(EpochStamped { + value: sandbox_id.value, + epoch: sync_epoch, + }); + Ok(sync_epoch) + } + + fn sync_state_from_vm(&mut self, vm: &mut HvfVm) -> Result<(), HvfSyncError> { + let epoch = self.sync_state_from(vm.id, &vm.regs, &vm.sregs, &vm.fpu)?; + // update epoch eagerly, since at this point the vcpu + // epoch has been updated to the max, so if we returned early (e.g. due + // to an error in the next call(s)) it would be possible + // to miss updates + vm.id.epoch = epoch; + vm.regs.epoch = epoch; + vm.fpu.epoch = epoch; + vm.sregs.epoch = epoch; + Ok(()) + } + + fn sync_state_to_vm(&mut self, vm: &mut HvfVm) -> Result<(), HvfSyncError> { + let Some(EpochStamped { + value: sandbox_id, + epoch, + }) = self.current_loaded + else { + // sync_state_to_vm is always used just after + // sync_state_from_vm, so this should be impossible + debug_assert!(false); + return Err(HvfSyncError::SyncInvariant( + "Missing loaded sandbox".to_string(), + )); + }; + if sandbox_id != vm.id.value { + // sync_state_to_vm is always used just after + // sync_state_from_vm, so this should be impossible + debug_assert!(false); + return Err(HvfSyncError::SyncInvariant( + "Wrong loaded sandbox".to_string(), + )); + } + vm.id.epoch = epoch; + vm.regs = EpochStamped { + value: self.get_vcpu_regs().map_err(RegisterError::GetRegs)?, + epoch, + }; + vm.fpu = EpochStamped { + value: self.get_vcpu_fpregs().map_err(RegisterError::GetFpu)?, + epoch, + }; + vm.sregs = EpochStamped { + value: self.get_vcpu_sregs().map_err(RegisterError::GetSregs)?, + epoch, + }; + Ok(()) + } + + fn advance_pc(&mut self) -> Result<(), HypervisorError> { + use bindings::hv_reg_t; + let old_pc = self.hv_vcpu_get_reg(hv_reg_t::HV_REG_PC)?; + self.hv_vcpu_set_reg(hv_reg_t::HV_REG_PC, old_pc + 4)?; + Ok(()) + } + + fn run(&mut self) -> Result { + let ret = unsafe { bindings::hv_vcpu_run(self.id) }; + if let Some(EpochStamped { ref mut epoch, .. }) = self.current_loaded { + *epoch += 1; + } + ret.is_success()?; + let exit = unsafe { self.exit.read() }; + + use bindings::hv_exit_reason_t; + let hl_exit = match exit.reason { + hv_exit_reason_t::HV_EXIT_REASON_CANCELED => VmExit::Cancelled(), + hv_exit_reason_t::HV_EXIT_REASON_EXCEPTION => { + #[inline(always)] + fn bits(x: u64) -> u64 { + (x & ((1 << (HIGH_BIT + 1)) - 1)) >> LOW_BIT + } + + let esr = exit.exception.syndrome.0; + let ipa = exit.exception.physical_address.0; + + let unknown_exit = VmExit::Unknown(format!( + "Unknown HVF vcpu exit ESR_EL2: {:16x} IPA {:16x}", + esr, ipa, + )); + + const ESR_EL2_EC_DATA_ABORT_LOWER_EL: u64 = 0b100100; + let ec = bits::<31, 26>(esr); + let isv = bits::<24, 24>(esr); + if ec == ESR_EL2_EC_DATA_ABORT_LOWER_EL && isv == 0b1 { + let is_translation_fault = + bits::<5, 2>(esr) == 0b0001 || bits::<5, 0>(esr) == 0b101011; + let is_permission_fault = bits::<5, 2>(esr) == 0b0011; + if is_translation_fault || is_permission_fault { + // For MMIO exits, always resume after the + // faulting instruction to match kvm behaviour + self.advance_pc()?; + + let wnr = bits::<6, 6>(esr); + if wnr == 1 { + let io_page_gpa = + const { hyperlight_common::layout::io_page().unwrap().0 }; + if ipa >= io_page_gpa + && let off = (ipa - io_page_gpa) as usize + && off < hyperlight_common::vmem::PAGE_SIZE + { + let port = off / core::mem::size_of::(); + if port == VmAction::Halt as usize { + VmExit::Halt() + } else { + let srt = bits::<20, 16>(esr); + let data = self.get_vcpu_regs()?.x[srt as usize]; + VmExit::IoOut(port as u16, data.to_ne_bytes().to_vec()) + } + } else { + VmExit::MmioWrite(ipa) + } + } else { + VmExit::MmioRead(ipa) + } + } else { + unknown_exit + } + } else { + unknown_exit + } + } + reason => VmExit::Unknown(format!("Unknown HVF vcpu exit reason: {}", reason.0)), + }; + Ok(hl_exit) + } +} + +impl Drop for HvfCpu { + fn drop(&mut self) { + unsafe { + if !self.destroyed_in_reset_vcpu { + bindings::hv_vcpu_destroy(self.id); + }; + } + } +} + +std::thread_local! { + static HVF_VCPU: RefCell> = const { RefCell::new(None) }; +} + +#[derive(Debug)] +pub(crate) struct HvfVm { + id: EpochStamped, + regs: EpochStamped, + fpu: EpochStamped, + sregs: EpochStamped, + memory_space: MemorySpace, + interrupt_handle: Arc, +} + +static HV_VM_CREATED: AtomicBool = AtomicBool::new(false); + +impl HvfVm { + pub(crate) fn new( + interrupt_handle: Arc, + ) -> Result { + static NEXT_AVAILABLE_ID: AtomicU64 = AtomicU64::new(0); + // If the vm for this process has not yet been created, create it + if !HV_VM_CREATED.swap(true, Ordering::Relaxed) { + let cfg = unsafe { bindings::hv_vm_config_create() }; + unsafe { bindings::hv_vm_create(cfg) }.unless_success(|e| { + HV_VM_CREATED.store(false, Ordering::Relaxed); + CreateVmError::CreateVmFd(e) + })? + } + Ok(Self { + id: EpochStamped { + value: SandboxId(NEXT_AVAILABLE_ID.fetch_add(1, Ordering::Relaxed)), + epoch: 0, + }, + regs: Default::default(), + fpu: Default::default(), + sregs: Default::default(), + memory_space: MemorySpace::new(), + interrupt_handle, + }) + } +} + +impl From for bindings::hv_memory_flags_t { + fn from(mrf: MemoryRegionFlags) -> bindings::hv_memory_flags_t { + let mut flags: bindings::hv_memory_flags_t = 0; + if mrf.contains(MemoryRegionFlags::READ) { + flags |= bindings::HV_MEMORY_READ as bindings::hv_memory_flags_t; + } + if mrf.contains(MemoryRegionFlags::WRITE) { + flags |= bindings::HV_MEMORY_WRITE as bindings::hv_memory_flags_t; + } + if mrf.contains(MemoryRegionFlags::EXECUTE) { + flags |= bindings::HV_MEMORY_EXEC as bindings::hv_memory_flags_t; + } + flags + } +} + +struct LoadedMemorySpace { + space_id: Option, + mappings: Vec>, +} +impl LoadedMemorySpace { + const fn new() -> Self { + Self { + space_id: None, + mappings: Vec::new(), + } + } + fn do_map(region: &MemoryRegion) -> Result<(), HypervisorError> { + unsafe { + bindings::hv_vm_map( + region.host_region.start as *mut core::ffi::c_void, + bindings::hv_ipa_t(region.guest_region.start as u64), + region.guest_region.end - region.guest_region.start, + region.flags.into(), + ) + } + .is_success() + } + fn do_unmap(region: &MemoryRegion) -> Result<(), HypervisorError> { + unsafe { + bindings::hv_vm_unmap( + bindings::hv_ipa_t(region.guest_region.start as u64), + region.guest_region.end - region.guest_region.start, + ) + } + .is_success() + } + fn update_mapping( + &mut self, + slot: usize, + region: Option, + ) -> Result<(), HypervisorError> { + if self.mappings.len() <= slot { + self.mappings.resize(slot + 1, None); + } + if let Some(ref old_region) = self.mappings[slot] { + Self::do_unmap(old_region)?; + } + if let Some(ref new_region) = region { + Self::do_map(new_region)?; + } + self.mappings[slot] = region; + Ok(()) + } + #[allow(clippy::collapsible_if, clippy::manual_flatten)] + fn sync_space(&mut self, space: &MemorySpace) -> Result<(), HypervisorError> { + if self.space_id == Some(space.id) { + return Ok(()); + } + self.space_id = Some(space.id); + for mapping in &self.mappings { + if let Some(region) = mapping { + Self::do_unmap(region)? + } + } + self.mappings = space.mappings.clone(); + for mapping in &self.mappings { + if let Some(region) = mapping { + Self::do_map(region)? + } + } + Ok(()) + } +} +static CURRENT_LOADED_MEMORY_SPACE: Mutex = Mutex::new(LoadedMemorySpace::new()); +#[derive(Debug)] +struct MemorySpace { + id: u64, + mappings: Vec>, +} +struct MemorySpaceInstalledGuard<'a> { + _loaded_mutex_guard: MutexGuard<'a, LoadedMemorySpace>, +} +enum MemorySpaceInstallError { + Sync(HypervisorError), + Timeout, +} +impl MemorySpace { + fn new() -> Self { + static NEXT_AVAILABLE_ID: AtomicU64 = AtomicU64::new(0); + Self { + id: NEXT_AVAILABLE_ID.fetch_add(1, Ordering::Relaxed), + mappings: Vec::new(), + } + } + + /// This function is used both as a performance optimisation e(when + /// this space is not being swapped out) and in order to allow + /// [`unmap_memory`] to guarantee that the region is no longer in + /// use in the kernel when it returns. + fn opportunistically_sync_slot(&mut self, slot: usize) -> Result<(), HypervisorError> { + // It suffices to use try_lock() here: if try_lock fails, + // then the lock is currently being held, but it must be held + // by a different memory space (since we have an &mut self + // reference) + if let Some(mut current) = CURRENT_LOADED_MEMORY_SPACE.try_lock() + && current.space_id == Some(self.id) + { + current.update_mapping(slot, self.mappings[slot].clone())?; + } + Ok(()) + } + + fn map_memory(&mut self, region: (u32, &MemoryRegion)) -> Result<(), HypervisorError> { + let slot = region.0 as usize; + if self.mappings.len() <= slot { + self.mappings.resize(slot + 1, None); + } + self.mappings[slot] = Some(region.1.clone()); + self.opportunistically_sync_slot(slot) + } + + fn unmap_memory(&mut self, region: (u32, &MemoryRegion)) -> Result<(), HypervisorError> { + let slot = region.0 as usize; + if self.mappings.len() <= slot { + self.mappings.resize(slot + 1, None); + } + self.mappings[slot] = None; + self.opportunistically_sync_slot(slot) + } + + fn install_in_cpu( + &mut self, + _cpu: &mut HvfCpu, + ) -> Result, MemorySpaceInstallError> { + let mut guard = CURRENT_LOADED_MEMORY_SPACE + .try_lock_for(Duration::from_millis(50)) + .ok_or(MemorySpaceInstallError::Timeout)?; + guard + .sync_space(self) + .map_err(MemorySpaceInstallError::Sync)?; + Ok(MemorySpaceInstalledGuard { + _loaded_mutex_guard: guard, + }) + } +} + +impl VirtualMachine for HvfVm { + unsafe fn map_memory( + &mut self, + region: (u32, &MemoryRegion), + ) -> std::result::Result<(), MapMemoryError> { + self.memory_space + .map_memory(region) + .map_err(MapMemoryError::Hypervisor) + } + + fn unmap_memory( + &mut self, + region: (u32, &MemoryRegion), + ) -> std::result::Result<(), UnmapMemoryError> { + self.memory_space + .unmap_memory(region) + .map_err(UnmapMemoryError::Hypervisor) + } + + fn run_vcpu( + &mut self, + #[cfg(feature = "trace_guest")] tc: &mut SandboxTraceContext, + ) -> std::result::Result { + HVF_VCPU.with_borrow_mut(|vcpu| { + // Sadly, Option::get_or_try_insert_with() is not yet stable :( + let vcpu = match vcpu { + None => vcpu.insert( + HvfCpu::new() + .map_err(HvfSyncError::CreateVcpu) + .map_err(RunVcpuError::HvfSync)?, + ), + Some(v) => v, + }; + self.interrupt_handle.set_vcpu(vcpu.id); + vcpu.sync_state_from_vm(self) + .map_err(RunVcpuError::HvfSync)?; + // TODO: replace unwrap() + let space_installed_guard = loop { + match self.memory_space.install_in_cpu(vcpu) { + Ok(guard) => break guard, + Err(MemorySpaceInstallError::Sync(e)) => { + return Err(RunVcpuError::HvfSync(HvfSyncError::MemorySpace(e))); + } + Err(MemorySpaceInstallError::Timeout) => { + let (_, cancel, debug) = + self.interrupt_handle.state().get_running_cancel_debug(); + if cancel || debug { + return Ok(VmExit::Cancelled()); + } else { + continue; + } + } + } + }; + let exit = vcpu.run().map_err(RunVcpuError::Unknown)?; + drop(space_installed_guard); + vcpu.sync_state_to_vm(self).map_err(RunVcpuError::HvfSync)?; + Ok(exit) + }) + } + + fn regs(&self) -> std::result::Result { + Ok(self.regs.value) + } + + fn set_regs(&mut self, regs: &CommonRegisters) -> std::result::Result<(), RegisterError> { + self.regs.value = *regs; + self.regs.epoch += 1; + Ok(()) + } + + fn fpu(&self) -> std::result::Result { + Ok(self.fpu.value) + } + + fn set_fpu(&mut self, fpu: &CommonFpu) -> std::result::Result<(), RegisterError> { + self.fpu.value = *fpu; + self.fpu.epoch += 1; + Ok(()) + } + + fn sregs(&self) -> std::result::Result { + Ok(self.sregs.value) + } + + fn set_sregs( + &mut self, + sregs: &CommonSpecialRegisters, + ) -> std::result::Result<(), RegisterError> { + self.sregs.value = *sregs; + self.sregs.epoch += 1; + Ok(()) + } + + fn debug_regs(&self) -> std::result::Result { + todo!() + } + + fn set_debug_regs(&self, _drs: &CommonDebugRegs) -> std::result::Result<(), RegisterError> { + todo!() + } + + #[cfg(target_arch = "aarch64")] + fn can_reset_vcpu(&self) -> bool { + true + } + + #[cfg(target_arch = "aarch64")] + fn reset_vcpu(&mut self) -> std::result::Result<(), ResetVcpuError> { + self.id.epoch += 1; + Ok(()) + } +} diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/aarch64.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/aarch64.rs index 07c6db953..162e4f678 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/aarch64.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/aarch64.rs @@ -286,7 +286,7 @@ impl VirtualMachine for KvmVm { }) } - fn set_regs(&self, regs: &CommonRegisters) -> std::result::Result<(), RegisterError> { + fn set_regs(&mut self, regs: &CommonRegisters) -> std::result::Result<(), RegisterError> { use crate::hypervisor::regs::kvm_reg::{PC, PSTATE, SP, X}; for (i, xi) in X.iter().enumerate() { xi.set(RegisterError::SetSregs, &self.vcpu_fd, regs.x[i])?; @@ -311,7 +311,7 @@ impl VirtualMachine for KvmVm { }) } - fn set_fpu(&self, fpu: &CommonFpu) -> Result<(), RegisterError> { + fn set_fpu(&mut self, fpu: &CommonFpu) -> Result<(), RegisterError> { use crate::hypervisor::regs::kvm_reg::{FPCR, FPSR, V}; for (i, vi) in V.iter().enumerate() { vi.set(RegisterError::SetFpu, &self.vcpu_fd, fpu.v[i])?; @@ -336,7 +336,7 @@ impl VirtualMachine for KvmVm { }) } - fn set_sregs(&self, sregs: &CommonSpecialRegisters) -> Result<(), RegisterError> { + fn set_sregs(&mut self, sregs: &CommonSpecialRegisters) -> Result<(), RegisterError> { use crate::hypervisor::regs::kvm_reg::{ CPACR_EL1, MAIR_EL1, SCTLR_EL1, SP_EL1, TCR_EL1, TTBR0_EL1, VBAR_EL1, }; diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs index 3dc8ec87a..a63a5e67a 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs @@ -343,7 +343,7 @@ impl VirtualMachine for KvmVm { Ok((&kvm_regs).into()) } - fn set_regs(&self, regs: &CommonRegisters) -> std::result::Result<(), RegisterError> { + fn set_regs(&mut self, regs: &CommonRegisters) -> std::result::Result<(), RegisterError> { let kvm_regs: kvm_regs = regs.into(); self.vcpu_fd .set_regs(&kvm_regs) @@ -361,7 +361,7 @@ impl VirtualMachine for KvmVm { Ok((&kvm_fpu).into()) } - fn set_fpu(&self, fpu: &CommonFpu) -> std::result::Result<(), RegisterError> { + fn set_fpu(&mut self, fpu: &CommonFpu) -> std::result::Result<(), RegisterError> { let kvm_fpu: kvm_fpu = fpu.into(); // Note: On KVM this ignores MXCSR. // See https://github.com/torvalds/linux/blob/d358e5254674b70f34c847715ca509e46eb81e6f/arch/x86/kvm/x86.c#L12554-L12599 @@ -379,7 +379,10 @@ impl VirtualMachine for KvmVm { Ok((&kvm_sregs).into()) } - fn set_sregs(&self, sregs: &CommonSpecialRegisters) -> std::result::Result<(), RegisterError> { + fn set_sregs( + &mut self, + sregs: &CommonSpecialRegisters, + ) -> std::result::Result<(), RegisterError> { let kvm_sregs: kvm_sregs = sregs.into(); self.vcpu_fd .set_sregs(&kvm_sregs) diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs index dac344711..f794f82a1 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs @@ -28,6 +28,9 @@ use crate::mem::memory_region::MemoryRegion; #[cfg(feature = "trace_guest")] use crate::sandbox::trace::TraceContext as SandboxTraceContext; +/// Hypervisor.framework functionality (MacOS) +#[cfg(hvf)] +pub(crate) mod hvf; /// KVM (Kernel-based Virtual Machine) functionality (linux) #[cfg(kvm)] pub(crate) mod kvm; @@ -77,6 +80,12 @@ pub fn get_available_hypervisor() -> &'static Option { } else { None } + } else if #[cfg(hvf)] { + if hvf::is_hypervisor_present() { + Some(HypervisorType::Hvf) + } else { + None + } } else { None } @@ -102,6 +111,9 @@ pub(crate) enum HypervisorType { #[cfg(target_os = "windows")] Whp, + + #[cfg(hvf)] + Hvf, } /// Minimum XSAVE buffer size: 512 bytes legacy region + 64 bytes header. @@ -122,6 +134,7 @@ compile_error!( ); /// The various reasons a VM's vCPU can exit +#[cfg_attr(target_os = "macos", allow(unused))] pub(crate) enum VmExit { /// The vCPU has exited due to a debug event (usually breakpoint) #[cfg(gdb)] @@ -205,6 +218,9 @@ pub enum RunVcpuError { #[cfg(target_arch = "aarch64")] #[error("Flush MMIO pending state failed: {0}")] FlushMmioPending(String), + #[cfg(hvf)] + #[error("HVF sync error: {0}")] + HvfSync(HvfSyncError), #[error("Unknown error: {0}")] Unknown(HypervisorError), } @@ -305,6 +321,25 @@ pub enum HypervisorError { #[cfg(target_os = "windows")] #[error("Windows error: {0}")] WindowsError(#[from] windows_result::Error), + #[cfg(hvf)] + #[error("HVF error: {0}")] + HvfError(hvf::bindings::hv_return_t), +} + +/// HVF-specific error synchronising vcpu state +#[cfg(hvf)] +#[derive(Debug, Clone, thiserror::Error)] +pub enum HvfSyncError { + #[error("Error creating VCPU: {0}")] + CreateVcpu(HypervisorError), + #[error("Error resetting VCPU: {0}")] + ResetVcpu(HypervisorError), + #[error("Error reading/writing registers: {0}")] + Register(#[from] RegisterError), + #[error("Error updating memory space: {0}")] + MemorySpace(HypervisorError), + #[error("Invariant violation: vcpu in unexpected sync state: {0}")] + SyncInvariant(String), } /// Trait for single-vCPU VMs. Provides a common interface for basic VM operations. @@ -341,17 +376,20 @@ pub(crate) trait VirtualMachine: Debug + Send { #[allow(dead_code)] fn regs(&self) -> std::result::Result; /// Set regs - fn set_regs(&self, regs: &CommonRegisters) -> std::result::Result<(), RegisterError>; + fn set_regs(&mut self, regs: &CommonRegisters) -> std::result::Result<(), RegisterError>; /// Get fpu regs #[allow(dead_code)] fn fpu(&self) -> std::result::Result; /// Set fpu regs - fn set_fpu(&self, fpu: &CommonFpu) -> std::result::Result<(), RegisterError>; + fn set_fpu(&mut self, fpu: &CommonFpu) -> std::result::Result<(), RegisterError>; /// Get special regs #[allow(dead_code)] fn sregs(&self) -> std::result::Result; /// Set special regs - fn set_sregs(&self, sregs: &CommonSpecialRegisters) -> std::result::Result<(), RegisterError>; + fn set_sregs( + &mut self, + sregs: &CommonSpecialRegisters, + ) -> std::result::Result<(), RegisterError>; /// Get the debug registers of the vCPU #[allow(dead_code)] fn debug_regs(&self) -> std::result::Result; diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs index 1fd50d29a..dbd3c885b 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs @@ -376,7 +376,7 @@ impl VirtualMachine for MshvVm { Ok((&mshv_regs).into()) } - fn set_regs(&self, regs: &CommonRegisters) -> std::result::Result<(), RegisterError> { + fn set_regs(&mut self, regs: &CommonRegisters) -> std::result::Result<(), RegisterError> { let mshv_regs: StandardRegisters = regs.into(); self.vcpu_fd .set_regs(&mshv_regs) @@ -392,7 +392,7 @@ impl VirtualMachine for MshvVm { Ok((&mshv_fpu).into()) } - fn set_fpu(&self, fpu: &CommonFpu) -> std::result::Result<(), RegisterError> { + fn set_fpu(&mut self, fpu: &CommonFpu) -> std::result::Result<(), RegisterError> { let mshv_fpu: FloatingPointUnit = fpu.into(); self.vcpu_fd .set_fpu(&mshv_fpu) @@ -408,7 +408,10 @@ impl VirtualMachine for MshvVm { Ok((&mshv_sregs).into()) } - fn set_sregs(&self, sregs: &CommonSpecialRegisters) -> std::result::Result<(), RegisterError> { + fn set_sregs( + &mut self, + sregs: &CommonSpecialRegisters, + ) -> std::result::Result<(), RegisterError> { let mshv_sregs: SpecialRegisters = sregs.into(); self.vcpu_fd .set_sregs(&mshv_sregs) diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs index 6de2b29f1..bb37e6c20 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs @@ -564,7 +564,7 @@ impl VirtualMachine for WhpVm { }) } - fn set_regs(&self, regs: &CommonRegisters) -> std::result::Result<(), RegisterError> { + fn set_regs(&mut self, regs: &CommonRegisters) -> std::result::Result<(), RegisterError> { let whp_regs: [(WHV_REGISTER_NAME, Align16); WHP_REGS_NAMES_LEN] = regs.into(); self.set_registers(&whp_regs) @@ -601,7 +601,7 @@ impl VirtualMachine for WhpVm { }) } - fn set_fpu(&self, fpu: &CommonFpu) -> std::result::Result<(), RegisterError> { + fn set_fpu(&mut self, fpu: &CommonFpu) -> std::result::Result<(), RegisterError> { let whp_fpu: [(WHV_REGISTER_NAME, Align16); WHP_FPU_NAMES_LEN] = fpu.into(); self.set_registers(&whp_fpu) @@ -638,7 +638,10 @@ impl VirtualMachine for WhpVm { }) } - fn set_sregs(&self, sregs: &CommonSpecialRegisters) -> std::result::Result<(), RegisterError> { + fn set_sregs( + &mut self, + sregs: &CommonSpecialRegisters, + ) -> std::result::Result<(), RegisterError> { let whp_regs: [(WHV_REGISTER_NAME, Align16); WHP_SREGS_NAMES_LEN] = sregs.into(); diff --git a/src/hyperlight_host/src/mem/layout.rs b/src/hyperlight_host/src/mem/layout.rs index 6422b9b11..3bc615cb4 100644 --- a/src/hyperlight_host/src/mem/layout.rs +++ b/src/hyperlight_host/src/mem/layout.rs @@ -63,7 +63,8 @@ limitations under the License. use std::fmt::Debug; use std::mem::size_of; -use hyperlight_common::mem::{HyperlightPEB, PAGE_SIZE_USIZE}; +use hyperlight_common::mem::HyperlightPEB; +use hyperlight_common::vmem::PAGE_SIZE; use tracing::{Span, instrument}; use super::memory_region::MemoryRegionType::{Code, Heap, InitData, Peb}; @@ -359,7 +360,7 @@ impl SandboxMemoryLayout { pub(crate) const MAX_MEMORY_SIZE: usize = (16 * 1024 * 1024 * 1024) - Self::BASE_ADDRESS; // 16 GiB - BASE_ADDRESS /// The base address of the sandbox's memory. - pub(crate) const BASE_ADDRESS: usize = 0x1000; + pub(crate) const BASE_ADDRESS: usize = 0x4000; // the offset into a sandbox's input/output buffer where the stack starts pub(crate) const STACK_POINTER_SIZE_BYTES: u64 = 8; @@ -544,6 +545,17 @@ impl SandboxMemoryLayout { let expected_final_offset = TryInto::::try_into(self.get_memory_size()?)?; + // This function is primarily used to construct + // GuestMemoryRegions used to populate initial guest page + // tables. Therefore, the regions above are aligned based on + // the guest page size. However, the total final size of the + // region mapped into the guest needs to be aligned based on + // the host page size. Therefore, align both of these values + // to both page sizes before comparing them. + let final_offset = final_offset.next_multiple_of(page_size::get()); + let expected_final_offset = + expected_final_offset.next_multiple_of(hyperlight_common::vmem::PAGE_SIZE); + if final_offset != expected_final_offset { return Err(new_error!( "Final offset does not match expected Final offset expected: {}, actual: {}", @@ -654,7 +666,7 @@ impl SandboxMemoryLayout { impl SandboxMemoryLayout { /// Offset of the PEB struct within the snapshot region. pub(crate) fn peb_offset(&self) -> usize { - self.code_size.next_multiple_of(PAGE_SIZE_USIZE) + self.code_size.next_multiple_of(PAGE_SIZE) } /// Guest physical address of the PEB. @@ -664,12 +676,12 @@ impl SandboxMemoryLayout { /// Offset of the guest heap buffer within the snapshot region. pub(crate) fn guest_heap_buffer_offset(&self) -> usize { - (self.peb_offset() + size_of::()).next_multiple_of(PAGE_SIZE_USIZE) + (self.peb_offset() + size_of::()).next_multiple_of(PAGE_SIZE) } /// Offset of the init data section within the snapshot region. pub(crate) fn init_data_offset(&self) -> usize { - (self.guest_heap_buffer_offset() + self.heap_size).next_multiple_of(PAGE_SIZE_USIZE) + (self.guest_heap_buffer_offset() + self.heap_size).next_multiple_of(PAGE_SIZE) } /// The code offset is always 0. @@ -705,8 +717,7 @@ impl SandboxMemoryLayout { /// Offset from the beginning of the scratch region to the location /// where page tables are eagerly copied on restore. pub(crate) fn get_pt_base_scratch_offset(&self) -> usize { - (self.input_data_size + self.output_data_size) - .next_multiple_of(hyperlight_common::vmem::PAGE_SIZE) + (self.input_data_size + self.output_data_size).next_multiple_of(PAGE_SIZE) } /// Base GPA to which the page tables are eagerly copied on restore. @@ -731,12 +742,12 @@ impl SandboxMemoryLayout { pub(crate) fn get_memory_size(&self) -> Result { let total_memory = self.get_unaligned_memory_size(); - // Size should be a multiple of page size. - let remainder = total_memory % PAGE_SIZE_USIZE; - let multiples = total_memory / PAGE_SIZE_USIZE; + // Size should be a multiple of host page size. + let remainder = total_memory % page_size::get(); + let multiples = total_memory / page_size::get(); let size = match remainder { 0 => total_memory, - _ => (multiples + 1) * PAGE_SIZE_USIZE, + _ => (multiples + 1) * page_size::get(), }; if size > Self::MAX_MEMORY_SIZE { @@ -749,8 +760,6 @@ impl SandboxMemoryLayout { #[cfg(test)] mod tests { - use hyperlight_common::mem::PAGE_SIZE_USIZE; - use super::*; // helper func for testing @@ -761,11 +770,11 @@ mod tests { // PEB let peb_and_array = size_of::(); - expected_size += peb_and_array.next_multiple_of(PAGE_SIZE_USIZE); + expected_size += peb_and_array.next_multiple_of(PAGE_SIZE); - expected_size += layout.heap_size.next_multiple_of(PAGE_SIZE_USIZE); + expected_size += layout.heap_size.next_multiple_of(PAGE_SIZE); - expected_size + expected_size.next_multiple_of(page_size::get()) } #[test] @@ -805,8 +814,8 @@ mod tests { let cfg = SandboxConfiguration::default(); let a = SandboxMemoryLayout::new(cfg, 4096, 0, None).unwrap(); let mut b = a; - b.snapshot_size = a.snapshot_size + PAGE_SIZE_USIZE; - b.set_pt_size(PAGE_SIZE_USIZE).unwrap(); + b.snapshot_size = a.snapshot_size + PAGE_SIZE; + b.set_pt_size(PAGE_SIZE).unwrap(); assert!(a.is_compatible_with(&b)); assert!(b.is_compatible_with(&a)); } @@ -818,12 +827,12 @@ mod tests { // Each mutation must independently break compatibility. let mutators: &[fn(&mut SandboxMemoryLayout)] = &[ - |l| l.input_data_size += PAGE_SIZE_USIZE, - |l| l.output_data_size += PAGE_SIZE_USIZE, - |l| l.heap_size += PAGE_SIZE_USIZE, - |l| l.code_size += PAGE_SIZE_USIZE, - |l| l.init_data_size += PAGE_SIZE_USIZE, - |l| l.scratch_size += PAGE_SIZE_USIZE, + |l| l.input_data_size += PAGE_SIZE, + |l| l.output_data_size += PAGE_SIZE, + |l| l.heap_size += PAGE_SIZE, + |l| l.code_size += PAGE_SIZE, + |l| l.init_data_size += PAGE_SIZE, + |l| l.scratch_size += PAGE_SIZE, |l| { l.init_data_permissions = Some(MemoryRegionFlags::READ); }, @@ -882,7 +891,7 @@ mod tests { ); pin_eq!( hyperlight_common::layout::SCRATCH_TOP_GPA, - 0x0000_000f_ffff_efff + 0x0000_000f_ffff_bfff ); } @@ -906,7 +915,7 @@ mod tests { pin_eq!(layout.guest_code_offset(), 0); pin_eq!(layout.peb_offset(), 0x1000); - pin_eq!(layout.peb_address(), 0x2000); + pin_eq!(layout.peb_address(), 0x5000); pin_eq!(layout.guest_heap_buffer_offset(), 0x2000); pin_eq!(layout.init_data_offset(), 0x4000); pin_eq!(layout.get_memory_size().unwrap(), 0x4000); @@ -955,10 +964,13 @@ mod tests { pin_eq!(layout.guest_code_offset(), 0); pin_eq!(layout.peb_offset(), 0x3000); - pin_eq!(layout.peb_address(), 0x4000); + pin_eq!(layout.peb_address(), 0x7000); pin_eq!(layout.guest_heap_buffer_offset(), 0x4000); pin_eq!(layout.init_data_offset(), 0x9000); - pin_eq!(layout.get_memory_size().unwrap(), 0x9000); + pin_eq!( + layout.get_memory_size().unwrap(), + 0x9000_usize.next_multiple_of(page_size::get()) + ); pin_eq!(layout.get_scratch_size(), 0x20000); pin_eq!(layout.get_pt_size(), 0); diff --git a/src/hyperlight_host/src/mem/memory_region.rs b/src/hyperlight_host/src/mem/memory_region.rs index 5d839647a..690bfe806 100644 --- a/src/hyperlight_host/src/mem/memory_region.rs +++ b/src/hyperlight_host/src/mem/memory_region.rs @@ -17,9 +17,7 @@ limitations under the License. use std::ops::Range; use bitflags::bitflags; -#[cfg(mshv3)] -use hyperlight_common::mem::PAGE_SHIFT; -use hyperlight_common::mem::PAGE_SIZE_USIZE; +use hyperlight_common::vmem::PAGE_SIZE; #[cfg(kvm)] use kvm_bindings::{KVM_MEM_READONLY, kvm_userspace_memory_region}; #[cfg(mshv3)] @@ -388,7 +386,7 @@ impl MemoryRegionVecBuilder { flags: MemoryRegionFlags, region_type: MemoryRegionType, ) -> usize { - let aligned_size = (size + PAGE_SIZE_USIZE - 1) & !(PAGE_SIZE_USIZE - 1); + let aligned_size = (size + PAGE_SIZE - 1) & !(PAGE_SIZE - 1); self.push(aligned_size, flags, region_type) } @@ -403,7 +401,7 @@ impl MemoryRegionVecBuilder { impl From<&MemoryRegion> for mshv_user_mem_region { fn from(region: &MemoryRegion) -> Self { let size = (region.guest_region.end - region.guest_region.start) as u64; - let guest_pfn = region.guest_region.start as u64 >> PAGE_SHIFT; + let guest_pfn = (region.guest_region.start / page_size::get()) as u64; let userspace_addr = region.host_region.start as u64; let flags: u8 = region.flags.iter().fold(0, |acc, flag| { diff --git a/src/hyperlight_host/src/mem/mgr.rs b/src/hyperlight_host/src/mem/mgr.rs index 7c74fea91..bfac57ac3 100644 --- a/src/hyperlight_host/src/mem/mgr.rs +++ b/src/hyperlight_host/src/mem/mgr.rs @@ -572,7 +572,8 @@ impl SandboxMemoryManager { // immediately after the snapshot in the guest PA space. let snapshot_pt_end = self.shared_mem.mem_size(); let snapshot_pt_size = self.layout.get_pt_size(); - let snapshot_pt_start = snapshot_pt_end - snapshot_pt_size; + let snapshot_pt_start = + snapshot_pt_end - snapshot_pt_size.next_multiple_of(page_size::get()); self.scratch_mem.with_exclusivity(|scratch| { #[cfg(not(unshared_snapshot_mem))] let bytes = &self.shared_mem.as_slice()[snapshot_pt_start..snapshot_pt_end]; diff --git a/src/hyperlight_host/src/mem/shared_mem.rs b/src/hyperlight_host/src/mem/shared_mem.rs index 4b706cc41..511da98ea 100644 --- a/src/hyperlight_host/src/mem/shared_mem.rs +++ b/src/hyperlight_host/src/mem/shared_mem.rs @@ -18,12 +18,11 @@ use std::any::type_name; use std::ffi::c_void; use std::io::Error; use std::mem::{align_of, size_of}; -#[cfg(target_os = "linux")] +#[cfg(unix)] use std::ptr::null_mut; use std::sync::{Arc, RwLock}; use bytemuck::Pod; -use hyperlight_common::mem::PAGE_SIZE_USIZE; use tracing::{Span, instrument}; #[cfg(target_os = "windows")] use windows::Win32::Foundation::{CloseHandle, HANDLE, INVALID_HANDLE_VALUE}; @@ -178,14 +177,14 @@ impl HostMapping { } /// RAII guard for an `mmap` reservation. Calls `munmap` on drop. -#[cfg(target_os = "linux")] +#[cfg(unix)] #[derive(Debug)] struct Mmap { base: *mut c_void, len: usize, } -#[cfg(target_os = "linux")] +#[cfg(unix)] impl Drop for Mmap { fn drop(&mut self) { // SAFETY: `self.base` and `self.len` are exactly what was @@ -294,7 +293,7 @@ impl Placeholder { fn split_front(self, front_size: usize) -> Result<(Placeholder, Placeholder)> { debug_assert!(front_size > 0 && front_size < self.size); - debug_assert!(front_size.is_multiple_of(PAGE_SIZE_USIZE)); + debug_assert!(front_size.is_multiple_of(page_size::get())); // SAFETY: `self` owns the placeholder reservation at // `[self.addr, self.addr + self.size)`. `MEM_RELEASE | // MEM_PRESERVE_PLACEHOLDER` is the Win32 idiom for splitting @@ -401,7 +400,7 @@ pub trait SharedMemory { /// need to be marked as `unsafe` because doing anything with this /// pointer itself requires `unsafe`. fn base_addr(&self) -> usize { - self.region().ptr() as usize + PAGE_SIZE_USIZE + self.region().ptr() as usize + page_size::get() } /// Return the base address of the host mapping of this region as @@ -409,14 +408,14 @@ pub trait SharedMemory { /// not need to be marked as `unsafe` because doing anything with /// this pointer itself requires `unsafe`. fn base_ptr(&self) -> *mut u8 { - self.region().ptr().wrapping_add(PAGE_SIZE_USIZE) + self.region().ptr().wrapping_add(page_size::get()) } /// Return the length of usable memory contained in `self`. /// The returned size does not include the size of the surrounding /// guard pages. fn mem_size(&self) -> usize { - self.region().size() - 2 * PAGE_SIZE_USIZE + self.region().size() - 2 * page_size::get() } /// Return the raw base address of the host mapping, including the @@ -448,7 +447,7 @@ pub trait SharedMemory { from_handle: self.region().file_mapping_handle().into(), handle_base: self.region().ptr() as usize, handle_size: self.region().size(), - offset: PAGE_SIZE_USIZE, + offset: page_size::get(), } } } @@ -537,7 +536,7 @@ impl ExclusiveSharedMemory { /// size in bytes. The region will be surrounded by guard pages. /// /// Return `Err` if shared memory could not be allocated. - #[cfg(target_os = "linux")] + #[cfg(unix)] #[instrument(skip_all, parent = Span::current(), level= "Trace")] pub fn new(min_size_bytes: usize) -> Result { use libc::{ @@ -552,13 +551,13 @@ impl ExclusiveSharedMemory { } let total_size = min_size_bytes - .checked_add(2 * PAGE_SIZE_USIZE) // guard page around the memory + .checked_add(2 * page_size::get()) // guard page around the memory .ok_or_else(|| new_error!("Memory required for sandbox exceeded usize::MAX"))?; - if total_size % PAGE_SIZE_USIZE != 0 { + if total_size % page_size::get() != 0 { return Err(new_error!( "shared memory must be a multiple of {}", - PAGE_SIZE_USIZE + page_size::get() )); } @@ -600,7 +599,7 @@ impl ExclusiveSharedMemory { // protect the guard pages #[cfg(not(miri))] { - let res = unsafe { mprotect(mmap.base, PAGE_SIZE_USIZE, PROT_NONE) }; + let res = unsafe { mprotect(mmap.base, page_size::get(), PROT_NONE) }; if res != 0 { return Err(HyperlightError::MprotectFailed( Error::last_os_error().raw_os_error(), @@ -608,8 +607,8 @@ impl ExclusiveSharedMemory { } let res = unsafe { mprotect( - (mmap.base as *const u8).add(total_size - PAGE_SIZE_USIZE) as *mut c_void, - PAGE_SIZE_USIZE, + (mmap.base as *const u8).add(total_size - page_size::get()) as *mut c_void, + page_size::get(), PROT_NONE, ) }; @@ -646,13 +645,13 @@ impl ExclusiveSharedMemory { } let total_size = min_size_bytes - .checked_add(2 * PAGE_SIZE_USIZE) + .checked_add(2 * page_size::get()) .ok_or_else(|| new_error!("Memory required for sandbox exceeded {}", usize::MAX))?; - if total_size % PAGE_SIZE_USIZE != 0 { + if total_size % page_size::get() != 0 { return Err(new_error!( "shared memory must be a multiple of {}", - PAGE_SIZE_USIZE + page_size::get() )); } @@ -719,7 +718,7 @@ impl ExclusiveSharedMemory { if let Err(e) = unsafe { VirtualProtect( first_guard_page_start, - PAGE_SIZE_USIZE, + page_size::get(), PAGE_NOACCESS, &mut unused_out_old_prot_flags, ) @@ -727,11 +726,11 @@ impl ExclusiveSharedMemory { log_then_return!(WindowsAPIError(e.clone())); } - let last_guard_page_start = unsafe { view.addr.add(total_size - PAGE_SIZE_USIZE) }; + let last_guard_page_start = unsafe { view.addr.add(total_size - page_size::get()) }; if let Err(e) = unsafe { VirtualProtect( last_guard_page_start, - PAGE_SIZE_USIZE, + page_size::get(), PAGE_NOACCESS, &mut unused_out_old_prot_flags, ) @@ -1489,7 +1488,7 @@ unsafe impl Sync for ReadonlySharedMemory {} impl ReadonlySharedMemory { pub(crate) fn from_bytes(contents: &[u8], guest_mapped_size: usize) -> Result { - if guest_mapped_size == 0 || !guest_mapped_size.is_multiple_of(PAGE_SIZE_USIZE) { + if guest_mapped_size == 0 || !guest_mapped_size.is_multiple_of(page_size::get()) { return Err(new_error!( "guest_mapped_size {} must be a non-zero multiple of PAGE_SIZE", guest_mapped_size @@ -1502,7 +1501,8 @@ impl ReadonlySharedMemory { contents.len() )); } - let mut anon = ExclusiveSharedMemory::new(contents.len())?; + let mut anon = + ExclusiveSharedMemory::new(contents.len().next_multiple_of(page_size::get()))?; anon.copy_from_slice(contents, 0)?; Ok(ReadonlySharedMemory { region: anon.region, @@ -1535,7 +1535,7 @@ impl ReadonlySharedMemory { )); } - if !len.is_multiple_of(PAGE_SIZE_USIZE) { + if !len.is_multiple_of(page_size::get()) { return Err(new_error!( "file length {} must be a multiple of PAGE_SIZE", len @@ -1544,7 +1544,7 @@ impl ReadonlySharedMemory { if guest_mapped_size == 0 || guest_mapped_size > len - || !guest_mapped_size.is_multiple_of(PAGE_SIZE_USIZE) + || !guest_mapped_size.is_multiple_of(page_size::get()) { return Err(new_error!( "guest_mapped_size {} must be a non-zero multiple of PAGE_SIZE no greater than file length {}", @@ -1563,7 +1563,7 @@ impl ReadonlySharedMemory { /// Linux: reserve `[guard][blob][guard]` as one anonymous /// `PROT_NONE` mapping, then `MAP_FIXED` the file over the /// middle slot. - #[cfg(target_os = "linux")] + #[cfg(unix)] fn map_file(file: &std::fs::File, len: usize) -> Result> { use std::os::unix::io::AsRawFd; @@ -1574,7 +1574,7 @@ impl ReadonlySharedMemory { mmap, off_t, size_t, }; - let total_size = len.checked_add(2 * PAGE_SIZE_USIZE).ok_or_else(|| { + let total_size = len.checked_add(2 * page_size::get()).ok_or_else(|| { new_error!("Memory required for file-backed mapping exceeded usize::MAX") })?; @@ -1619,7 +1619,7 @@ impl ReadonlySharedMemory { let file_prot = PROT_READ; // SAFETY: `total_size = len + 2 * PAGE_SIZE_USIZE`, so // `base + PAGE_SIZE_USIZE` is in-bounds of the reservation. - let usable_ptr = unsafe { (base as *mut u8).add(PAGE_SIZE_USIZE) }; + let usable_ptr = unsafe { (base as *mut u8).add(page_size::get()) }; // SAFETY: `usable_ptr..usable_ptr + len` lies entirely within // the reservation owned by `reservation`. `MAP_FIXED` // replaces that sub-range in place; on failure the @@ -1655,7 +1655,7 @@ impl ReadonlySharedMemory { fn map_file(file: &std::fs::File, len: usize) -> Result> { use std::os::windows::io::AsRawHandle; - let total_size = len.checked_add(2 * PAGE_SIZE_USIZE).ok_or_else(|| { + let total_size = len.checked_add(2 * page_size::get()).ok_or_else(|| { new_error!("Memory required for file-backed mapping exceeded usize::MAX") })?; @@ -1668,7 +1668,7 @@ impl ReadonlySharedMemory { // 2. Split the placeholder into three adjacent slots. The // leading and trailing slots stay unmapped and act as // guard pages. The middle slot will receive the file view. - let (leading, middle, trailing) = whole.split_into_three(PAGE_SIZE_USIZE, len)?; + let (leading, middle, trailing) = whole.split_into_three(page_size::get(), len)?; // 3. Create a read-only file mapping section over the file. // SAFETY: `file_handle` is a valid file HANDLE borrowed from @@ -1757,7 +1757,7 @@ impl SharedMemory for ReadonlySharedMemory { from_handle: self.region().file_mapping_handle().into(), handle_base: self.region().ptr() as usize, handle_size: self.region().size(), - offset: PAGE_SIZE_USIZE, + offset: page_size::get(), }, WindowsMapping::FileBacked { .. } => super::memory_region::HostRegionBase { from_handle: self.region().file_mapping_handle().into(), @@ -1791,7 +1791,6 @@ impl PartialEq for ReadonlySharedMemory { #[cfg(test)] mod tests { - use hyperlight_common::mem::PAGE_SIZE_USIZE; #[cfg(not(miri))] use proptest::prelude::*; @@ -1804,7 +1803,7 @@ mod tests { #[test] fn fill() { - let mem_size: usize = 4096; + let mem_size: usize = page_size::get(); let eshm = ExclusiveSharedMemory::new(mem_size).unwrap(); let (mut hshm, _) = eshm.build(); @@ -1822,7 +1821,7 @@ mod tests { assert!(vec[2048..3072].iter().all(|&x| x == 3)); assert!(vec[3072..4096].iter().all(|&x| x == 4)); - hshm.fill(5, 0, 4096).unwrap(); + hshm.fill(5, 0, mem_size).unwrap(); let vec2 = hshm .with_exclusivity(|e| e.copy_all_to_vec().unwrap()) @@ -1837,7 +1836,7 @@ mod tests { /// would overflow `usize`. #[test] fn bounds_check_overflow() { - let mem_size: usize = 4096; + let mem_size: usize = page_size::get(); let mut eshm = ExclusiveSharedMemory::new(mem_size).unwrap(); // ExclusiveSharedMemory methods @@ -1863,7 +1862,7 @@ mod tests { #[test] fn copy_into_from() -> Result<()> { - let mem_size: usize = 4096; + let mem_size: usize = page_size::get(); let vec_len = 10; let eshm = ExclusiveSharedMemory::new(mem_size)?; let (hshm, _) = eshm.build(); @@ -1954,7 +1953,7 @@ mod tests { #[test] fn clone() { - let eshm = ExclusiveSharedMemory::new(PAGE_SIZE_USIZE).unwrap(); + let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap(); let (hshm1, _) = eshm.build(); let hshm2 = hshm1.clone(); @@ -1991,7 +1990,7 @@ mod tests { #[test] fn copy_all_to_vec() { let mut data = vec![b'a', b'b', b'c']; - data.resize(4096, 0); + data.resize(page_size::get(), 0); let mut eshm = ExclusiveSharedMemory::new(data.len()).unwrap(); eshm.copy_from_slice(data.as_slice(), 0).unwrap(); let ret_vec = eshm.copy_all_to_vec().unwrap(); @@ -2014,11 +2013,11 @@ mod tests { // where another test allocates memory at the same address between our // drop and the mapping check. Ensure UNIQUE_SIZE is not used by any // other test in the codebase to avoid this. - const UNIQUE_SIZE: usize = PAGE_SIZE_USIZE * 17; + let unique_size: usize = page_size::get() * 17; let pid = std::process::id(); - let eshm = ExclusiveSharedMemory::new(UNIQUE_SIZE).unwrap(); + let eshm = ExclusiveSharedMemory::new(unique_size).unwrap(); let (hshm1, gshm) = eshm.build(); let hshm2 = hshm1.clone(); @@ -2068,7 +2067,7 @@ mod tests { #[test] fn copy_with_various_alignments() { // Use a buffer large enough to test all alignment cases - let mem_size: usize = 4096; + let mem_size: usize = page_size::get(); let eshm = ExclusiveSharedMemory::new(mem_size).unwrap(); let (hshm, _) = eshm.build(); @@ -2095,7 +2094,7 @@ mod tests { /// Test copy operations with lengths smaller than chunk size (< 16 bytes) #[test] fn copy_small_lengths() { - let mem_size: usize = 4096; + let mem_size: usize = page_size::get(); let eshm = ExclusiveSharedMemory::new(mem_size).unwrap(); let (hshm, _) = eshm.build(); @@ -2114,7 +2113,7 @@ mod tests { /// Test copy operations with lengths that don't align to chunk boundaries #[test] fn copy_non_aligned_lengths() { - let mem_size: usize = 4096; + let mem_size: usize = page_size::get(); let eshm = ExclusiveSharedMemory::new(mem_size).unwrap(); let (hshm, _) = eshm.build(); @@ -2136,7 +2135,7 @@ mod tests { /// Test copy with exactly one chunk (16 bytes) #[test] fn copy_exact_chunk_size() { - let mem_size: usize = 4096; + let mem_size: usize = page_size::get(); let eshm = ExclusiveSharedMemory::new(mem_size).unwrap(); let (hshm, _) = eshm.build(); @@ -2153,7 +2152,7 @@ mod tests { /// Test fill with various alignment offsets #[test] fn fill_with_various_alignments() { - let mem_size: usize = 4096; + let mem_size: usize = page_size::get(); let eshm = ExclusiveSharedMemory::new(mem_size).unwrap(); let (mut hshm, _) = eshm.build(); @@ -2182,7 +2181,7 @@ mod tests { /// Test fill with lengths smaller than chunk size #[test] fn fill_small_lengths() { - let mem_size: usize = 4096; + let mem_size: usize = page_size::get(); let eshm = ExclusiveSharedMemory::new(mem_size).unwrap(); let (mut hshm, _) = eshm.build(); @@ -2206,7 +2205,7 @@ mod tests { /// Test fill with non-aligned lengths #[test] fn fill_non_aligned_lengths() { - let mem_size: usize = 4096; + let mem_size: usize = page_size::get(); let eshm = ExclusiveSharedMemory::new(mem_size).unwrap(); let (mut hshm, _) = eshm.build(); @@ -2232,7 +2231,7 @@ mod tests { /// Test edge cases: length 0 and length 1 #[test] fn copy_edge_cases() { - let mem_size: usize = 4096; + let mem_size: usize = page_size::get(); let eshm = ExclusiveSharedMemory::new(mem_size).unwrap(); let (hshm, _) = eshm.build(); @@ -2254,7 +2253,7 @@ mod tests { /// Test combined: unaligned start + non-aligned length #[test] fn copy_unaligned_start_and_length() { - let mem_size: usize = 4096; + let mem_size: usize = page_size::get(); let eshm = ExclusiveSharedMemory::new(mem_size).unwrap(); let (hshm, _) = eshm.build(); @@ -2296,7 +2295,7 @@ mod tests { #[test] fn normal_push_pop_roundtrip() { - let mem_size = 4096; + let mem_size = page_size::get(); let mut hshm = make_buffer(mem_size); // Size-prefixed flatbuffer-like payload: [size: u32 LE][payload] @@ -2312,7 +2311,7 @@ mod tests { #[test] fn malicious_flatbuffer_size_prefix() { - let mem_size = 4096; + let mem_size = page_size::get(); let mut hshm = make_buffer(mem_size); let payload = b"small"; @@ -2335,7 +2334,7 @@ mod tests { #[test] fn malicious_element_offset_too_small() { - let mem_size = 4096; + let mem_size = page_size::get(); let mut hshm = make_buffer(mem_size); let payload = b"test"; @@ -2360,7 +2359,7 @@ mod tests { #[test] fn malicious_element_offset_past_stack_pointer() { - let mem_size = 4096; + let mem_size = page_size::get(); let mut hshm = make_buffer(mem_size); let payload = b"test"; @@ -2385,7 +2384,7 @@ mod tests { #[test] fn malicious_flatbuffer_size_off_by_one() { - let mem_size = 4096; + let mem_size = page_size::get(); let mut hshm = make_buffer(mem_size); let payload = b"abcd"; @@ -2410,7 +2409,7 @@ mod tests { /// `stack_pointer_rel - last_element_offset_rel - 8`. #[test] fn back_pointer_near_stack_pointer_underflow() { - let mem_size = 4096; + let mem_size = page_size::get(); let mut hshm = make_buffer(mem_size); let payload = b"test"; @@ -2436,7 +2435,7 @@ mod tests { /// Size prefix of 0xFFFF_FFFD causes u32 overflow: 0xFFFF_FFFD + 4 wraps. #[test] fn size_prefix_u32_overflow() { - let mem_size = 4096; + let mem_size = page_size::get(); let mut hshm = make_buffer(mem_size); let payload = b"test"; @@ -2481,7 +2480,7 @@ mod tests { fn read() { setup_signal_handler(); - let eshm = ExclusiveSharedMemory::new(4096).unwrap(); + let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap(); let (hshm, _) = eshm.build(); let guard_page_ptr = hshm.raw_ptr(); unsafe { std::ptr::read_volatile(guard_page_ptr) }; @@ -2492,7 +2491,7 @@ mod tests { fn write() { setup_signal_handler(); - let eshm = ExclusiveSharedMemory::new(4096).unwrap(); + let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap(); let (hshm, _) = eshm.build(); let guard_page_ptr = hshm.raw_ptr(); unsafe { std::ptr::write_volatile(guard_page_ptr, 0u8) }; @@ -2503,7 +2502,7 @@ mod tests { fn exec() { setup_signal_handler(); - let eshm = ExclusiveSharedMemory::new(4096).unwrap(); + let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap(); let (hshm, _) = eshm.build(); let guard_page_ptr = hshm.raw_ptr(); let func: fn() = unsafe { std::mem::transmute(guard_page_ptr) }; @@ -2550,7 +2549,6 @@ mod tests { mod from_file_tests { use std::io::Write; - use hyperlight_common::mem::PAGE_SIZE_USIZE; use tempfile::NamedTempFile; use crate::mem::shared_mem::{ReadonlySharedMemory, SharedMemory}; @@ -2570,10 +2568,10 @@ mod tests { #[test] fn from_file_success_single_page() { - let tmp = make_temp_file(PAGE_SIZE_USIZE); - let mut rsm = ReadonlySharedMemory::from_file(tmp.as_file(), PAGE_SIZE_USIZE) + let tmp = make_temp_file(page_size::get()); + let mut rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get()) .expect("from_file should succeed"); - assert_eq!(rsm.mem_size(), PAGE_SIZE_USIZE); + assert_eq!(rsm.mem_size(), page_size::get()); rsm.with_contents(|slice| { for (i, b) in slice.iter().enumerate() { assert_eq!(*b, (i & 0xff) as u8); @@ -2584,31 +2582,31 @@ mod tests { #[test] fn from_file_success_smaller_guest_mapped_size() { - let tmp = make_temp_file(2 * PAGE_SIZE_USIZE); - let rsm = ReadonlySharedMemory::from_file(tmp.as_file(), PAGE_SIZE_USIZE) + let tmp = make_temp_file(2 * page_size::get()); + let rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get()) .expect("from_file should succeed"); - assert_eq!(rsm.mem_size(), 2 * PAGE_SIZE_USIZE); + assert_eq!(rsm.mem_size(), 2 * page_size::get()); } #[test] fn from_file_rejects_empty_file() { let tmp = make_temp_file(0); - let err = ReadonlySharedMemory::from_file(tmp.as_file(), PAGE_SIZE_USIZE) + let err = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get()) .expect_err("empty file should be rejected"); assert!(format!("{}", err).contains("size 0")); } #[test] fn from_file_rejects_unaligned_file_length() { - let tmp = make_temp_file(PAGE_SIZE_USIZE + 1); - let err = ReadonlySharedMemory::from_file(tmp.as_file(), PAGE_SIZE_USIZE) + let tmp = make_temp_file(page_size::get() + 1); + let err = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get()) .expect_err("unaligned file length should be rejected"); assert!(format!("{}", err).contains("multiple of PAGE_SIZE")); } #[test] fn from_file_rejects_zero_guest_mapped_size() { - let tmp = make_temp_file(PAGE_SIZE_USIZE); + let tmp = make_temp_file(page_size::get()); let err = ReadonlySharedMemory::from_file(tmp.as_file(), 0) .expect_err("zero guest_mapped_size should be rejected"); assert!(format!("{}", err).contains("guest_mapped_size")); @@ -2616,16 +2614,16 @@ mod tests { #[test] fn from_file_rejects_unaligned_guest_mapped_size() { - let tmp = make_temp_file(2 * PAGE_SIZE_USIZE); - let err = ReadonlySharedMemory::from_file(tmp.as_file(), PAGE_SIZE_USIZE + 1) + let tmp = make_temp_file(2 * page_size::get()); + let err = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get() + 1) .expect_err("unaligned guest_mapped_size should be rejected"); assert!(format!("{}", err).contains("guest_mapped_size")); } #[test] fn from_file_rejects_guest_mapped_size_exceeding_file() { - let tmp = make_temp_file(PAGE_SIZE_USIZE); - let err = ReadonlySharedMemory::from_file(tmp.as_file(), 2 * PAGE_SIZE_USIZE) + let tmp = make_temp_file(page_size::get()); + let err = ReadonlySharedMemory::from_file(tmp.as_file(), 2 * page_size::get()) .expect_err("guest_mapped_size > file length should be rejected"); assert!(format!("{}", err).contains("guest_mapped_size")); } @@ -2636,8 +2634,6 @@ mod tests { /// test in the parent module re-executes each one in a subprocess /// and asserts the trap occurred. mod guard_page_crash_tests { - use hyperlight_common::mem::PAGE_SIZE_USIZE; - use super::make_temp_file; use crate::mem::shared_mem::{ReadonlySharedMemory, SharedMemory}; @@ -2645,10 +2641,10 @@ mod tests { #[test] #[ignore] pub(super) fn leading_guard_page_traps() { - let tmp = make_temp_file(PAGE_SIZE_USIZE); - let rsm = ReadonlySharedMemory::from_file(tmp.as_file(), PAGE_SIZE_USIZE) + let tmp = make_temp_file(page_size::get()); + let rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get()) .expect("from_file should succeed"); - let guard_ptr = unsafe { rsm.base_ptr().sub(PAGE_SIZE_USIZE) }; + let guard_ptr = unsafe { rsm.base_ptr().sub(page_size::get()) }; println!("reached_guard"); let _ = unsafe { std::ptr::read_volatile(guard_ptr) }; println!("survived_guard"); @@ -2658,8 +2654,8 @@ mod tests { #[test] #[ignore] pub(super) fn trailing_guard_page_traps() { - let tmp = make_temp_file(PAGE_SIZE_USIZE); - let rsm = ReadonlySharedMemory::from_file(tmp.as_file(), PAGE_SIZE_USIZE) + let tmp = make_temp_file(page_size::get()); + let rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get()) .expect("from_file should succeed"); let guard_ptr = unsafe { rsm.base_ptr().add(rsm.mem_size()) }; println!("reached_guard"); @@ -2767,7 +2763,12 @@ mod tests { #[cfg(unix)] { use std::os::unix::process::ExitStatusExt; - status.signal() == Some(libc::SIGSEGV) + let expected_signal = if cfg!(target_os = "macos") { + libc::SIGBUS + } else { + libc::SIGSEGV + }; + status.signal() == Some(expected_signal) } #[cfg(windows)] { diff --git a/src/hyperlight_host/src/mem/shared_mem_tests.rs b/src/hyperlight_host/src/mem/shared_mem_tests.rs index 09e3c4f8e..1da949333 100644 --- a/src/hyperlight_host/src/mem/shared_mem_tests.rs +++ b/src/hyperlight_host/src/mem/shared_mem_tests.rs @@ -20,8 +20,6 @@ use std::convert::TryFrom; use std::fmt::Debug; use std::mem::size_of; -use hyperlight_common::mem::PAGE_SIZE_USIZE; - use crate::{Result, log_then_return, new_error}; /// A function that knows how to read data of type `T` from a @@ -53,7 +51,7 @@ where T: PartialEq + Debug + Clone + TryFrom, U: Debug + Clone, { - let mem_size = PAGE_SIZE_USIZE; + let mem_size = page_size::get(); let test_read = |mem_size, offset| { let sm = shared_memory_new(mem_size)?; (reader)(&sm, offset) diff --git a/src/hyperlight_host/src/sandbox/config.rs b/src/hyperlight_host/src/sandbox/config.rs index f12387a0b..29b102468 100644 --- a/src/hyperlight_host/src/sandbox/config.rs +++ b/src/hyperlight_host/src/sandbox/config.rs @@ -142,13 +142,13 @@ impl SandboxConfiguration { } /// Sets the interrupt retry delay - #[cfg(target_os = "linux")] + #[cfg(any(kvm, mshv3, hvf))] pub fn set_interrupt_retry_delay(&mut self, delay: Duration) { self.interrupt_retry_delay = delay; } /// Get the delay between retries for interrupts - #[cfg(target_os = "linux")] + #[cfg(any(kvm, mshv3, hvf))] pub fn get_interrupt_retry_delay(&self) -> Duration { self.interrupt_retry_delay } diff --git a/src/hyperlight_host/src/sandbox/file_mapping.rs b/src/hyperlight_host/src/sandbox/file_mapping.rs index 4f3ed2a4d..58f1dce6f 100644 --- a/src/hyperlight_host/src/sandbox/file_mapping.rs +++ b/src/hyperlight_host/src/sandbox/file_mapping.rs @@ -71,7 +71,7 @@ pub(crate) enum HostFileResources { view_base: *mut c_void, }, /// Linux: `mmap` base pointer. - #[cfg(target_os = "linux")] + #[cfg(unix)] Linux { mmap_base: *mut c_void, mmap_size: usize, @@ -103,7 +103,7 @@ impl Drop for PreparedFileMapping { tracing::error!("PreparedFileMapping::drop: CloseHandle failed: {:?}", e); } }, - #[cfg(target_os = "linux")] + #[cfg(unix)] HostFileResources::Linux { mmap_base, mmap_size, @@ -168,7 +168,7 @@ impl PreparedFileMapping { region_type: MemoryRegionType::MappedFile, }) } - #[cfg(target_os = "linux")] + #[cfg(unix)] HostFileResources::Linux { mmap_base, mmap_size, diff --git a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs index 8e60c68e7..f7a7f1856 100644 --- a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs +++ b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs @@ -1532,9 +1532,8 @@ mod tests { } fn page_aligned_memory(src: &[u8]) -> GuestSharedMemory { - use hyperlight_common::mem::PAGE_SIZE_USIZE; - - let len = src.len().div_ceil(PAGE_SIZE_USIZE) * PAGE_SIZE_USIZE; + let page_size = page_size::get(); + let len = src.len().div_ceil(page_size) * page_size; let mut mem = ExclusiveSharedMemory::new(len).unwrap(); mem.copy_from_slice(src, 0).unwrap(); @@ -2643,15 +2642,16 @@ mod tests { }; // Use multi-page regions so partial overlap is geometrically possible - let mem1 = page_aligned_memory(&[0xAA; 8192]); // 2 pages - let mem2 = page_aligned_memory(&[0xBB; 8192]); // 2 pages + let ps = page_size::get(); + let mem1 = page_aligned_memory(&vec![0xAA; ps * 2]); // 2 pages + let mem2 = page_aligned_memory(&vec![0xBB; ps * 2]); // 2 pages let guest_base: usize = 0x200000000; let region1 = region_for_memory(&mem1, guest_base, MemoryRegionFlags::READ); unsafe { sbox.map_region(®ion1).unwrap() }; // region2 starts one page before region1, overlapping by one page - let overlap_base = guest_base - 0x1000; + let overlap_base = guest_base - ps; let region2 = region_for_memory(&mem2, overlap_base, MemoryRegionFlags::READ); let err = unsafe { sbox.map_region(®ion2) }.unwrap_err(); assert!( diff --git a/src/hyperlight_host/src/sandbox/snapshot/file/config.rs b/src/hyperlight_host/src/sandbox/snapshot/file/config.rs index 5e9fe6c02..6d549325b 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file/config.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file/config.rs @@ -64,6 +64,7 @@ pub(super) enum Hypervisor { Kvm, Mshv, Whp, + Hvf, } impl Hypervisor { @@ -79,6 +80,8 @@ impl Hypervisor { Some(HypervisorType::Mshv) => Some(Self::Mshv), #[cfg(target_os = "windows")] Some(HypervisorType::Whp) => Some(Self::Whp), + #[cfg(hvf)] + Some(HypervisorType::Hvf) => Some(Self::Hvf), None => None, } } @@ -88,6 +91,7 @@ impl Hypervisor { Self::Kvm => "KVM", Self::Mshv => "MSHV", Self::Whp => "WHP", + Self::Hvf => "HVF", } } @@ -99,6 +103,7 @@ impl Hypervisor { Self::Kvm => "kvm", Self::Mshv => "mshv", Self::Whp => "whp", + Self::Hvf => "hvf", } } } @@ -132,6 +137,10 @@ impl CpuVendor { // `0x` prefix padded to width 4, e.g. Apple `0x61`, Arm `0x41`. Self(format!("{implementer:#04x}")) } + #[cfg(all(target_arch = "aarch64", target_os = "macos"))] + { + Self("0x61".to_string()) + } } pub(super) fn as_str(&self) -> &str { @@ -674,6 +683,8 @@ mod tests { #[cfg(all(target_arch = "aarch64", target_os = "linux"))] // MIDR_EL1 implementer byte for Apple silicon. assert_eq!(v, "0x61", "unexpected aarch64 CPU implementer"); + #[cfg(all(target_arch = "aarch64", target_os = "macos"))] + assert_eq!(v, "0x61", "unexpected aarch64 CPU implementer"); } /// The architecture the current host is not running. diff --git a/src/hyperlight_host/src/sandbox/snapshot/file/media_types.rs b/src/hyperlight_host/src/sandbox/snapshot/file/media_types.rs index 661bd4a04..347e1e8b6 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file/media_types.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file/media_types.rs @@ -27,7 +27,7 @@ pub(in crate::sandbox::snapshot) const MT_SNAPSHOT_CURRENT: &str = MT_SNAPSHOT_V /// ABI version for the snapshot memory blob. Bumped when the /// host-guest contract for the snapshot bytes changes. See /// docs/snapshot-versioning.md. -pub(in crate::sandbox::snapshot) const SNAPSHOT_ABI_VERSION: u32 = 1; +pub(in crate::sandbox::snapshot) const SNAPSHOT_ABI_VERSION: u32 = 2; /// OCI standard annotation key for a manifest's tag inside an image /// index. Set on the manifest descriptor in `index.json`, not on the diff --git a/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs b/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs index 4dae80a9e..b650f338e 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs @@ -498,6 +498,7 @@ fn cfg_current_hypervisor() -> &'static str { "kvm" => "kvm", "mshv" => "mshv", "whp" => "whp", + "hvf" => "hvf", other => panic!("unknown hypervisor tag {other}"), } } @@ -1045,7 +1046,7 @@ fn save_appends_into_existing_layout_with_new_tag() { ); let hv = anns["dev.hyperlight.snapshot.hypervisor"].as_str().unwrap(); assert!( - ["kvm", "mshv", "whp"].contains(&hv), + ["kvm", "mshv", "whp", "hvf"].contains(&hv), "unexpected hypervisor annotation: {}", hv ); diff --git a/src/hyperlight_host/src/sandbox/snapshot/mod.rs b/src/hyperlight_host/src/sandbox/snapshot/mod.rs index c062efad7..c7bdc3f34 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/mod.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/mod.rs @@ -550,6 +550,11 @@ impl Snapshot { } pt_buf.set_root(pt_buf.initial_root()); + snapshot_memory.resize( + snapshot_memory.len().next_multiple_of(page_size::get()), + 0u8, + ); + // Phase 4: finalize PT bytes. let pt_data = pt_buf.into_bytes(); layout.set_pt_size(pt_data.len())?; @@ -564,7 +569,7 @@ impl Snapshot { // `map_file_cow` regions installed immediately after the // snapshot in guest PA space. let guest_visible_size = memory.len() - layout.get_pt_size(); - debug_assert!(guest_visible_size.is_multiple_of(PAGE_SIZE)); + debug_assert!(guest_visible_size.is_multiple_of(page_size::get())); layout.set_snapshot_size(guest_visible_size); Ok(Self { @@ -727,14 +732,16 @@ mod tests { CommonSpecialRegisters::default() } - const SIMPLE_PT_BASE: usize = PAGE_SIZE + SandboxMemoryLayout::BASE_ADDRESS; + fn simple_pt_base() -> usize { + page_size::get() + SandboxMemoryLayout::BASE_ADDRESS + } fn make_simple_pt_mem(contents: &[u8]) -> SnapshotSharedMemory { - let pt_buf = GuestPageTableBuffer::new(SIMPLE_PT_BASE); + let pt_buf = GuestPageTableBuffer::new(simple_pt_base()); let mapping = Mapping { phys_base: SandboxMemoryLayout::BASE_ADDRESS as u64, virt_base: SandboxMemoryLayout::BASE_ADDRESS as u64, - len: PAGE_SIZE as u64, + len: page_size::get() as u64, kind: MappingKind::Basic(BasicMapping { readable: true, writable: true, @@ -745,10 +752,10 @@ mod tests { super::map_specials(&pt_buf, PAGE_SIZE); let pt_bytes = pt_buf.into_bytes(); - let mut snapshot_mem = vec![0u8; PAGE_SIZE + pt_bytes.len()]; - snapshot_mem[0..PAGE_SIZE].copy_from_slice(contents); - snapshot_mem[PAGE_SIZE..].copy_from_slice(&pt_bytes); - ReadonlySharedMemory::from_bytes(&snapshot_mem, PAGE_SIZE) + let mut snapshot_mem = vec![0u8; page_size::get() + pt_bytes.len()]; + snapshot_mem[0..page_size::get()].copy_from_slice(contents); + snapshot_mem[page_size::get()..].copy_from_slice(&pt_bytes); + ReadonlySharedMemory::from_bytes(&snapshot_mem, page_size::get()) .unwrap() .to_mgr_snapshot_mem() .unwrap() @@ -759,12 +766,12 @@ mod tests { let scratch_mem = ExclusiveSharedMemory::new(cfg.get_scratch_size()).unwrap(); let mgr = SandboxMemoryManager::new( SandboxMemoryLayout::new(cfg, 4096, 0x3000, None).unwrap(), - make_simple_pt_mem(&[0u8; PAGE_SIZE]), + make_simple_pt_mem(&vec![0u8; page_size::get()]), scratch_mem, super::NextAction::None, ); let (mgr, _) = mgr.build().unwrap(); - (mgr, SIMPLE_PT_BASE as u64) + (mgr, simple_pt_base() as u64) } #[test] @@ -772,7 +779,7 @@ mod tests { let (mut mgr, pt_base) = make_simple_pt_mgr(); // Create first snapshot with pattern A - let pattern_a = vec![0xAA; PAGE_SIZE]; + let pattern_a = vec![0xAA; page_size::get()]; let snapshot_a = super::Snapshot::new( &mut make_simple_pt_mem(&pattern_a).build().0, &mut mgr.scratch_mem, @@ -790,7 +797,7 @@ mod tests { .unwrap(); // Create second snapshot with pattern B - let pattern_b = vec![0xBB; PAGE_SIZE]; + let pattern_b = vec![0xBB; page_size::get()]; let snapshot_b = super::Snapshot::new( &mut make_simple_pt_mem(&pattern_b).build().0, &mut mgr.scratch_mem, diff --git a/src/hyperlight_host/src/sandbox/snapshot/tripwires.rs b/src/hyperlight_host/src/sandbox/snapshot/tripwires.rs index ee04373a8..f8cd3c9bb 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/tripwires.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/tripwires.rs @@ -28,7 +28,7 @@ use super::file::{ MT_CONFIG_CURRENT, MT_SNAPSHOT_CURRENT, OCI_LAYOUT_VERSION, SNAPSHOT_ABI_VERSION, }; -const EXPECTED_ABI_VERSION: u32 = 1; +const EXPECTED_ABI_VERSION: u32 = 2; const EXPECTED_MT_CONFIG: &str = "application/vnd.hyperlight.snapshot.config.v1+json"; const EXPECTED_MT_SNAPSHOT: &str = "application/vnd.hyperlight.snapshot.memory.v1"; const EXPECTED_OCI_LAYOUT_VERSION: &str = "1.0.0"; @@ -54,14 +54,15 @@ const _: () = { }; const _: () = { - use hyperlight_common::mem::{HyperlightPEB, PAGE_SIZE_USIZE}; + use hyperlight_common::mem::HyperlightPEB; + use hyperlight_common::vmem::PAGE_SIZE; // The loading host derives `guest_heap_buffer_offset`, and every // offset after it, from the PEB's page-rounded size. Existing // snapshots place the PEB in a single page, so the only thing that // must hold is that the PEB keeps fitting in one page. Field layout // is not pinned: only the captured guest reads the fields, and it // travels inside the snapshot, so it stays self-consistent. - abi_assert!(std::mem::size_of::() <= PAGE_SIZE_USIZE); + abi_assert!(std::mem::size_of::() <= PAGE_SIZE); }; const _: () = { @@ -86,7 +87,7 @@ const _: () = { const _: () = { use crate::mem::layout::SandboxMemoryLayout; - abi_assert!(SandboxMemoryLayout::BASE_ADDRESS == 0x1000); + abi_assert!(SandboxMemoryLayout::BASE_ADDRESS == 0x4000); }; const fn str_eq(a: &str, b: &str) -> bool { diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index b449ea68d..b71217c5e 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -1472,11 +1472,14 @@ fn interrupt_infinite_moving_loop_stress_test() { use std::thread; // We have a high thread count to stress test and to have interesting interleavings - const NUM_THREADS: usize = 200; + #[cfg(not(hvf))] + let num_threads: usize = 200; + #[cfg(hvf)] + let num_threads: usize = 30; let mut handles = vec![]; - for _ in 0..NUM_THREADS { + for _ in 0..num_threads { handles.push(thread::spawn(move || { let entered_guest = Arc::new(AtomicBool::new(false)); let entered_guest_clone = entered_guest.clone(); diff --git a/src/hyperlight_host/tests/sandbox_host_tests.rs b/src/hyperlight_host/tests/sandbox_host_tests.rs index e0daf969b..294f0e936 100644 --- a/src/hyperlight_host/tests/sandbox_host_tests.rs +++ b/src/hyperlight_host/tests/sandbox_host_tests.rs @@ -22,6 +22,8 @@ use hyperlight_host::{ GuestBinary, HyperlightError, MultiUseSandbox, Result, UninitializedSandbox, new_error, }; use hyperlight_testing::simple_guest_as_string; +#[cfg(hvf)] +use serial_test::serial; pub mod common; // pub to disable dead_code warning use crate::common::{ @@ -123,7 +125,7 @@ fn invalid_guest_function_name() { #[test] fn set_static() { let mut cfg: SandboxConfiguration = Default::default(); - cfg.set_scratch_size(0x100A000); + cfg.set_scratch_size(0x100C000); with_all_sandboxes_cfg(Some(cfg), |mut sandbox| { let fn_name = "SetStatic"; let res = sandbox.call::(fn_name, ()); @@ -285,6 +287,7 @@ fn simple_test() { } #[test] +#[cfg_attr(hvf, serial(thread_heavy))] fn simple_test_parallel() { let handles: Vec<_> = (0..50) .map(|_| { @@ -329,8 +332,13 @@ fn callback_test() { } #[test] +#[cfg_attr(hvf, serial(thread_heavy))] fn callback_test_parallel() { - let handles: Vec<_> = (0..100) + #[cfg(not(hvf))] + let n_threads = 100; + #[cfg(hvf)] + let n_threads = 50; + let handles: Vec<_> = (0..n_threads) .map(|_| { std::thread::spawn(|| { callback_test_helper(); diff --git a/src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs b/src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs index 32572f417..fcb48c7f5 100644 --- a/src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs +++ b/src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs @@ -21,11 +21,14 @@ limitations under the License. //! publish. See `docs/snapshot-versioning.md`. /// Goldens version, a `vMAJOR.MINOR` string. -pub(crate) const GOLDENS_VERSION: &str = "v1.0"; +pub(crate) const GOLDENS_VERSION: &str = "v2.0"; /// Old majors kept loadable through a compatibility path, verified /// alongside `GOLDENS_VERSION`. A backwards-compatible break (Option 2) /// adds the outgoing version here. See `docs/snapshot-versioning.md`. +#[cfg(target_arch = "aarch64")] +pub(crate) const COMPAT_VERSIONS: &[&str] = &[]; +#[cfg(target_arch = "x86_64")] pub(crate) const COMPAT_VERSIONS: &[&str] = &[]; /// Every version the verify test checks: the current one and each kept diff --git a/src/hyperlight_host/tests/snapshot_goldens/platform.rs b/src/hyperlight_host/tests/snapshot_goldens/platform.rs index 54be335c9..bb274538f 100644 --- a/src/hyperlight_host/tests/snapshot_goldens/platform.rs +++ b/src/hyperlight_host/tests/snapshot_goldens/platform.rs @@ -26,12 +26,14 @@ use crate::goldens_version::GOLDENS_VERSION; #[derive(Copy, Clone, Debug, PartialEq, Eq)] enum Hypervisor { - #[cfg_attr(target_os = "windows", allow(dead_code))] + #[cfg_attr(not(kvm), allow(dead_code))] Kvm, - #[cfg_attr(target_os = "windows", allow(dead_code))] + #[cfg_attr(not(mshv3), allow(dead_code))] Mshv, #[cfg_attr(not(target_os = "windows"), allow(dead_code))] Whp, + #[cfg_attr(not(hvf), allow(dead_code))] + Hvf, } impl Hypervisor { @@ -40,6 +42,7 @@ impl Hypervisor { Self::Kvm => "kvm", Self::Mshv => "mshv", Self::Whp => "whp", + Self::Hvf => "hvf", } } @@ -61,7 +64,11 @@ impl Hypervisor { { Some(Self::Whp) } - #[cfg(not(any(target_os = "linux", target_os = "windows")))] + #[cfg(hvf)] + { + Some(Self::Hvf) + } + #[cfg(not(any(target_os = "linux", target_os = "windows", hvf)))] { None }