vmm: add diff snapshots

Every snapshot currently dumps the entire guest RAM, so the pause a
snapshot imposes grows with the memory size regardless of how little
the guest has changed. Iterative checkpointing and warm pre-copy (take
a snapshot, let the guest run, take another) repeat that full-memory
cost every round.

Add a diff snapshot: `vm.snapshot` accepts a `snapshot_type` of `full`
(default) or `diff`, exposed as `ch-remote snapshot --diff`. The first
diff of a series writes a full baseline and enables dirty-page
tracking; each subsequent diff dumps only the pages dirtied since the
previous one, so its pause is proportional to the dirtied memory
rather than to the guest RAM size. Dirty pages are harvested after the
device snapshot, so pages touched by snapshot side effects are
included. They are written as a sparse `memory-ranges.diff` whose
extents sit at the same offsets as in the baseline `memory-ranges`, so
a delta applies onto the baseline without translation. A full
snapshot, a memory layout change, a migration, restore, or deleting
the VM ends the series; the next diff starts a new one.

Restore takes the chain directly: `vm.restore` accepts `memory_chain`,
the ancestor snapshot URLs oldest first, with `source_url` pointing at
the newest delta. Memory fills from the baseline, then every delta's
dirty extents replay in order through the same SEEK_DATA walk the
eager restore already uses; device state and config come from the
newest delta as before. A delta whose length does not match the
restored layout is rejected before any page is touched, a filesystem
that cannot report extents fails the restore rather than zeroing
undirtied pages, and `memory_chain` is incompatible with
`memory_restore_mode=ondemand`.

Dirty tracking is enabled lazily on the first diff rather than at
boot, so a VM that never takes a diff snapshot pays no runtime cost.

Validated: an integration test writes tmpfs markers before and after
the baseline and reads both back through a chain restore; unit tests
cover extent replay over holes and the layout-length check. On a
three-phase workload that dirtied one phase between snapshots, the
diff pause measured ~17x shorter than the full one.

Signed-off-by: CMGS <ilskdw@gmail.com>

Author: CMGS

cloud-hypervisor/src/bin/ch-remote.rs

@@ -487,12 +487,10 @@ fn rest_api_do_command(matches: &ArgMatches, socket: &mut UnixStream) -> ApiResu
                 .map_err(Error::HttpApiClient)
         }
         Some("snapshot") => {
+            let sub = matches.subcommand_matches("snapshot").unwrap();
             let snapshot_config = snapshot_config(
-                matches
-                    .subcommand_matches("snapshot")
-                    .unwrap()
-                    .get_one::<String>("snapshot_config")
-                    .unwrap(),
+                sub.get_one::<String>("snapshot_config").unwrap(),
+                sub.get_flag("diff"),
             );
             simple_api_command(socket, "PUT", "snapshot", Some(&snapshot_config))
                 .map_err(Error::HttpApiClient)
@@ -711,12 +709,10 @@ fn dbus_api_do_command(matches: &ArgMatches, proxy: &DBusApi1ProxyBlocking<'_>)
             proxy.api_vm_add_vsock(&vsock_config)
         }
         Some("snapshot") => {
+            let sub = matches.subcommand_matches("snapshot").unwrap();
             let snapshot_config = snapshot_config(
-                matches
-                    .subcommand_matches("snapshot")
-                    .unwrap()
-                    .get_one::<String>("snapshot_config")
-                    .unwrap(),
+                sub.get_one::<String>("snapshot_config").unwrap(),
+                sub.get_flag("diff"),
             );
             proxy.api_vm_snapshot(&snapshot_config)
         }
@@ -923,9 +919,14 @@ fn add_vsock_config(config: &str) -> Result<String, Error> {
     Ok(vsock_config)
 }
 
-fn snapshot_config(url: &str) -> String {
+fn snapshot_config(url: &str, diff: bool) -> String {
     let snapshot_config = api::VmSnapshotConfig {
         destination_url: String::from(url),
+        snapshot_type: if diff {
+            api::VmSnapshotType::Diff
+        } else {
+            api::VmSnapshotType::Full
+        },
     };
 
     serde_json::to_string(&snapshot_config).unwrap()
@@ -1165,6 +1166,15 @@ fn get_cli_commands_sorted() -> Box<[Command]> {
         Command::new("shutdown-vmm").about("Shutdown the VMM"),
         Command::new("snapshot")
             .about("Create a snapshot from VM")
+            .arg(
+                Arg::new("diff")
+                    .long("diff")
+                    .action(clap::ArgAction::SetTrue)
+                    .help(
+                        "Write only pages dirtied since the previous snapshot; \
+                         the first --diff takes a full baseline and starts dirty tracking",
+                    ),
+            )
             .arg(
                 Arg::new("snapshot_config")
                     .index(1)

cloud-hypervisor/tests/integration.rs

@@ -8289,6 +8289,12 @@ mod ivshmem {
         );
     }
 
+    #[test]
+    #[cfg(not(feature = "mshv"))]
+    fn test_snapshot_restore_diff_chain() {
+        snapshot_restore_common::_test_snapshot_restore_diff_chain();
+    }
+
     #[test]
     #[cfg(not(feature = "mshv"))]
     fn test_snapshot_restore_with_resume() {
@@ -8428,7 +8434,7 @@ mod ivshmem {
 
 #[cfg(not(feature = "mshv"))]
 mod snapshot_restore_common {
-    use std::fs::{read_to_string, remove_dir_all};
+    use std::fs::{create_dir, read_to_string, remove_dir_all};
     use std::process::Command;
 
     use crate::*;
@@ -8488,6 +8494,141 @@ mod snapshot_restore_common {
         ));
     }
 
+    fn snapshot_diff(api_socket: &str, url: &str) -> bool {
+        let output = Command::new(clh_command("ch-remote"))
+            .args([
+                &format!("--api-socket={api_socket}"),
+                "snapshot",
+                "--diff",
+                url,
+            ])
+            .output()
+            .unwrap();
+        if !output.status.success() {
+            eprintln!(
+                "ch-remote snapshot --diff failed: {}",
+                String::from_utf8_lossy(&output.stderr)
+            );
+        }
+        output.status.success()
+    }
+
+    // A diff series baseline plus one delta, restored through memory_chain.
+    // tmpfs markers written before and after the baseline prove the delta
+    // rebases onto the baseline: marker A lives in the baseline pages, marker
+    // B only in the delta's dirty extents.
+    pub(crate) fn _test_snapshot_restore_diff_chain() {
+        let disk_config = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string());
+        let guest = Guest::new(Box::new(disk_config));
+        let kernel_path = direct_kernel_boot_path();
+
+        let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir));
+        let net_params = format!(
+            "id=net123,tap=,mac={},ip={},mask=255.255.255.128",
+            guest.network.guest_mac0, guest.network.host_ip0
+        );
+
+        let mut child = GuestCommand::new(&guest)
+            .args(["--api-socket", &api_socket_source])
+            .args(["--cpus", "boot=1"])
+            .args(["--memory", "size=1G"])
+            .args(["--kernel", kernel_path.to_str().unwrap()])
+            .args([
+                "--disk",
+                format!(
+                    "path={}",
+                    guest.disk_config.disk(DiskType::OperatingSystem).unwrap()
+                )
+                .as_str(),
+                format!(
+                    "path={}",
+                    guest.disk_config.disk(DiskType::CloudInit).unwrap()
+                )
+                .as_str(),
+            ])
+            .args(["--net", net_params.as_str()])
+            .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
+            .capture_output()
+            .spawn()
+            .unwrap();
+
+        let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir);
+        let base_dir = format!("{snapshot_dir}/base");
+        let delta_dir = format!("{snapshot_dir}/delta");
+        create_dir(&base_dir).unwrap();
+        create_dir(&delta_dir).unwrap();
+
+        let r = panic::catch_unwind(|| {
+            guest.wait_vm_boot().unwrap();
+
+            guest
+                .ssh_command("echo diffmarkerA > /dev/shm/marker_a")
+                .unwrap();
+
+            // First diff of a series writes the full baseline.
+            assert!(remote_command(&api_socket_source, "pause", None));
+            assert!(snapshot_diff(
+                &api_socket_source,
+                format!("file://{base_dir}").as_str(),
+            ));
+            assert!(remote_command(&api_socket_source, "resume", None));
+
+            guest
+                .ssh_command("echo diffmarkerB > /dev/shm/marker_b")
+                .unwrap();
+
+            // Second diff carries only the pages dirtied since the baseline.
+            assert!(remote_command(&api_socket_source, "pause", None));
+            assert!(snapshot_diff(
+                &api_socket_source,
+                format!("file://{delta_dir}").as_str(),
+            ));
+        });
+
+        kill_child(&mut child);
+        let output = child.wait_with_output().unwrap();
+        handle_child_output(r, &output);
+
+        // Restore from the delta, chaining it onto the baseline.
+        let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir));
+        let mut child = GuestCommand::new(&guest)
+            .args(["--api-socket", &api_socket_restored])
+            .args([
+                "--restore",
+                format!(
+                    "source_url=file://{delta_dir},memory_chain=[file://{base_dir}],resume=true"
+                )
+                .as_str(),
+            ])
+            .capture_output()
+            .spawn()
+            .unwrap();
+
+        let r = panic::catch_unwind(|| {
+            assert!(wait_until(Duration::from_secs(30), || remote_command(
+                &api_socket_restored,
+                "info",
+                None
+            )));
+
+            // Both markers must survive: A from the baseline pages the delta
+            // never dirtied, B from the delta's extents.
+            assert_eq!(
+                guest.ssh_command("cat /dev/shm/marker_a").unwrap().trim(),
+                "diffmarkerA"
+            );
+            assert_eq!(
+                guest.ssh_command("cat /dev/shm/marker_b").unwrap().trim(),
+                "diffmarkerB"
+            );
+        });
+
+        let _ = remove_dir_all(snapshot_dir.as_str());
+        kill_child(&mut child);
+        let output = child.wait_with_output().unwrap();
+        handle_child_output(r, &output);
+    }
+
     /// Easy disambiguation between snapshot/restore variants.
     #[derive(Clone, Copy, Default)]
     pub(crate) struct SnapshotRestoreTest {

docs/snapshot_restore.md

@@ -60,6 +60,32 @@ be needed.
 `state.json` contains the virtual machine state. It is used to restore each
 component in the state it was left before the snapshot occurred.
 
+## Diff snapshots
+
+Passing `snapshot_type=diff` (`ch-remote snapshot --diff <url>`) writes only
+the pages dirtied since the previous snapshot of the series. The first `diff`
+request takes a full baseline and enables dirty-page tracking; each subsequent
+one dumps the delta, so the pause cost is proportional to the amount of dirtied
+memory rather than to the guest RAM size. A `full` request (or any snapshot
+failure, a memory layout change, a migration, or deleting the VM) ends the
+series; the next `diff` starts a new one with a fresh baseline.
+
+A delta directory contains `config.json`, `state.json` and
+`memory-ranges.diff`, a sparse file whose extents sit at the same offsets as
+in the baseline `memory-ranges`. Restore it by pointing `source_url` at the
+newest delta and listing its ancestors, oldest first, in `memory_chain`:
+
+```bash
+./ch-remote --api-socket=/tmp/cloud-hypervisor.sock restore \
+    source_url=file:///foo/diff2,memory_chain=[file:///foo/base,file:///foo/diff1]
+```
+
+Memory is filled from the baseline, then each delta's dirty extents are
+replayed in order; device state and config come from the newest delta. The
+files must all come from one uninterrupted series and sit on a filesystem
+with `SEEK_DATA`/`SEEK_HOLE` support, and `memory_chain` cannot be combined
+with `memory_restore_mode=ondemand`.
+
 ## Restore a Cloud Hypervisor VM
 
 Given that one has access to an existing snapshot in `/home/foo/snapshot`,

fuzz/fuzz_targets/http_api.rs

@@ -15,7 +15,7 @@ use vm_migration::MigratableError;
 use vmm::api::http::*;
 use vmm::api::{
     ApiRequest, RequestHandler, VmInfoResponse, VmReceiveMigrationData, VmSendMigrationData,
-    VmmPingResponse,
+    VmSnapshotConfig, VmmPingResponse,
 };
 use vmm::config::RestoreConfig;
 use vmm::vm::{Error as VmError, VmState};
@@ -100,7 +100,7 @@ impl RequestHandler for StubApiRequestHandler {
         Ok(())
     }
 
-    fn vm_snapshot(&mut self, _: &str) -> Result<(), VmError> {
+    fn vm_snapshot(&mut self, _: &VmSnapshotConfig) -> Result<(), VmError> {
         Ok(())
     }
 

vmm/src/api/mod.rs

@@ -262,10 +262,25 @@ pub struct VmRemoveDeviceData {
     pub id: String,
 }
 
+/// Type of a VM snapshot: full memory dump or dirty-pages-only delta.
+#[derive(Copy, Clone, Default, Deserialize, Serialize, Debug, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum VmSnapshotType {
+    /// Complete guest memory dump.
+    #[default]
+    Full,
+    /// Only pages dirtied since the previous snapshot of the series. The
+    /// first diff request takes a full baseline and starts dirty tracking.
+    Diff,
+}
+
 #[derive(Clone, Deserialize, Serialize, Default, Debug)]
 pub struct VmSnapshotConfig {
     /// The snapshot destination URL
     pub destination_url: String,
+    /// Full dump or dirty-pages delta.
+    #[serde(default)]
+    pub snapshot_type: VmSnapshotType,
 }
 
 #[derive(Clone, Deserialize, Serialize, Default, Debug)]
@@ -839,7 +854,7 @@ pub trait RequestHandler {
 
     fn vm_resume(&mut self) -> Result<(), VmError>;
 
-    fn vm_snapshot(&mut self, destination_url: &str) -> Result<(), VmError>;
+    fn vm_snapshot(&mut self, config: &VmSnapshotConfig) -> Result<(), VmError>;
 
     fn vm_restore(&mut self, restore_cfg: RestoreConfig) -> Result<(), VmError>;
 
@@ -1960,7 +1975,7 @@ impl ApiAction for VmSnapshot {
             info!("API request event: VmSnapshot {config:?}");
 
             let response = vmm
-                .vm_snapshot(&config.destination_url)
+                .vm_snapshot(&config)
                 .map_err(ApiError::VmSnapshot)
                 .map(|_| ApiResponsePayload::Empty);
 
@@ -2454,4 +2469,21 @@ mod unit_tests {
         )
         .unwrap_err();
     }
+
+    #[test]
+    fn test_vm_snapshot_config_snapshot_type() {
+        let config: VmSnapshotConfig =
+            serde_json::from_str(r#"{"destination_url": "file:///foo"}"#).unwrap();
+        assert_eq!(config.snapshot_type, VmSnapshotType::Full);
+
+        let config: VmSnapshotConfig =
+            serde_json::from_str(r#"{"destination_url": "file:///foo", "snapshot_type": "diff"}"#)
+                .unwrap();
+        assert_eq!(config.snapshot_type, VmSnapshotType::Diff);
+
+        serde_json::from_str::<VmSnapshotConfig>(
+            r#"{"destination_url": "file:///foo", "snapshot_type": "bogus"}"#,
+        )
+        .unwrap_err();
+    }
 }

vmm/src/api/openapi/cloud-hypervisor.yaml

@@ -1475,6 +1475,14 @@ components:
       properties:
         destination_url:
           type: string
+        snapshot_type:
+          type: string
+          enum: [full, diff]
+          default: full
+          description:
+            With "diff", only pages dirtied since the previous snapshot of the
+            series are written; the first "diff" takes a full baseline and
+            starts dirty tracking.
 
     VmCoredumpData:
       type: object
@@ -1498,6 +1506,13 @@ components:
           type: boolean
         memory_restore_mode:
           $ref: "#/components/schemas/MemoryRestoreMode"
+        memory_chain:
+          type: array
+          items:
+            type: string
+          description:
+            Ancestor snapshot URLs of a diff-snapshot series, oldest (the
+            full baseline) first; source_url is the newest delta.
         resume:
           type: boolean
         zone_updates: