diff --git a/cmd/core/vmconfig.go b/cmd/core/vmconfig.go index b693d28d..d8f5f73b 100644 --- a/cmd/core/vmconfig.go +++ b/cmd/core/vmconfig.go @@ -32,6 +32,7 @@ func VMConfigFromFlags(cmd *cobra.Command, image string) (*types.VMConfig, error user, _ := cmd.Flags().GetString("user") password, _ := cmd.Flags().GetString("password") noDirectIO, _ := cmd.Flags().GetBool("no-direct-io") + noWatchdog, _ := cmd.Flags().GetBool("no-watchdog") windows, _ := cmd.Flags().GetBool("windows") sharedMemory, _ := cmd.Flags().GetBool("shared-memory") hugePages, _ := cmd.Flags().GetBool("hugepages") @@ -65,6 +66,7 @@ func VMConfigFromFlags(cmd *cobra.Command, image string) (*types.VMConfig, error Image: image, Network: network, NoDirectIO: noDirectIO, + NoWatchdog: noWatchdog, Windows: windows, SharedMemory: sharedMemory, HugePages: hugePages, @@ -106,6 +108,7 @@ func CloneVMConfigFromFlags(cmd *cobra.Command, snapCfg types.SnapshotConfig) (* if cmd.Flags().Changed("no-direct-io") { noDirectIO, _ = cmd.Flags().GetBool("no-direct-io") } + noWatchdog := snapCfg.NoWatchdog restoreMode, err := restoreModeFromFlags(cmd) if err != nil { @@ -130,6 +133,7 @@ func CloneVMConfigFromFlags(cmd *cobra.Command, snapCfg types.SnapshotConfig) (* ImageType: snapCfg.ImageType, Network: network, NoDirectIO: noDirectIO, + NoWatchdog: noWatchdog, Windows: snapCfg.Windows, SharedMemory: snapCfg.SharedMemory, HugePages: snapCfg.HugePages, diff --git a/cmd/core/vmconfig_test.go b/cmd/core/vmconfig_test.go index 190aca1b..9045ba32 100644 --- a/cmd/core/vmconfig_test.go +++ b/cmd/core/vmconfig_test.go @@ -90,18 +90,20 @@ func TestCloneVMConfigKnobFlagsOverrideSnapshot(t *testing.T) { snapCfg := types.SnapshotConfig{Config: types.Config{ CPU: 2, Memory: 1 << 30, Storage: 10 << 30, CPUWeight: 40, CPUQuotaUs: 200000, CPUBurstUs: 50000, + NoWatchdog: true, }} tests := []struct { - name string - set map[string]string - wantWeight int - wantQuota int64 - wantBurst int64 - wantErr bool + name string + set map[string]string + wantWeight int + wantQuota int64 + wantBurst int64 + wantNoWatchdog bool + wantErr bool }{ - {name: "no flags ignore snapshot knobs", wantWeight: 0, wantQuota: 0, wantBurst: 0}, - {name: "flags set the clone's policy", set: map[string]string{"cpu-weight": "10", "cpu-burst-us": "100000", "cpu-quota-us": "150000"}, wantWeight: 10, wantQuota: 150000, wantBurst: 100000}, + {name: "no flags ignore snapshot knobs", wantWeight: 0, wantQuota: 0, wantBurst: 0, wantNoWatchdog: true}, + {name: "flags set the clone's policy", set: map[string]string{"cpu-weight": "10", "cpu-burst-us": "100000", "cpu-quota-us": "150000"}, wantWeight: 10, wantQuota: 150000, wantBurst: 100000, wantNoWatchdog: true}, {name: "invalid flag rejected", set: map[string]string{"cpu-weight": "20000"}, wantErr: true}, } for _, tt := range tests { @@ -135,6 +137,9 @@ func TestCloneVMConfigKnobFlagsOverrideSnapshot(t *testing.T) { t.Errorf("knobs = %d/%d/%d, want %d/%d/%d", got.CPUWeight, got.CPUQuotaUs, got.CPUBurstUs, tt.wantWeight, tt.wantQuota, tt.wantBurst) } + if got.NoWatchdog != tt.wantNoWatchdog { + t.Errorf("NoWatchdog = %v, want %v", got.NoWatchdog, tt.wantNoWatchdog) + } }) } } diff --git a/cmd/vm/commands.go b/cmd/vm/commands.go index 0e400f1c..76d2cb8a 100644 --- a/cmd/vm/commands.go +++ b/cmd/vm/commands.go @@ -336,6 +336,7 @@ func addVMFlags(cmd *cobra.Command) { cmd.Flags().String("user", "root", "guest username for cloud-init (cloudimg only)") cmd.Flags().String("password", "cocoon", "guest password for cloud-init (cloudimg only)") cmd.Flags().Bool("no-direct-io", false, "disable O_DIRECT on writable disks (use page cache instead; CH only)") + cmd.Flags().Bool("no-watchdog", false, "omit the virtio watchdog device (CH only; use when the guest driver cannot safely handle reboot)") cmd.Flags().Bool("windows", false, "Windows guest (UEFI boot, kvm_hyperv=on, no cidata)") cmd.Flags().Bool("shared-memory", false, "enable CH memory shared=on; required to attach vhost-user-fs later (CH only, fixed for VM lifetime)") cmd.Flags().Bool("hugepages", false, "back guest memory with hugetlbfs (CH only, fixed for VM lifetime); snapshots of such a VM restore via eager copy, never mmap") diff --git a/cmd/vm/debug.go b/cmd/vm/debug.go index a03b2a9f..194840d9 100644 --- a/cmd/vm/debug.go +++ b/cmd/vm/debug.go @@ -224,6 +224,8 @@ func printCommonCHArgs(s chDebugSpec) { if s.Balloon > 0 { fmt.Printf(" --balloon size=%dM,deflate_on_oom=on,free_page_reporting=on \\\n", s.Balloon) } - fmt.Print(" --watchdog \\\n") + if !s.VMCfg.NoWatchdog { + fmt.Print(" --watchdog \\\n") + } fmt.Println(" --serial tty --console off") } diff --git a/cmd/vm/run.go b/cmd/vm/run.go index c0836474..6a3a0898 100644 --- a/cmd/vm/run.go +++ b/cmd/vm/run.go @@ -328,6 +328,9 @@ func (h Handler) prepareClone(ctx context.Context, cmd *cobra.Command, conf *con if err = vmCfg.Validate(); err != nil { return cloneSetup{}, err } + if err = validateBackendFlags(conf, vmCfg); err != nil { + return cloneSetup{}, err + } // Envelope pins share create's digest-lock window; a record-backed clone's source pin already protects these. releasePins, err := cmdcore.PinEnvelopeBlobs(ctx, conf, cfg.ImageBlobIDs) if err != nil { @@ -476,7 +479,7 @@ func (h Handler) createVM(cmd *cobra.Command, image string) (context.Context, *t return ctx, info, hyper, nil } -// validateBackendFlags fast-fails flag combinations the selected backend can never launch; boot-mode-dependent checks live in validateBootCompat. Shared by create and debug so the capability gate list cannot drift. +// validateBackendFlags fast-fails flag combinations the selected backend can never launch; boot-mode-dependent checks live in validateBootCompat. Shared by create, clone, and debug so the capability gate list cannot drift. func validateBackendFlags(conf *config.Config, vmCfg *types.VMConfig) error { if !conf.UseFirecracker { return nil @@ -490,6 +493,8 @@ func validateBackendFlags(conf *config.Config, vmCfg *types.VMConfig) error { return fmt.Errorf("--fc and --hugepages are mutually exclusive: Firecracker cannot restore hugetlbfs-backed snapshots") case vmCfg.Mergeable: return fmt.Errorf("--fc and --mergeable are mutually exclusive: Firecracker has no KSM madvise knob") + case vmCfg.NoWatchdog: + return fmt.Errorf("--fc and --no-watchdog are mutually exclusive: Firecracker does not expose a virtio watchdog") } return nil } diff --git a/docs/cli.md b/docs/cli.md index 5942e879..db53ccc5 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -118,6 +118,7 @@ Applies to `cocoon vm create`, `cocoon vm run`, and `cocoon vm debug`: | `--user` | `root` | Guest username for cloud-init (cloudimg only) | | `--password` | `cocoon` | Guest password for cloud-init (cloudimg only) | | `--no-direct-io` | `false` | Disable O_DIRECT on writable disks (use page cache; CH only, useful for dev/test with few VMs) | +| `--no-watchdog` | `false` | Omit the virtio watchdog device (CH only; use for guests whose watchdog driver cannot safely handle reboot) | | `--data-disk` | empty (repeatable) | Attach an extra data disk: `size=20G[,name=...][,fstype=ext4|none][,mount=/mnt/x][,directio=on|off|auto]`. See [Data Disks](vm.md#data-disks) | | `--windows` | `false` | Windows guest (UEFI boot, kvm_hyperv=on, no cidata) | | `--shared-memory` | `false` | Enable CH `memory shared=on`; required for later `vm fs attach` (CH only, fixed for VM lifetime) | diff --git a/docs/known-issues.md b/docs/known-issues.md index 5a4ae08e..e134354b 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -171,6 +171,8 @@ virtio-win 0.1.240 did not have this problem because its balloon driver gave up **Workaround if balloon is needed**: increase `stop_timeout_seconds` to 180+ and apply the watchdog-pause patch to Cloud Hypervisor (pause the watchdog timer when `vm.power-button` is received). +**Workaround for guest-initiated reboot**: if the guest driver leaves the watchdog armed, create the VM with `--no-watchdog`. This trades device-level hang reset for reboot safety and should be paired with an external liveness policy where automatic recovery is required. + ## Installing patched binaries for Windows See [cocoonstack/windows](https://github.com/cocoonstack/windows) for download and installation instructions. diff --git a/docs/vm.md b/docs/vm.md index 21563558..e840bb55 100644 --- a/docs/vm.md +++ b/docs/vm.md @@ -89,7 +89,7 @@ With `cgroup_cpus=0-14`, the reserved core 15 has no VM competition and acts as - **Mergeable memory / KSM** (Cloud Hypervisor only): opt-in via `--mergeable` at golden creation; guest memory is madvised `MADV_MERGEABLE` so host KSM can dedup identical pages across VMs — the flag persists through snapshot/clone/restore (it lives in the snapshot's CH config, not the CLI), so build the golden with it or rebuild. cocoon only sets the madvise: enabling and tuning the scanner (`/sys/kernel/mm/ksm/run`, `pages_to_scan`) is the operator's. Excludes `--hugepages`/`--shared-memory` (KSM merges only plain private pages); mmap-cloned siblings already share untouched pages via the page cache, so KSM's gain is dirtied-but-equal and cross-golden pages — measure density on your fleet, and weigh ksmd CPU plus the cross-VM dedup timing side channel in multi-tenant setups - **Disk I/O**: multi-queue virtio-blk; readonly base disks keep host page cache (`direct=off`), writable raw COW and data disks use O_DIRECT (`direct=on`) to avoid host cache buildup and guest flush storms, and qcow2 overlays stay buffered — Cloud Hypervisor applies the disk's `direct` flag to the backing file too, and O_DIRECT there would give every VM its own read of the shared base instead of one page-cache copy - **Balloon**: 25% of memory auto-returned via virtio-balloon with deflate-on-OOM and free-page reporting (VMs with < 256 MiB memory skip balloon) -- **Watchdog**: hardware watchdog enabled by default for automatic guest reset on hang +- **Watchdog**: hardware watchdog enabled by default for automatic guest reset on hang; `--no-watchdog` is an explicit compatibility opt-out for guests whose watchdog driver is unsafe during reboot ## Cloud-init & First Boot diff --git a/hypervisor/cloudhypervisor/args.go b/hypervisor/cloudhypervisor/args.go index 09a0c0e1..24b64ddb 100644 --- a/hypervisor/cloudhypervisor/args.go +++ b/hypervisor/cloudhypervisor/args.go @@ -51,7 +51,7 @@ func buildVMConfig(rec *hypervisor.VMRecord, consoleSockPath string, allowed []i CPUs: chCPUs{BootVCPUs: cpu, MaxVCPUs: hypervisor.HostCPUCount(), KVMHyperV: rec.Config.Windows}, Memory: chMemory{Size: mem, HugePages: rec.Config.HugePages, Shared: rec.Config.SharedMemory, Mergeable: rec.Config.Mergeable}, RNG: chRNG{Src: "/dev/urandom"}, - Watchdog: true, + Watchdog: !rec.Config.NoWatchdog, Vsock: &chVsock{CID: hypervisor.VsockGuestCID, Socket: hypervisor.VsockSockPath(rec.RunDir)}, } diff --git a/hypervisor/cloudhypervisor/args_test.go b/hypervisor/cloudhypervisor/args_test.go index 5b1d6ed1..9a739139 100644 --- a/hypervisor/cloudhypervisor/args_test.go +++ b/hypervisor/cloudhypervisor/args_test.go @@ -54,6 +54,24 @@ func TestEffectiveDirectIO(t *testing.T) { } } +func TestWatchdogPolicy(t *testing.T) { + for _, tt := range []struct { + name string + noWatchdog bool + want bool + }{ + {name: "default enabled", want: true}, + {name: "explicitly disabled", noWatchdog: true, want: false}, + } { + t.Run(tt.name, func(t *testing.T) { + rec := &hypervisor.VMRecord{VM: types.VM{Config: types.VMConfig{Config: types.Config{NoWatchdog: tt.noWatchdog}}}} + if got := buildVMConfig(rec, "", nil).Watchdog; got != tt.want { + t.Fatalf("Watchdog = %v, want %v", got, tt.want) + } + }) + } +} + func TestQcow2OverlayDiskArgs(t *testing.T) { sc := &types.StorageConfig{Path: "/v/overlay.qcow2", Role: types.StorageRoleCOW} got := diskToCLIArg(storageConfigToDisk(sc, 1, 0, false, nil)) diff --git a/types/config.go b/types/config.go index 92d224a5..6e6bb3cb 100644 --- a/types/config.go +++ b/types/config.go @@ -18,6 +18,7 @@ type Config struct { ImageType string `json:"image_type,omitempty"` // backend type, ImageTypeOCI / ImageTypeCloudImg Network string `json:"network,omitempty"` // CNI conflist name; empty = default NoDirectIO bool `json:"no_direct_io,omitempty"` // disable O_DIRECT on writable disks + NoWatchdog bool `json:"no_watchdog,omitempty"` // omit the virtio watchdog device (guest/driver compatibility) Windows bool `json:"windows,omitempty"` // Windows guest: UEFI boot, kvm_hyperv=on, no cidata // SharedMemory toggles CH memory shared=on (vhost-user-fs prerequisite); fixed at create, persists through clone/restore. SharedMemory bool `json:"shared_memory,omitempty"`