Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cmd/core/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,10 @@ func RouteRefs(ctx context.Context, hypers []hypervisor.Hypervisor, refs []strin
return result, nil
}

// ReconcileState returns the effective display state; a stale-running VM also loses its runtime paths — they died with the process, and a reused PTY number must not be advertised.
func ReconcileState(vm *types.VM) string {
if vm.State == types.VMStateRunning && !utils.IsProcessAlive(vm.PID) {
vm.SocketPath, vm.VsockSocket, vm.ConsolePath = "", "", ""
return "stopped (stale)"
}
return string(vm.State)
Expand Down
25 changes: 25 additions & 0 deletions cmd/core/utils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,31 @@ func TestPersistSnapshotDirCleansCaptureOnDirectError(t *testing.T) {
}
}

func TestReconcileStateClearsRuntimePathsOnStaleRunning(t *testing.T) {
vm := &types.VM{
State: types.VMStateRunning,
PID: 0,
SocketPath: "/run/api.sock",
VsockSocket: "/run/vsock.uds",
ConsolePath: "/dev/pts/3",
}
if got := ReconcileState(vm); got != "stopped (stale)" {
t.Fatalf("ReconcileState = %q, want %q", got, "stopped (stale)")
}
if vm.SocketPath != "" || vm.VsockSocket != "" || vm.ConsolePath != "" {
t.Errorf("stale-running VM keeps runtime paths: socket=%q vsock=%q console=%q",
vm.SocketPath, vm.VsockSocket, vm.ConsolePath)
}

alive := &types.VM{State: types.VMStateRunning, PID: os.Getpid(), ConsolePath: "/dev/pts/3"}
if got := ReconcileState(alive); got != string(types.VMStateRunning) {
t.Fatalf("ReconcileState = %q, want running", got)
}
if alive.ConsolePath == "" {
t.Error("live VM lost its console path")
}
}

// directErrSnap is a DirectCreator whose CreateFromDir always fails; the embedded interface panics on any other call, so the test proves the failure path alone.
type directErrSnap struct {
snapshot.Snapshot
Expand Down
3 changes: 1 addition & 2 deletions cmd/core/vmconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,6 @@ func CloneVMConfigFromFlags(cmd *cobra.Command, snapCfg types.SnapshotConfig) (*
if cmd.Flags().Changed("no-direct-io") {
noDirectIO, _ = cmd.Flags().GetBool("no-direct-io")
}
noWatchdog := snapCfg.NoWatchdog

restoreMode, err := restoreModeFromFlags(cmd)
if err != nil {
Expand All @@ -133,7 +132,7 @@ func CloneVMConfigFromFlags(cmd *cobra.Command, snapCfg types.SnapshotConfig) (*
ImageType: snapCfg.ImageType,
Network: network,
NoDirectIO: noDirectIO,
NoWatchdog: noWatchdog,
NoWatchdog: snapCfg.NoWatchdog,
Windows: snapCfg.Windows,
SharedMemory: snapCfg.SharedMemory,
HugePages: snapCfg.HugePages,
Expand Down
5 changes: 1 addition & 4 deletions cmd/vm/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -328,9 +328,6 @@ func (h Handler) prepareClone(ctx context.Context, cmd *cobra.Command, conf *con
if err = vmCfg.Validate(); err != nil {
return cloneSetup{}, err
}
if err = validateBackendFlags(conf, vmCfg); err != nil {
return cloneSetup{}, err
}
// Envelope pins share create's digest-lock window; a record-backed clone's source pin already protects these.
releasePins, err := cmdcore.PinEnvelopeBlobs(ctx, conf, cfg.ImageBlobIDs)
if err != nil {
Expand Down Expand Up @@ -479,7 +476,7 @@ func (h Handler) createVM(cmd *cobra.Command, image string) (context.Context, *t
return ctx, info, hyper, nil
}

// validateBackendFlags fast-fails flag combinations the selected backend can never launch; boot-mode-dependent checks live in validateBootCompat. Shared by create, clone, and debug so the capability gate list cannot drift.
// validateBackendFlags fast-fails flag combinations the selected backend can never launch; boot-mode-dependent checks live in validateBootCompat. Shared by create and debug so the capability gate list cannot drift.
func validateBackendFlags(conf *config.Config, vmCfg *types.VMConfig) error {
if !conf.UseFirecracker {
return nil
Expand Down
4 changes: 4 additions & 0 deletions cmd/vm/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,10 @@ func statusOnce(ctx context.Context, hypers []hypervisor.Hypervisor, filters []s
}
vms = applyFilters(vms, filters)
sortVMs(vms)
// JSON serializes vms as-is, so stale-running records must reconcile here, not per output row.
for _, vm := range vms {
vm.State = types.VMState(cmdcore.ReconcileState(vm))
}
return renderVMList(vms, format, scopeDir)
}

Expand Down
2 changes: 2 additions & 0 deletions docs/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,8 @@ Applies to `cocoon vm debug`:
| ---------------- | -------- | ------------------------------------------------- |
| `--escape-char` | `^]` | Escape character (single char or `^X` caret notation) |

For a running VM, `cocoon vm inspect` reports the console resolved at boot (start, clone, restore) as `console_path`: the `console.sock` UDS (UEFI serial, Firecracker relay) or the Cloud Hypervisor-allocated PTY (`/dev/pts/N`, direct-boot OCI). External supervisors can read the console from there without opening the owner-only API socket. A direct-boot VM booted by an older cocoon reports it from its next start.

### Exec Flags

`cocoon vm exec` runs a command inside a running VM via the cocoon-agent (vsock, no SSH). Stdin/stdout/stderr stream like `kubectl exec`; the host shell sees the guest command's exit code.
Expand Down
1 change: 1 addition & 0 deletions hypervisor/backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
const (
APISocketName = "api.sock"
ConsoleSockName = "console.sock"
ConsolePTYName = "console.pty"
VsockSockName = "vsock.uds"

// VsockGuestCID is constant — per-VM isolation comes from distinct UDS paths.
Expand Down
1 change: 1 addition & 0 deletions hypervisor/cloudhypervisor/clone.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ func (ch *CloudHypervisor) cloneAfterExtractParsed(ctx context.Context, vmID str
}); err != nil {
return nil, err
}
saveConsolePTY(ctx, vmID, runDir, sockPath, directBoot)

info := &types.VM{
ID: vmID, Hypervisor: typ, State: types.VMStateRunning,
Expand Down
1 change: 1 addition & 0 deletions hypervisor/cloudhypervisor/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ func (ch *CloudHypervisor) restoreAfterExtract(ctx context.Context, vmID string,
if err = resumeVM(ctx, hc); err != nil {
return nil, fmt.Errorf("vm.resume: %w", err)
}
saveConsolePTY(ctx, vmID, rec.RunDir, sockPath, directBoot)

logger.Infof(ctx, "VM %s restored from snapshot", vmID)
return ch.FinalizeRestore(ctx, vmID, vmCfg, rec, pid)
Expand Down
4 changes: 4 additions & 0 deletions hypervisor/cloudhypervisor/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ func (ch *CloudHypervisor) startOne(ctx context.Context, id string) error {
ch.saveCmdline(ctx, rec, args)
return ch.launchProcess(ctx, rec, args, rec.ResolvedNetnsPath(), false)
},
PostLaunch: func(ctx context.Context, rec *hypervisor.VMRecord, sockPath string, _ int) error {
saveConsolePTY(ctx, rec.ID, rec.RunDir, sockPath, hypervisor.IsDirectBoot(rec.BootConfig))
return nil
},
})
}

Expand Down
16 changes: 15 additions & 1 deletion hypervisor/cloudhypervisor/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ const (
chMemoryRestoreMmap chMemoryRestoreMode = "CopyOnWrite"
)

var runtimeFiles = []string{hypervisor.APISocketName, pidFileName, hypervisor.ConsoleSockName, cmdlineFileName, hypervisor.VsockSockName}
var runtimeFiles = []string{hypervisor.APISocketName, pidFileName, hypervisor.ConsoleSockName, hypervisor.ConsolePTYName, cmdlineFileName, hypervisor.VsockSockName}

type chMemoryRestoreMode string

Expand Down Expand Up @@ -288,6 +288,20 @@ func resolveConsole(ctx context.Context, vmID, sockPath, consoleSock string, dir
return consoleSock
}

// saveConsolePTY records the per-boot PTY path for vm inspect; best-effort — the guest is up either way.
func saveConsolePTY(ctx context.Context, vmID, runDir, sockPath string, directBoot bool) {
if !directBoot {
return
}
pty := resolveConsole(ctx, vmID, sockPath, "", true)
if pty == "" {
return
}
if err := utils.AtomicWriteFileNoSync(hypervisor.ConsolePTYPath(runDir), []byte(pty), 0o600); err != nil {
log.WithFunc("cloudhypervisor.saveConsolePTY").Warnf(ctx, "save console PTY for %s: %v", vmID, err)
}
}

// qemuExpandImage grows a qcow2 disk to targetSize iff its virtual size is smaller.
func qemuExpandImage(ctx context.Context, path string, targetSize int64) error {
hdr, ok, err := utils.ReadQcow2Header(path)
Expand Down
56 changes: 56 additions & 0 deletions hypervisor/cloudhypervisor/utils_test.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
package cloudhypervisor

import (
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"testing"

"github.com/cocoonstack/cocoon/hypervisor"
"github.com/cocoonstack/cocoon/utils"
)

Expand Down Expand Up @@ -37,3 +42,54 @@ func TestIsAlreadyInStateError(t *testing.T) {
})
}
}

func TestSaveConsolePTYWritesQueriedPath(t *testing.T) {
runDir := t.TempDir()
sockPath := serveVMInfo(t, "/dev/pts/7")

saveConsolePTY(t.Context(), "vm1", runDir, sockPath, true)

got, err := os.ReadFile(hypervisor.ConsolePTYPath(runDir))
if err != nil {
t.Fatalf("read console.pty: %v", err)
}
if string(got) != "/dev/pts/7" {
t.Errorf("console.pty = %q, want %q", got, "/dev/pts/7")
}
}

func TestSaveConsolePTYSkipsUEFI(t *testing.T) {
runDir := t.TempDir()

saveConsolePTY(t.Context(), "vm1", runDir, filepath.Join(runDir, "api.sock"), false)

if utils.FileExists(hypervisor.ConsolePTYPath(runDir)) {
t.Error("console.pty written for a UEFI boot")
}
}

func serveVMInfo(t *testing.T, ptyPath string) string {
t.Helper()
// os.MkdirTemp, not t.TempDir: unix socket paths cap at ~104 bytes.
sockDir, err := os.MkdirTemp("", "ch")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.RemoveAll(sockDir) })
sockPath := filepath.Join(sockDir, "api.sock")
ln, err := net.Listen("unix", sockPath)
if err != nil {
t.Fatalf("listen %s: %v", sockPath, err)
}
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/vm.info", func(w http.ResponseWriter, _ *http.Request) {
resp := chVMInfoResponse{Config: chVMInfoConfig{Console: chRuntimeFile{Mode: "Pty", File: ptyPath}}}
if err := json.NewEncoder(w).Encode(resp); err != nil {
t.Errorf("encode vm.info: %v", err)
}
})
srv := &http.Server{Handler: mux}
go srv.Serve(ln) //nolint:errcheck
t.Cleanup(func() { _ = srv.Close() })
return sockPath
}
21 changes: 16 additions & 5 deletions hypervisor/inspect.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"maps"
"os"
"slices"
"strings"

"github.com/cocoonstack/cocoon/types"
"github.com/cocoonstack/cocoon/utils"
Expand Down Expand Up @@ -48,6 +49,8 @@ func (b *Backend) List(ctx context.Context) ([]*types.VM, error) {
func (b *Backend) ToVM(rec *VMRecord) *types.VM {
info := rec.VM
info.Hypervisor = b.Typ
// Clear first: clone/restore persist boot-time sockets into the record, which would leak as stale paths once the VM stops.
info.SocketPath, info.VsockSocket, info.ConsolePath = "", "", ""
if info.State == types.VMStateRunning {
SetRunningSockets(&info, rec.RunDir)
info.PID, _ = utils.ReadPIDFile(b.PIDFilePath(rec.RunDir))
Expand Down Expand Up @@ -128,15 +131,23 @@ func (b *Backend) UpdateRecord(ctx context.Context, vmID string, mutate func(*VM
})
}

// SetRunningSockets fills a running VM's live sockets (API socket, bound vsock UDS) from runDir — for clone/restore records that skip ToVM.
// SetRunningSockets fills a running VM's live sockets (API socket, bound vsock UDS, guest console) from runDir — for clone/restore records that skip ToVM.
func SetRunningSockets(info *types.VM, runDir string) {
info.SocketPath = SocketPath(runDir)
if p := VsockSockPath(runDir); isVsockBound(p) {
if p := VsockSockPath(runDir); utils.FileExists(p) {
info.VsockSocket = p
}
info.ConsolePath = consolePathFromRunDir(runDir)
}

func isVsockBound(path string) bool {
_, err := os.Stat(path)
return err == nil
// consolePathFromRunDir prefers the console UDS (UEFI serial, FC relay); direct-boot CH VMs instead leave the PTY path saved at boot.
func consolePathFromRunDir(runDir string) string {
if p := ConsoleSockPath(runDir); utils.FileExists(p) {
return p
}
pty, err := os.ReadFile(ConsolePTYPath(runDir))
if err != nil {
return ""
}
return strings.TrimSpace(string(pty))
}
82 changes: 82 additions & 0 deletions hypervisor/inspect_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package hypervisor

import (
"os"
"testing"

"github.com/cocoonstack/cocoon/types"
)

func TestToVMClearsPersistedRuntimeFieldsWhenStopped(t *testing.T) {
b, _ := newMeteringTestBackend(t)
seedVMRecord(t, b, "vm1", 1, 1<<30, 10<<30, true)
if err := b.dbUpdate(t.Context(), func(idx *VMIndex) error {
idx.VMs["vm1"].State = types.VMStateStopped
idx.VMs["vm1"].SocketPath = "/stale/api.sock"
idx.VMs["vm1"].VsockSocket = "/stale/vsock.uds"
idx.VMs["vm1"].ConsolePath = "/dev/pts/9"
return nil
}); err != nil {
t.Fatalf("seed runtime fields: %v", err)
}

rec, err := b.LoadRecord(t.Context(), "vm1")
if err != nil {
t.Fatalf("load record: %v", err)
}
info := b.ToVM(&rec)
if info.SocketPath != "" || info.VsockSocket != "" || info.ConsolePath != "" {
t.Errorf("stopped VM leaks runtime paths: socket=%q vsock=%q console=%q",
info.SocketPath, info.VsockSocket, info.ConsolePath)
}
}

func TestToVMRunningConsolePath(t *testing.T) {
tests := []struct {
name string
setup func(t *testing.T, runDir string)
want func(runDir string) string
}{
{
name: "console sock",
setup: func(t *testing.T, runDir string) { writeRunFile(t, ConsoleSockPath(runDir), "") },
want: ConsoleSockPath,
},
{
name: "pty file",
setup: func(t *testing.T, runDir string) { writeRunFile(t, ConsolePTYPath(runDir), "/dev/pts/5\n") },
want: func(string) string { return "/dev/pts/5" },
},
{
name: "no console",
setup: func(*testing.T, string) {},
want: func(string) string { return "" },
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b, _ := newMeteringTestBackend(t)
seedRunningVM(t, b, "vm1", 1, 1<<30, 10<<30)
rec, err := b.LoadRecord(t.Context(), "vm1")
if err != nil {
t.Fatalf("load record: %v", err)
}
tt.setup(t, rec.RunDir)

info := b.ToVM(&rec)
if want := tt.want(rec.RunDir); info.ConsolePath != want {
t.Errorf("ConsolePath = %q, want %q", info.ConsolePath, want)
}
if info.SocketPath != SocketPath(rec.RunDir) {
t.Errorf("SocketPath = %q, want %q", info.SocketPath, SocketPath(rec.RunDir))
}
})
}
}

func writeRunFile(t *testing.T, path, content string) {
t.Helper()
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatalf("write %s: %v", path, err)
}
}
2 changes: 2 additions & 0 deletions hypervisor/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,8 @@ func SocketPath(runDir string) string { return filepath.Join(runDir, APISocketNa

func ConsoleSockPath(runDir string) string { return filepath.Join(runDir, ConsoleSockName) }

func ConsolePTYPath(runDir string) string { return filepath.Join(runDir, ConsolePTYName) }

func VsockSockPath(runDir string) string { return filepath.Join(runDir, VsockSockName) }

// BalloonSize returns (bytes, enabled); disabled on Windows (virtio-win driver loops on deflation) and below MinBalloonMemory.
Expand Down
1 change: 1 addition & 0 deletions types/vm.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ type VM struct {
PID int `json:"pid"`
SocketPath string `json:"socket_path,omitempty"` // CH API Unix socket
VsockSocket string `json:"vsock_socket,omitempty"` // hybrid vsock UDS for cocoon-agent
ConsolePath string `json:"console_path,omitempty"` // guest console: console.sock UDS or CH-allocated PTY (direct boot)

NetSetup

Expand Down