diff --git a/Makefile b/Makefile index 770d44e8..08c14fa8 100644 --- a/Makefile +++ b/Makefile @@ -295,14 +295,12 @@ test-linux: ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries if [ -n "$(TEST)" ]; then \ echo "Running specific test: $(TEST)"; \ sudo env "PATH=$$TEST_PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" "CI=$${CI:-}" \ - "HYPEMAN_TEST_NETWORK_TMPDIR=$${HYPEMAN_TEST_NETWORK_TMPDIR:-}" \ "HYPEMAN_TEST_PREWARM_DIR=$${HYPEMAN_TEST_PREWARM_DIR:-}" \ "HYPEMAN_TEST_PREWARM_STRICT=$${HYPEMAN_TEST_PREWARM_STRICT:-}" \ "HYPEMAN_TEST_REGISTRY=$${HYPEMAN_TEST_REGISTRY:-}" \ go test -tags containers_image_openpgp -run=$(TEST) $$VERBOSE_FLAG -timeout=$(TEST_TIMEOUT) ./...; \ else \ sudo env "PATH=$$TEST_PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" "CI=$${CI:-}" \ - "HYPEMAN_TEST_NETWORK_TMPDIR=$${HYPEMAN_TEST_NETWORK_TMPDIR:-}" \ "HYPEMAN_TEST_PREWARM_DIR=$${HYPEMAN_TEST_PREWARM_DIR:-}" \ "HYPEMAN_TEST_PREWARM_STRICT=$${HYPEMAN_TEST_PREWARM_STRICT:-}" \ "HYPEMAN_TEST_REGISTRY=$${HYPEMAN_TEST_REGISTRY:-}" \ diff --git a/cmd/api/api/snapshots.go b/cmd/api/api/snapshots.go index 00b6d06b..620d9574 100644 --- a/cmd/api/api/snapshots.go +++ b/cmd/api/api/snapshots.go @@ -175,9 +175,7 @@ func (s *ApiService) ForkSnapshot(ctx context.Context, request oapi.ForkSnapshot return oapi.ForkSnapshot400JSONResponse{Code: "invalid_request", Message: "request body is required"}, nil } - domainReq := instances.ForkSnapshotRequest{ - Name: request.Body.Name, - } + domainReq := instances.ForkSnapshotRequest{Name: request.Body.Name} if request.Body.TargetState != nil { domainReq.TargetState = instances.State(*request.Body.TargetState) } diff --git a/cmd/api/api/snapshots_test.go b/cmd/api/api/snapshots_test.go index 1cfaf1d8..d0730133 100644 --- a/cmd/api/api/snapshots_test.go +++ b/cmd/api/api/snapshots_test.go @@ -1,35 +1,14 @@ package api import ( - "context" "testing" "time" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances" - "github.com/kernel/hypeman/lib/oapi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -type captureForkSnapshotManager struct { - instances.Manager - lastID string - lastReq *instances.ForkSnapshotRequest - result *instances.Instance - err error -} - -func (m *captureForkSnapshotManager) ForkSnapshot(ctx context.Context, snapshotID string, req instances.ForkSnapshotRequest) (*instances.Instance, error) { - reqCopy := req - m.lastID = snapshotID - m.lastReq = &reqCopy - if m.err != nil { - return nil, m.err - } - return m.result, nil -} - func TestSnapshotScheduleToOAPIPreservesZeroMaxCount(t *testing.T) { t.Parallel() @@ -51,39 +30,3 @@ func TestSnapshotScheduleToOAPIPreservesZeroMaxCount(t *testing.T) { require.NotNil(t, out.Retention.MaxAge) assert.Equal(t, "24h0m0s", *out.Retention.MaxAge) } - -func TestForkSnapshotSuccess(t *testing.T) { - t.Parallel() - svc := newTestService(t) - - forked := &instances.Instance{ - StoredMetadata: instances.StoredMetadata{ - Id: "forked-instance", - Name: "forked-instance", - Image: "docker.io/library/alpine:latest", - CreatedAt: time.Now(), - HypervisorType: hypervisor.TypeFirecracker, - }, - State: instances.StateRunning, - } - mockMgr := &captureForkSnapshotManager{ - Manager: svc.InstanceManager, - result: forked, - } - svc.InstanceManager = mockMgr - - resp, err := svc.ForkSnapshot(ctx(), oapi.ForkSnapshotRequestObject{ - SnapshotId: "snap-123", - Body: &oapi.ForkSnapshotRequest{ - Name: "forked-instance", - }, - }) - require.NoError(t, err) - - created, ok := resp.(oapi.ForkSnapshot201JSONResponse) - require.True(t, ok, "expected 201 response") - assert.Equal(t, "forked-instance", created.Name) - assert.Equal(t, "snap-123", mockMgr.lastID) - require.NotNil(t, mockMgr.lastReq) - assert.Equal(t, "forked-instance", mockMgr.lastReq.Name) -} diff --git a/lib/forkvm/README.md b/lib/forkvm/README.md index d224b220..bb6567c7 100644 --- a/lib/forkvm/README.md +++ b/lib/forkvm/README.md @@ -14,23 +14,6 @@ to work across implementations. For networked forks, the fork gets a fresh host/guest identity (IP, MAC, TAP) instead of reusing the source identity. -## Resume network handoff - -Networked standby/running forks need a new host-side allocation, but the guest -memory snapshot still contains the source VM's old interface state. On restore, -Hypeman prepares the fork's TAP/IP/MAC before the VM resumes, then hands the new -guest network config to the guest through a small mailbox embedded in snapshot -memory. After resume, VMGenID tells the guest-agent that this is a restored VM; -the guest-agent reads the mailbox and applies the new MAC, address, route, and -neighbor state with netlink. - -For API calls that return a running fork, Hypeman waits for a guest UDP -"applied" ack before returning, so the fast path still avoids making -host-initiated guest RPC/vsock contact as the first post-resume dependency. If -the mailbox path is unavailable, cannot start its ack listener, or does not ack -in time, restore falls back to the older host-initiated guest network -reconfigure path. - ## Fork data copy behavior - Guest directory copy is **sparse-only** for regular files. diff --git a/lib/instances/create.go b/lib/instances/create.go index e2823c5e..df991674 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -474,8 +474,6 @@ func (m *manager) createInstance( stored.Volumes = req.Volumes } - ensureGuestInitiatedResumeNetworkMailbox(stored) - // 16. Create config disk (needs Instance for buildVMConfig) inst := &Instance{StoredMetadata: *stored} var proxyGuestConfig *egressproxy.GuestConfig diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 1268d90c..606b7671 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -319,24 +319,6 @@ func TestApplyForkTargetStateStoppedRefreshesSnapshotForkCID(t *testing.T) { assert.Equal(t, generateVsockCID(forkID), updated.StoredMetadata.VsockCID) } -func TestForkReturnReadinessDoesNotUseMailboxEligibility(t *testing.T) { - t.Parallel() - - stored := StoredMetadata{ - HypervisorType: hypervisor.TypeFirecracker, - NetworkEnabled: true, - } - ensureGuestInitiatedResumeNetworkMailbox(&stored) - require.True(t, guestInitiatedResumeNetworkMailbox(&stored)) - - inst := &Instance{ - StoredMetadata: stored, - State: StateInitializing, - } - assert.True(t, forkReturnNeedsGuestAgentReady(inst, false)) - assert.False(t, forkReturnNeedsGuestAgentReady(inst, true)) -} - func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { t.Parallel() startedAt := time.Now().Add(-2 * time.Minute) diff --git a/lib/instances/guest_resume_network.go b/lib/instances/guest_resume_network.go deleted file mode 100644 index 0d880eb6..00000000 --- a/lib/instances/guest_resume_network.go +++ /dev/null @@ -1,235 +0,0 @@ -package instances - -import ( - "bytes" - "context" - "encoding/binary" - "fmt" - stdnet "net" - "os" - "path/filepath" - "strings" - "sync" - "time" - - "github.com/kernel/hypeman/lib/hypervisor" - "github.com/kernel/hypeman/lib/logger" - "github.com/kernel/hypeman/lib/mailbox" - "github.com/nrednav/cuid2" - "go.opentelemetry.io/otel/attribute" - "golang.org/x/sys/unix" -) - -const firecrackerSnapshotMemoryFile = "memory" - -const guestResumeNetworkUDPAckTimeout = 5 * time.Second - -var guestResumeNetworkMailboxOffsets sync.Map - -type guestResumeNetworkUDPAck struct { - received time.Time - text string -} - -type guestResumeNetworkUDPWaiter struct { - conn *stdnet.UDPConn - ch chan guestResumeNetworkUDPAck -} - -func guestInitiatedResumeNetworkMailbox(stored *StoredMetadata) bool { - token := guestInitiatedResumeNetworkMailboxToken(stored) - return stored != nil && - stored.HypervisorType == hypervisor.TypeFirecracker && - strings.TrimSpace(stored.Env[mailbox.MailboxEnv]) == "1" && - mailbox.ValidToken(token) -} - -func guestInitiatedResumeNetworkMailboxToken(stored *StoredMetadata) string { - if stored == nil { - return "" - } - return strings.TrimSpace(stored.Env[mailbox.MailboxTokenEnv]) -} - -func ensureGuestInitiatedResumeNetworkMailbox(stored *StoredMetadata) { - if stored == nil || - stored.HypervisorType != hypervisor.TypeFirecracker || - !stored.NetworkEnabled || - stored.SkipGuestAgent { - return - } - if stored.Env == nil { - stored.Env = make(map[string]string) - } - stored.Env[mailbox.MailboxEnv] = "1" - if token := guestInitiatedResumeNetworkMailboxToken(stored); !mailbox.ValidToken(token) { - stored.Env[mailbox.MailboxTokenEnv] = cuid2.Generate() - } -} - -func newGuestResumeNetworkPayload(cfg *guestNetworkConfig) mailbox.Payload { - return mailbox.Payload{ - InterfaceName: "eth0", - MAC: cfg.mac, - IPv4: cfg.ip, - Prefix: uint32(cfg.prefix), - Gateway: cfg.gateway, - } -} - -func startGuestResumeNetworkUDPWaiter() (*guestResumeNetworkUDPWaiter, error) { - conn, err := stdnet.ListenUDP("udp4", &stdnet.UDPAddr{IP: stdnet.IPv4zero, Port: 0}) - if err != nil { - return nil, fmt.Errorf("listen for guest resume network UDP ack: %w", err) - } - - w := &guestResumeNetworkUDPWaiter{ - conn: conn, - ch: make(chan guestResumeNetworkUDPAck, 128), - } - go w.readLoop() - return w, nil -} - -func (w *guestResumeNetworkUDPWaiter) Port() uint32 { - if w == nil || w.conn == nil { - return 0 - } - return uint32(w.conn.LocalAddr().(*stdnet.UDPAddr).Port) -} - -func (w *guestResumeNetworkUDPWaiter) Close() { - if w == nil || w.conn == nil { - return - } - _ = w.conn.Close() -} - -func (w *guestResumeNetworkUDPWaiter) readLoop() { - buf := make([]byte, 1024) - for { - n, _, err := w.conn.ReadFromUDP(buf) - if err != nil { - return - } - w.ch <- guestResumeNetworkUDPAck{ - received: time.Now(), - text: strings.TrimSpace(string(buf[:n])), - } - } -} - -func (w *guestResumeNetworkUDPWaiter) WaitApplied(ctx context.Context, mac, ip string) (time.Duration, string, error) { - if w == nil { - return 0, "", fmt.Errorf("guest resume network UDP waiter is nil") - } - - start := time.Now() - wantMAC := "mac=" + strings.ToLower(mac) - wantIP := "ip=" + ip - for { - select { - case ack := <-w.ch: - text := strings.ToLower(ack.text) - if strings.Contains(text, "stage=applied") && strings.Contains(text, wantMAC) && strings.Contains(text, wantIP) { - return ack.received.Sub(start), ack.text, nil - } - case <-ctx.Done(): - return 0, "", ctx.Err() - } - } -} - -func (m *manager) waitForGuestResumeNetworkUDPAck(ctx context.Context, waiter *guestResumeNetworkUDPWaiter, stored *StoredMetadata, cfg *guestNetworkConfig) error { - if waiter == nil || cfg == nil { - return nil - } - - log := logger.FromContext(ctx) - waitCtx, waitSpanEnd := m.startLifecycleStep(ctx, "guest.resume_network.udp_ack_wait", - attribute.String("instance_id", stored.Id), - attribute.String("hypervisor", string(stored.HypervisorType)), - attribute.String("operation", "guest_resume_network_udp_ack_wait"), - ) - waitCtx, cancel := context.WithTimeout(waitCtx, guestResumeNetworkUDPAckTimeout) - defer cancel() - - elapsed, ack, err := waiter.WaitApplied(waitCtx, cfg.mac, cfg.ip) - waitSpanEnd(err) - if err != nil { - return err - } - log.InfoContext(ctx, "guest resume network UDP ack received", "instance_id", stored.Id, "elapsed", elapsed, "ack", ack) - return nil -} - -func patchGuestResumeNetworkMailbox(snapshotDir, token string, payload *mailbox.Payload) error { - payloadBytes, err := mailbox.MarshalPayload(payload) - if err != nil { - return err - } - - file, err := os.OpenFile(filepath.Join(snapshotDir, firecrackerSnapshotMemoryFile), os.O_RDWR, 0) - if err != nil { - return fmt.Errorf("open snapshot memory for resume network mailbox: %w", err) - } - defer file.Close() - - info, err := file.Stat() - if err != nil { - return fmt.Errorf("stat snapshot memory for resume network mailbox: %w", err) - } - if info.Size() <= 0 { - return fmt.Errorf("resume network mailbox memory file is empty") - } - - marker, err := mailbox.Marker(token) - if err != nil { - return err - } - idx, err := findGuestResumeNetworkMailbox(file, info.Size(), marker, token) - if err != nil { - return err - } - if idx+int64(mailbox.MailboxPayloadOffset)+int64(len(payloadBytes)) > info.Size() { - return fmt.Errorf("resume network mailbox marker is too close to end of memory file") - } - - if _, err := file.WriteAt(payloadBytes, idx+int64(mailbox.MailboxPayloadOffset)); err != nil { - return fmt.Errorf("write resume network mailbox payload: %w", err) - } - var u32 [4]byte - binary.LittleEndian.PutUint32(u32[:], uint32(len(payloadBytes))) - if _, err := file.WriteAt(u32[:], idx+int64(mailbox.MailboxLengthOffset)); err != nil { - return fmt.Errorf("write resume network mailbox payload length: %w", err) - } - binary.LittleEndian.PutUint32(u32[:], 1) - if _, err := file.WriteAt(u32[:], idx+int64(mailbox.MailboxSeqOffset)); err != nil { - return fmt.Errorf("write resume network mailbox sequence: %w", err) - } - return nil -} - -func findGuestResumeNetworkMailbox(file *os.File, size int64, marker []byte, token string) (int64, error) { - if cached, ok := guestResumeNetworkMailboxOffsets.Load(token); ok { - if offset, ok := cached.(int64); ok && offset >= 0 && offset+int64(len(marker)) <= size { - buf := make([]byte, len(marker)) - if _, err := file.ReadAt(buf, offset); err == nil && bytes.Equal(buf, marker) { - return offset, nil - } - } - } - - data, err := unix.Mmap(int(file.Fd()), 0, int(size), unix.PROT_READ, unix.MAP_SHARED) - if err != nil { - return 0, fmt.Errorf("mmap snapshot memory for resume network mailbox: %w", err) - } - defer unix.Munmap(data) - - idx := bytes.Index(data, marker) - if idx < 0 { - return 0, fmt.Errorf("resume network mailbox marker not found") - } - guestResumeNetworkMailboxOffsets.Store(token, int64(idx)) - return int64(idx), nil -} diff --git a/lib/instances/restore.go b/lib/instances/restore.go index c8645e0c..d2db078c 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -248,14 +248,6 @@ func (m *manager) restoreInstance( proxyRegistered = true } - resumeNetworkHandoff, err := m.prepareResumeNetworkHandoff(ctx, stored, allocatedNet, snapshotDir) - if err != nil { - log.ErrorContext(ctx, "failed to prepare guest resume network handoff", "instance_id", id, "error", err) - releaseNetwork() - return nil, fmt.Errorf("prepare guest resume network handoff: %w", err) - } - defer resumeNetworkHandoff.Close() - // 5. Transition: Standby → Paused (start hypervisor + restore) restoreCtx, restoreSpanEnd := m.startLifecycleStep(ctx, "restore_from_snapshot", attribute.String("instance_id", id), @@ -310,15 +302,15 @@ func (m *manager) restoreInstance( attribute.String("hypervisor", string(stored.HypervisorType)), attribute.String("operation", "reconfigure_guest_network"), ) - reconfigureErr := resumeNetworkHandoff.AfterResume(reconfigureCtx) - reconfigureSpanEnd(reconfigureErr) - if reconfigureErr != nil { - log.ErrorContext(ctx, "failed to configure guest network after restore", "instance_id", id, "error", reconfigureErr) + if err := reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet); err != nil { + reconfigureSpanEnd(err) + log.ErrorContext(ctx, "failed to configure guest network after restore", "instance_id", id, "error", err) _ = hv.Shutdown(ctx) m.rollbackAdmissionAllocationActive(stored) releaseNetwork() - return nil, fmt.Errorf("configure guest network after restore: %w", reconfigureErr) + return nil, fmt.Errorf("configure guest network after restore: %w", err) } + reconfigureSpanEnd(nil) } // 8. Delete snapshot after successful restore unless the hypervisor is keeping it diff --git a/lib/instances/restore_egress_test.go b/lib/instances/restore_egress_test.go index c3983780..f65b0eb1 100644 --- a/lib/instances/restore_egress_test.go +++ b/lib/instances/restore_egress_test.go @@ -1,13 +1,8 @@ package instances import ( - "encoding/binary" - "encoding/json" - "os" "testing" - "github.com/kernel/hypeman/lib/hypervisor" - "github.com/kernel/hypeman/lib/mailbox" "github.com/kernel/hypeman/lib/network" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -35,25 +30,7 @@ func TestNetworkConfigFromAllocation_PreservesDNS(t *testing.T) { assert.Equal(t, alloc.TAPDevice, cfg.TAPDevice) } -func TestGuestNetworkReconfigureConfig_AppliesAllocatedMAC(t *testing.T) { - t.Parallel() - - alloc := &network.Allocation{ - IP: "10.102.146.62", - MAC: "02:00:00:85:17:c8", - Gateway: "10.102.0.1", - Netmask: "255.255.0.0", - } - - cfg, err := guestNetworkReconfigureConfig(alloc) - require.NoError(t, err) - assert.Equal(t, "10.102.146.62", cfg.ip) - assert.Equal(t, "02:00:00:85:17:c8", cfg.mac) - assert.Equal(t, "10.102.0.1", cfg.gateway) - assert.Equal(t, 16, cfg.prefix) -} - -func TestGuestNetworkReconfigureCommand_FallbackPreservesShellBehavior(t *testing.T) { +func TestGuestNetworkReconfigureCommand_AppliesAllocatedMAC(t *testing.T) { t.Parallel() alloc := &network.Allocation{ @@ -65,15 +42,18 @@ func TestGuestNetworkReconfigureCommand_FallbackPreservesShellBehavior(t *testin cmd, err := guestNetworkReconfigureCommand(alloc) require.NoError(t, err) + assert.Contains(t, cmd, "ip link set dev eth0 down") assert.Contains(t, cmd, "ip link set dev eth0 address 02:00:00:85:17:c8") assert.Contains(t, cmd, "ip addr add 10.102.146.62/16 dev eth0") + assert.Contains(t, cmd, "ip route replace default via 10.102.0.1 dev eth0") assert.Contains(t, cmd, "(ip neigh flush dev eth0 || true)") + assert.NotContains(t, cmd, "cat /sys/class/net/eth0/address") } -func TestGuestNetworkReconfigureConfig_RequiresAllocatedMAC(t *testing.T) { +func TestGuestNetworkReconfigureCommand_RequiresAllocatedMAC(t *testing.T) { t.Parallel() - _, err := guestNetworkReconfigureConfig(&network.Allocation{ + _, err := guestNetworkReconfigureCommand(&network.Allocation{ IP: "10.102.146.62", Gateway: "10.102.0.1", Netmask: "255.255.0.0", @@ -82,87 +62,6 @@ func TestGuestNetworkReconfigureConfig_RequiresAllocatedMAC(t *testing.T) { assert.Contains(t, err.Error(), "missing network allocation MAC") } -func TestPatchGuestResumeNetworkMailbox(t *testing.T) { - t.Parallel() - - dir := t.TempDir() - token := "test-token" - mem := make([]byte, 4096) - copy(mem[512:], mailbox.MailboxMagic) - copy(mem[512+len(mailbox.MailboxMagic):], token) - require.NoError(t, os.WriteFile(dir+"/"+firecrackerSnapshotMemoryFile, mem, 0644)) - - payload := &mailbox.Payload{ - InterfaceName: "eth0", - MAC: "02:00:00:85:17:c8", - IPv4: "10.102.146.62", - Prefix: 16, - Gateway: "10.102.0.1", - AckPort: 43210, - } - require.NoError(t, patchGuestResumeNetworkMailbox(dir, token, payload)) - - patched, err := os.ReadFile(dir + "/" + firecrackerSnapshotMemoryFile) - require.NoError(t, err) - - offset := 512 - require.Equal(t, uint32(1), binary.LittleEndian.Uint32(patched[offset+mailbox.MailboxSeqOffset:])) - payloadLen := binary.LittleEndian.Uint32(patched[offset+mailbox.MailboxLengthOffset:]) - require.NotZero(t, payloadLen) - - var decoded mailbox.Payload - err = json.Unmarshal(patched[offset+mailbox.MailboxPayloadOffset:offset+mailbox.MailboxPayloadOffset+int(payloadLen)], &decoded) - require.NoError(t, err) - assert.Equal(t, *payload, decoded) -} - -func TestEnsureGuestInitiatedResumeNetworkMailbox(t *testing.T) { - t.Parallel() - - stored := &StoredMetadata{ - HypervisorType: hypervisor.TypeFirecracker, - NetworkEnabled: true, - } - ensureGuestInitiatedResumeNetworkMailbox(stored) - - require.Equal(t, "1", stored.Env[mailbox.MailboxEnv]) - token := stored.Env[mailbox.MailboxTokenEnv] - require.NotEmpty(t, token) - require.LessOrEqual(t, len(token), mailbox.MailboxTokenMaxLen) - assert.True(t, guestInitiatedResumeNetworkMailbox(stored)) -} - -func TestEnsureGuestInitiatedResumeNetworkMailboxPreservesToken(t *testing.T) { - t.Parallel() - - stored := &StoredMetadata{ - HypervisorType: hypervisor.TypeFirecracker, - NetworkEnabled: true, - Env: map[string]string{ - mailbox.MailboxTokenEnv: "existing-token", - }, - } - ensureGuestInitiatedResumeNetworkMailbox(stored) - - assert.Equal(t, "1", stored.Env[mailbox.MailboxEnv]) - assert.Equal(t, "existing-token", stored.Env[mailbox.MailboxTokenEnv]) -} - -func TestEnsureGuestInitiatedResumeNetworkMailboxRequiresEligibleGuest(t *testing.T) { - t.Parallel() - - cases := []StoredMetadata{ - {HypervisorType: hypervisor.TypeCloudHypervisor, NetworkEnabled: true}, - {HypervisorType: hypervisor.TypeFirecracker, NetworkEnabled: false}, - {HypervisorType: hypervisor.TypeFirecracker, NetworkEnabled: true, SkipGuestAgent: true}, - } - for _, tc := range cases { - stored := tc - ensureGuestInitiatedResumeNetworkMailbox(&stored) - assert.False(t, guestInitiatedResumeNetworkMailbox(&stored)) - } -} - func TestRequiresRestoreConfigDiskRefresh(t *testing.T) { t.Parallel() diff --git a/lib/instances/resume_network_handoff.go b/lib/instances/resume_network_handoff.go deleted file mode 100644 index 073fc92e..00000000 --- a/lib/instances/resume_network_handoff.go +++ /dev/null @@ -1,82 +0,0 @@ -package instances - -import ( - "context" - - "github.com/kernel/hypeman/lib/logger" - "github.com/kernel/hypeman/lib/network" - "go.opentelemetry.io/otel/attribute" -) - -type resumeNetworkHandoff struct { - manager *manager - stored *StoredMetadata - allocatedNet *network.Allocation - ackWaiter *guestResumeNetworkUDPWaiter - ackCfg *guestNetworkConfig - patched bool -} - -func (m *manager) prepareResumeNetworkHandoff(ctx context.Context, stored *StoredMetadata, allocatedNet *network.Allocation, snapshotDir string) (*resumeNetworkHandoff, error) { - h := &resumeNetworkHandoff{ - manager: m, - stored: stored, - allocatedNet: allocatedNet, - } - if allocatedNet == nil || stored.SkipGuestAgent || !guestInitiatedResumeNetworkMailbox(stored) { - return h, nil - } - - log := logger.FromContext(ctx) - resumeNetworkCfg, err := guestNetworkReconfigureConfig(allocatedNet) - if err != nil { - log.WarnContext(ctx, "failed to build guest resume network mailbox payload; falling back to host-initiated reconfigure", "instance_id", stored.Id, "error", err) - return h, nil - } - - waiter, err := startGuestResumeNetworkUDPWaiter() - if err != nil { - log.WarnContext(ctx, "failed to start guest resume network UDP ack waiter; falling back to host-initiated reconfigure", "instance_id", stored.Id, "error", err) - return h, nil - } - - payload := newGuestResumeNetworkPayload(resumeNetworkCfg) - payload.AckPort = waiter.Port() - _, patchSpanEnd := m.startLifecycleStep(ctx, "guest.resume_network.mailbox_patch", - attribute.String("instance_id", stored.Id), - attribute.String("hypervisor", string(stored.HypervisorType)), - attribute.String("operation", "guest_resume_network_mailbox_patch"), - ) - err = patchGuestResumeNetworkMailbox(snapshotDir, guestInitiatedResumeNetworkMailboxToken(stored), &payload) - patchSpanEnd(err) - if err != nil { - waiter.Close() - log.WarnContext(ctx, "failed to patch guest resume network mailbox; falling back to host-initiated reconfigure", "instance_id", stored.Id, "error", err) - return h, nil - } - - h.ackWaiter = waiter - h.ackCfg = resumeNetworkCfg - h.patched = true - return h, nil -} - -func (h *resumeNetworkHandoff) Close() { - if h != nil && h.ackWaiter != nil { - h.ackWaiter.Close() - } -} - -func (h *resumeNetworkHandoff) AfterResume(ctx context.Context) error { - if h == nil || h.allocatedNet == nil || h.stored.SkipGuestAgent { - return nil - } - if h.patched { - err := h.manager.waitForGuestResumeNetworkUDPAck(ctx, h.ackWaiter, h.stored, h.ackCfg) - if err == nil { - return nil - } - logger.FromContext(ctx).ErrorContext(ctx, "guest resume network UDP ack wait failed; falling back to host-initiated reconfigure", "instance_id", h.stored.Id, "error", err) - } - return reconfigureGuestNetwork(ctx, h.stored, h.allocatedNet) -} diff --git a/lib/instances/start.go b/lib/instances/start.go index dd550a96..181aaa8d 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -141,7 +141,6 @@ func (m *manager) startInstance( }) } } - ensureGuestInitiatedResumeNetworkMailbox(stored) // 4b. Recreate vGPU mdev if this instance had a GPU profile // Note: GPU availability was already validated in step 2b diff --git a/lib/instances/test_network_config_test.go b/lib/instances/test_network_config_test.go index 419f9fe8..c26efcdb 100644 --- a/lib/instances/test_network_config_test.go +++ b/lib/instances/test_network_config_test.go @@ -183,7 +183,7 @@ func allocateTestNetworkLease(testName string, seq uint32) (*testNetworkLease, e } func withTestSubnetLock(fn func() error) error { - lockPath := filepath.Join(testNetworkTempDir(), "hypeman-test-network.lock") + lockPath := filepath.Join(os.TempDir(), "hypeman-test-network.lock") lockFile, err := openTestSubnetLockFile(lockPath) if err != nil { return fmt.Errorf("open subnet lock file: %w", err) @@ -220,14 +220,7 @@ func openTestSubnetLockFile(lockPath string) (*os.File, error) { } func testSubnetLeaseFilePath() string { - return filepath.Join(testNetworkTempDir(), "hypeman-test-network-leases.json") -} - -func testNetworkTempDir() string { - if dir := os.Getenv("HYPEMAN_TEST_NETWORK_TMPDIR"); dir != "" { - return dir - } - return os.TempDir() + return filepath.Join(os.TempDir(), "hypeman-test-network-leases.json") } func loadSubnetLeases() (map[string]subnetLease, error) { diff --git a/lib/mailbox/README.md b/lib/mailbox/README.md deleted file mode 100644 index 3e834b7c..00000000 --- a/lib/mailbox/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# mailbox - -The resume network handoff uses a small pre-armed mailbox in guest memory to avoid making the first post-resume operation a host-initiated guest RPC. When a Firecracker guest boots with mailbox env enabled, the guest agent allocates and locks a fixed-size buffer containing a magic marker and token. That buffer is captured in standby snapshots. - -Before restore, the host finds the marker in the snapshot memory file, writes a JSON network payload into the buffer, and flips the sequence field. After resume, VMGenID wakes the guest watcher, which reads the payload, applies the new network identity locally, and sends a UDP applied ack to the host. If the mailbox is missing, cannot be patched, or does not ack in time, restore falls back to the host-initiated network reconfigure path. diff --git a/lib/mailbox/mailbox.go b/lib/mailbox/mailbox.go deleted file mode 100644 index 832f852f..00000000 --- a/lib/mailbox/mailbox.go +++ /dev/null @@ -1,64 +0,0 @@ -package mailbox - -import ( - "encoding/json" - "fmt" -) - -const MailboxEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX" -const MailboxTokenEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX_TOKEN" - -const MailboxSize = 4096 -const MailboxMagic = "HYPEMAN_RESUME_NETWORK_MAILBOX_V1\x00" -const MailboxSeqOffset = 64 -const MailboxLengthOffset = 68 -const MailboxPayloadOffset = 72 -const MailboxTokenMaxLen = MailboxSeqOffset - len(MailboxMagic) - -type Payload struct { - InterfaceName string `json:"interface_name"` - MAC string `json:"mac"` - IPv4 string `json:"ipv4"` - Prefix uint32 `json:"prefix"` - Gateway string `json:"gateway"` - AckPort uint32 `json:"ack_port,omitempty"` -} - -func ValidToken(token string) bool { - return token != "" && len(token) <= MailboxTokenMaxLen -} - -func Marker(token string) ([]byte, error) { - if !ValidToken(token) { - return nil, fmt.Errorf("resume network mailbox token is invalid") - } - marker := make([]byte, 0, len(MailboxMagic)+len(token)) - marker = append(marker, MailboxMagic...) - marker = append(marker, token...) - return marker, nil -} - -func MarshalPayload(payload *Payload) ([]byte, error) { - if payload == nil { - return nil, fmt.Errorf("resume network mailbox payload is nil") - } - payloadBytes, err := json.Marshal(payload) - if err != nil { - return nil, fmt.Errorf("marshal resume network mailbox payload: %w", err) - } - if len(payloadBytes) > MailboxSize-MailboxPayloadOffset { - return nil, fmt.Errorf("resume network mailbox payload too large: %d bytes", len(payloadBytes)) - } - return payloadBytes, nil -} - -func DecodePayloadFrame(buf []byte, payloadLen uint32) (Payload, error) { - if payloadLen == 0 || int(payloadLen) > len(buf)-MailboxPayloadOffset { - return Payload{}, fmt.Errorf("invalid mailbox payload length %d", payloadLen) - } - var payload Payload - if err := json.Unmarshal(buf[MailboxPayloadOffset:MailboxPayloadOffset+int(payloadLen)], &payload); err != nil { - return Payload{}, fmt.Errorf("decode mailbox payload: %w", err) - } - return payload, nil -} diff --git a/lib/mailbox/mailbox_test.go b/lib/mailbox/mailbox_test.go deleted file mode 100644 index db0b5c2b..00000000 --- a/lib/mailbox/mailbox_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package mailbox - -import ( - "encoding/binary" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMarker(t *testing.T) { - t.Parallel() - - marker, err := Marker("token") - require.NoError(t, err) - assert.Equal(t, MailboxMagic+"token", string(marker)) - - _, err = Marker("") - require.Error(t, err) - - _, err = Marker(strings.Repeat("x", MailboxTokenMaxLen+1)) - require.Error(t, err) -} - -func TestPayloadRoundTrip(t *testing.T) { - t.Parallel() - - want := &Payload{ - InterfaceName: "eth0", - MAC: "02:00:00:85:17:c8", - IPv4: "10.102.146.62", - Prefix: 16, - Gateway: "10.102.0.1", - AckPort: 43210, - } - payloadBytes, err := MarshalPayload(want) - require.NoError(t, err) - - buf := make([]byte, MailboxSize) - copy(buf[MailboxPayloadOffset:], payloadBytes) - binary.LittleEndian.PutUint32(buf[MailboxLengthOffset:], uint32(len(payloadBytes))) - - got, err := DecodePayloadFrame(buf, binary.LittleEndian.Uint32(buf[MailboxLengthOffset:])) - require.NoError(t, err) - assert.Equal(t, *want, got) -} diff --git a/lib/system/guest_agent/main.go b/lib/system/guest_agent/main.go index a968e89a..46ee3d7a 100644 --- a/lib/system/guest_agent/main.go +++ b/lib/system/guest_agent/main.go @@ -53,10 +53,8 @@ func main() { } // Create gRPC server - guestSvc := &guestServer{} - startResumeNetworkWatcher(guestSvc) grpcServer := grpc.NewServer() - pb.RegisterGuestServiceServer(grpcServer, guestSvc) + pb.RegisterGuestServiceServer(grpcServer, &guestServer{}) // Serve gRPC over vsock if err := grpcServer.Serve(l); err != nil { diff --git a/lib/system/guest_agent/resume_network.go b/lib/system/guest_agent/resume_network.go deleted file mode 100644 index f8708976..00000000 --- a/lib/system/guest_agent/resume_network.go +++ /dev/null @@ -1,185 +0,0 @@ -//go:build linux - -package main - -import ( - "bufio" - "context" - "encoding/binary" - "fmt" - "io" - "log" - "net" - "os" - "strconv" - "strings" - "sync/atomic" - "time" - "unsafe" - - pb "github.com/kernel/hypeman/lib/guest" - "github.com/kernel/hypeman/lib/mailbox" - "golang.org/x/sys/unix" -) - -const vmgenIDKmsgSignal = "crng reseeded due to virtual machine fork" -const resumeNetworkMailboxPayloadTimeout = 5 * time.Second - -type vmGenIDResumeWaiter struct { - file *os.File - reader *bufio.Reader -} - -func startResumeNetworkWatcher(s *guestServer) { - if strings.TrimSpace(os.Getenv(mailbox.MailboxEnv)) != "1" { - return - } - - mailbox := newResumeNetworkMailbox() - if mailbox == nil { - return - } - - go resumeNetworkMailboxLoop(s, mailbox) -} - -func newResumeNetworkMailbox() []byte { - token := strings.TrimSpace(os.Getenv(mailbox.MailboxTokenEnv)) - if !mailbox.ValidToken(token) { - log.Printf("[guest-agent] resume network mailbox disabled: invalid %s", mailbox.MailboxTokenEnv) - return nil - } - - buf := make([]byte, mailbox.MailboxSize) - copy(buf, mailbox.MailboxMagic) - copy(buf[len(mailbox.MailboxMagic):mailbox.MailboxSeqOffset], token) - if err := unix.Mlock(buf); err != nil { - log.Printf("[guest-agent] resume network mailbox mlock failed: %v", err) - } - log.Printf("[guest-agent] resume network mailbox armed token=%s", token) - return buf -} - -func resumeNetworkMailboxLoop(s *guestServer, mailbox []byte) { - for { - waiter, err := newVMGenIDResumeWaiter() - if err != nil { - log.Printf("[guest-agent] resume network VMGenID prepare failed: %v", err) - time.Sleep(100 * time.Millisecond) - continue - } - - start := time.Now() - if err := waiter.Wait(); err != nil { - waiter.Close() - log.Printf("[guest-agent] resume network VMGenID wait failed: %v", err) - time.Sleep(100 * time.Millisecond) - continue - } - waiter.Close() - - if err := waitAndApplyResumeNetworkMailbox(s, mailbox); err != nil { - log.Printf("[guest-agent] resume network mailbox apply failed: %v", err) - time.Sleep(25 * time.Millisecond) - continue - } - log.Printf("[guest-agent] resume network mailbox applied in %s", time.Since(start)) - } -} - -func waitAndApplyResumeNetworkMailbox(s *guestServer, buf []byte) error { - return waitAndApplyResumeNetworkMailboxWithTimeout(s, buf, resumeNetworkMailboxPayloadTimeout) -} - -func waitAndApplyResumeNetworkMailboxWithTimeout(s *guestServer, buf []byte, timeout time.Duration) error { - deadline := time.Now().Add(timeout) - for { - seq := atomicLoadUint32(buf[mailbox.MailboxSeqOffset:]) - if seq == 0 { - if time.Now().After(deadline) { - return fmt.Errorf("resume network mailbox payload was not patched within %s", timeout) - } - time.Sleep(100 * time.Microsecond) - continue - } - - payloadLen := binary.LittleEndian.Uint32(buf[mailbox.MailboxLengthOffset:]) - payload, err := mailbox.DecodePayloadFrame(buf, payloadLen) - if err != nil { - return err - } - - _, err = s.ReconfigureNetwork(context.Background(), &pb.ReconfigureNetworkRequest{ - InterfaceName: payload.InterfaceName, - Mac: payload.MAC, - Ipv4: payload.IPv4, - Prefix: payload.Prefix, - Gateway: payload.Gateway, - }) - if err != nil { - return err - } - sendResumeNetworkAck(payload, "applied") - atomicStoreUint32(buf[mailbox.MailboxSeqOffset:], 0) - return nil - } -} - -func sendResumeNetworkAck(payload mailbox.Payload, stage string) { - if payload.AckPort == 0 || payload.Gateway == "" { - return - } - - addr := net.JoinHostPort(payload.Gateway, strconv.FormatUint(uint64(payload.AckPort), 10)) - conn, err := net.DialTimeout("udp4", addr, 100*time.Millisecond) - if err != nil { - log.Printf("[guest-agent] resume network ack dial failed: %v", err) - return - } - defer conn.Close() - - _, _ = fmt.Fprintf(conn, "stage=%s mac=%s ip=%s\n", stage, payload.MAC, payload.IPv4) -} - -func atomicLoadUint32(buf []byte) uint32 { - return atomic.LoadUint32((*uint32)(unsafe.Pointer(&buf[0]))) -} - -func atomicStoreUint32(buf []byte, value uint32) { - atomic.StoreUint32((*uint32)(unsafe.Pointer(&buf[0])), value) -} - -func newVMGenIDResumeWaiter() (*vmGenIDResumeWaiter, error) { - f, err := os.Open("/dev/kmsg") - if err != nil { - return nil, fmt.Errorf("open /dev/kmsg: %w", err) - } - - if _, err := f.Seek(0, io.SeekEnd); err != nil { - log.Printf("[guest-agent] warning: failed to seek /dev/kmsg to end: %v", err) - } - - return &vmGenIDResumeWaiter{ - file: f, - reader: bufio.NewReader(f), - }, nil -} - -func (w *vmGenIDResumeWaiter) Close() { - if w == nil || w.file == nil { - return - } - _ = w.file.Close() -} - -func (w *vmGenIDResumeWaiter) Wait() error { - for { - line, err := w.reader.ReadString('\n') - if err != nil { - return fmt.Errorf("read /dev/kmsg: %w", err) - } - if strings.Contains(line, vmgenIDKmsgSignal) { - return nil - } - } -} diff --git a/lib/system/guest_agent/resume_network_other.go b/lib/system/guest_agent/resume_network_other.go deleted file mode 100644 index dfa74e07..00000000 --- a/lib/system/guest_agent/resume_network_other.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build !linux - -package main - -type resumeNetworkController struct{} - -func startResumeNetworkWatcher(_ *guestServer) {} diff --git a/lib/system/guest_agent/resume_network_test.go b/lib/system/guest_agent/resume_network_test.go deleted file mode 100644 index 9e30a533..00000000 --- a/lib/system/guest_agent/resume_network_test.go +++ /dev/null @@ -1,20 +0,0 @@ -//go:build linux - -package main - -import ( - "testing" - "time" - - "github.com/kernel/hypeman/lib/mailbox" - "github.com/stretchr/testify/require" -) - -func TestWaitAndApplyResumeNetworkMailboxTimesOutWhenPayloadMissing(t *testing.T) { - buf := make([]byte, mailbox.MailboxSize) - - err := waitAndApplyResumeNetworkMailboxWithTimeout(&guestServer{}, buf, 5*time.Millisecond) - - require.Error(t, err) - require.Contains(t, err.Error(), "payload was not patched") -}