diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 2c85284b..9d59515d 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -657,9 +657,10 @@ func (s *ApiService) ForkInstance(ctx context.Context, request oapi.ForkInstance } result, err := s.InstanceManager.ForkInstance(ctx, inst.Id, instances.ForkInstanceRequest{ - Name: request.Body.Name, - FromRunning: request.Body.FromRunning != nil && *request.Body.FromRunning, - TargetState: targetState, + Name: request.Body.Name, + FromRunning: request.Body.FromRunning != nil && *request.Body.FromRunning, + TargetState: targetState, + WaitForNetwork: request.Body.WaitForNetwork, }) if err != nil { switch { diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 4e7863aa..896a0632 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -1165,13 +1165,15 @@ func TestForkInstance_Success(t *testing.T) { result: forked, } svc.InstanceManager = mockMgr + waitForNetwork := false resp, err := svc.ForkInstance( mw.WithResolvedInstance(ctx(), source.Id, source), oapi.ForkInstanceRequestObject{ Id: source.Id, Body: &oapi.ForkInstanceRequest{ - Name: "forked-instance", + Name: "forked-instance", + WaitForNetwork: &waitForNetwork, }, }, ) @@ -1185,6 +1187,8 @@ func TestForkInstance_Success(t *testing.T) { assert.Equal(t, "forked-instance", mockMgr.lastReq.Name) assert.False(t, mockMgr.lastReq.FromRunning) assert.Equal(t, instances.State(""), mockMgr.lastReq.TargetState) + require.NotNil(t, mockMgr.lastReq.WaitForNetwork) + assert.False(t, *mockMgr.lastReq.WaitForNetwork) } func TestForkInstance_NotSupported(t *testing.T) { diff --git a/cmd/api/api/snapshots.go b/cmd/api/api/snapshots.go index 620d9574..ffe35474 100644 --- a/cmd/api/api/snapshots.go +++ b/cmd/api/api/snapshots.go @@ -175,7 +175,10 @@ func (s *ApiService) ForkSnapshot(ctx context.Context, request oapi.ForkSnapshot return oapi.ForkSnapshot400JSONResponse{Code: "invalid_request", Message: "request body is required"}, nil } - domainReq := instances.ForkSnapshotRequest{Name: request.Body.Name} + domainReq := instances.ForkSnapshotRequest{ + Name: request.Body.Name, + WaitForNetwork: request.Body.WaitForNetwork, + } if request.Body.TargetState != nil { domainReq.TargetState = instances.State(*request.Body.TargetState) } diff --git a/cmd/api/api/snapshots_test.go b/cmd/api/api/snapshots_test.go index d0730133..f87e9148 100644 --- a/cmd/api/api/snapshots_test.go +++ b/cmd/api/api/snapshots_test.go @@ -1,14 +1,35 @@ package api import ( + "context" "testing" "time" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/oapi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +type captureForkSnapshotManager struct { + instances.Manager + lastID string + lastReq *instances.ForkSnapshotRequest + result *instances.Instance + err error +} + +func (m *captureForkSnapshotManager) ForkSnapshot(ctx context.Context, snapshotID string, req instances.ForkSnapshotRequest) (*instances.Instance, error) { + reqCopy := req + m.lastID = snapshotID + m.lastReq = &reqCopy + if m.err != nil { + return nil, m.err + } + return m.result, nil +} + func TestSnapshotScheduleToOAPIPreservesZeroMaxCount(t *testing.T) { t.Parallel() @@ -30,3 +51,43 @@ func TestSnapshotScheduleToOAPIPreservesZeroMaxCount(t *testing.T) { require.NotNil(t, out.Retention.MaxAge) assert.Equal(t, "24h0m0s", *out.Retention.MaxAge) } + +func TestForkSnapshotMapsWaitForNetwork(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + forked := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "forked-instance", + Name: "forked-instance", + Image: "docker.io/library/alpine:latest", + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeFirecracker, + }, + State: instances.StateRunning, + } + mockMgr := &captureForkSnapshotManager{ + Manager: svc.InstanceManager, + result: forked, + } + svc.InstanceManager = mockMgr + waitForNetwork := false + + resp, err := svc.ForkSnapshot(ctx(), oapi.ForkSnapshotRequestObject{ + SnapshotId: "snap-123", + Body: &oapi.ForkSnapshotRequest{ + Name: "forked-instance", + WaitForNetwork: &waitForNetwork, + }, + }) + require.NoError(t, err) + + created, ok := resp.(oapi.ForkSnapshot201JSONResponse) + require.True(t, ok, "expected 201 response") + assert.Equal(t, "forked-instance", created.Name) + assert.Equal(t, "snap-123", mockMgr.lastID) + require.NotNil(t, mockMgr.lastReq) + assert.Equal(t, "forked-instance", mockMgr.lastReq.Name) + require.NotNil(t, mockMgr.lastReq.WaitForNetwork) + assert.False(t, *mockMgr.lastReq.WaitForNetwork) +} diff --git a/lib/guest/client.go b/lib/guest/client.go index f46b1149..3e36c140 100644 --- a/lib/guest/client.go +++ b/lib/guest/client.go @@ -159,6 +159,92 @@ type ReconfigureNetworkOptions struct { WaitForAgent time.Duration } +type ArmResumeNetworkOptions struct { + PollIntervalMS uint32 + WaitForAgent time.Duration +} + +func ArmResumeNetworkInInstance(ctx context.Context, dialer hypervisor.VsockDialer, opts ArmResumeNetworkOptions) error { + if opts.WaitForAgent == 0 { + return armResumeNetworkOnce(ctx, dialer, opts) + } + + ctx, span := otel.Tracer("hypeman/guest").Start(ctx, "guest.arm_resume_network", trace.WithAttributes( + attribute.Bool("wait_for_agent", true), + attribute.Int64("wait_for_agent_ms", opts.WaitForAgent.Milliseconds()), + attribute.Int("poll_interval_ms", int(opts.PollIntervalMS)), + )) + defer span.End() + + deadline := time.Now().Add(opts.WaitForAgent) + start := time.Now() + attempts := 0 + retryableAttempts := 0 + firstRetryableErrorType := "" + lastRetryableErrorType := "" + lastRetryInterval := time.Duration(0) + + for { + attempts++ + err := armResumeNetworkOnce(ctx, dialer, opts) + if err == nil { + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.SetStatus(otelcodes.Ok, "") + return nil + } + if !isRetryableConnectionError(err) { + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.RecordError(err) + span.SetStatus(otelcodes.Error, err.Error()) + return err + } + + retryableAttempts++ + errType := retryableConnectionErrorType(err) + if firstRetryableErrorType == "" { + firstRetryableErrorType = errType + } + lastRetryableErrorType = errType + CloseConn(dialer.Key()) + + if time.Now().After(deadline) { + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.RecordError(err) + span.SetStatus(otelcodes.Error, err.Error()) + return err + } + + retryInterval := guestExecRetryInterval(time.Since(start)) + lastRetryInterval = retryInterval + select { + case <-ctx.Done(): + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.RecordError(ctx.Err()) + span.SetStatus(otelcodes.Error, ctx.Err().Error()) + return ctx.Err() + case <-time.After(retryInterval): + } + } +} + +func armResumeNetworkOnce(ctx context.Context, dialer hypervisor.VsockDialer, opts ArmResumeNetworkOptions) error { + grpcConn, err := GetOrCreateConn(ctx, dialer) + if err != nil { + return fmt.Errorf("get grpc connection: %w", err) + } + client := NewGuestServiceClient(grpcConn) + + _, span := otel.Tracer("hypeman/guest").Start(ctx, "guest.arm_resume_network.rpc") + _, err = client.ArmResumeNetwork(ctx, &ArmResumeNetworkRequest{ + PollIntervalMs: opts.PollIntervalMS, + }) + finishGuestNetworkStepSpan(span, err) + if err != nil { + return fmt.Errorf("arm resume network rpc: %w", err) + } + return nil +} + func ReconfigureNetworkInInstance(ctx context.Context, dialer hypervisor.VsockDialer, opts ReconfigureNetworkOptions) error { if opts.WaitForAgent == 0 { return reconfigureNetworkOnce(ctx, dialer, opts) diff --git a/lib/guest/guest.pb.go b/lib/guest/guest.pb.go index a239fc97..23c75f24 100644 --- a/lib/guest/guest.pb.go +++ b/lib/guest/guest.pb.go @@ -1380,6 +1380,88 @@ func (*ReconfigureNetworkResponse) Descriptor() ([]byte, []int) { return file_lib_guest_guest_proto_rawDescGZIP(), []int{18} } +// ArmResumeNetworkRequest arms the guest-side fast reconnect loop before snapshot +type ArmResumeNetworkRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + PollIntervalMs uint32 `protobuf:"varint,1,opt,name=poll_interval_ms,json=pollIntervalMs,proto3" json:"poll_interval_ms,omitempty"` // Fast reconnect interval while armed; 0 uses the guest default + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ArmResumeNetworkRequest) Reset() { + *x = ArmResumeNetworkRequest{} + mi := &file_lib_guest_guest_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ArmResumeNetworkRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ArmResumeNetworkRequest) ProtoMessage() {} + +func (x *ArmResumeNetworkRequest) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ArmResumeNetworkRequest.ProtoReflect.Descriptor instead. +func (*ArmResumeNetworkRequest) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{19} +} + +func (x *ArmResumeNetworkRequest) GetPollIntervalMs() uint32 { + if x != nil { + return x.PollIntervalMs + } + return 0 +} + +// ArmResumeNetworkResponse acknowledges that the fast reconnect loop is armed +type ArmResumeNetworkResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ArmResumeNetworkResponse) Reset() { + *x = ArmResumeNetworkResponse{} + mi := &file_lib_guest_guest_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ArmResumeNetworkResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ArmResumeNetworkResponse) ProtoMessage() {} + +func (x *ArmResumeNetworkResponse) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ArmResumeNetworkResponse.ProtoReflect.Descriptor instead. +func (*ArmResumeNetworkResponse) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{20} +} + var File_lib_guest_guest_proto protoreflect.FileDescriptor const file_lib_guest_guest_proto_rawDesc = "" + @@ -1479,14 +1561,18 @@ const file_lib_guest_guest_proto_rawDesc = "" + "\x04ipv4\x18\x03 \x01(\tR\x04ipv4\x12\x16\n" + "\x06prefix\x18\x04 \x01(\rR\x06prefix\x12\x18\n" + "\agateway\x18\x05 \x01(\tR\agateway\"\x1c\n" + - "\x1aReconfigureNetworkResponse2\xae\x03\n" + + "\x1aReconfigureNetworkResponse\"C\n" + + "\x17ArmResumeNetworkRequest\x12(\n" + + "\x10poll_interval_ms\x18\x01 \x01(\rR\x0epollIntervalMs\"\x1a\n" + + "\x18ArmResumeNetworkResponse2\x83\x04\n" + "\fGuestService\x123\n" + "\x04Exec\x12\x12.guest.ExecRequest\x1a\x13.guest.ExecResponse(\x010\x01\x12F\n" + "\vCopyToGuest\x12\x19.guest.CopyToGuestRequest\x1a\x1a.guest.CopyToGuestResponse(\x01\x12L\n" + "\rCopyFromGuest\x12\x1b.guest.CopyFromGuestRequest\x1a\x1c.guest.CopyFromGuestResponse0\x01\x12;\n" + "\bStatPath\x12\x16.guest.StatPathRequest\x1a\x17.guest.StatPathResponse\x12;\n" + "\bShutdown\x12\x16.guest.ShutdownRequest\x1a\x17.guest.ShutdownResponse\x12Y\n" + - "\x12ReconfigureNetwork\x12 .guest.ReconfigureNetworkRequest\x1a!.guest.ReconfigureNetworkResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" + "\x12ReconfigureNetwork\x12 .guest.ReconfigureNetworkRequest\x1a!.guest.ReconfigureNetworkResponse\x12S\n" + + "\x10ArmResumeNetwork\x12\x1e.guest.ArmResumeNetworkRequest\x1a\x1f.guest.ArmResumeNetworkResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" var ( file_lib_guest_guest_proto_rawDescOnce sync.Once @@ -1500,7 +1586,7 @@ func file_lib_guest_guest_proto_rawDescGZIP() []byte { return file_lib_guest_guest_proto_rawDescData } -var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 22) var file_lib_guest_guest_proto_goTypes = []any{ (*ExecRequest)(nil), // 0: guest.ExecRequest (*ExecStart)(nil), // 1: guest.ExecStart @@ -1521,12 +1607,14 @@ var file_lib_guest_guest_proto_goTypes = []any{ (*ShutdownResponse)(nil), // 16: guest.ShutdownResponse (*ReconfigureNetworkRequest)(nil), // 17: guest.ReconfigureNetworkRequest (*ReconfigureNetworkResponse)(nil), // 18: guest.ReconfigureNetworkResponse - nil, // 19: guest.ExecStart.EnvEntry + (*ArmResumeNetworkRequest)(nil), // 19: guest.ArmResumeNetworkRequest + (*ArmResumeNetworkResponse)(nil), // 20: guest.ArmResumeNetworkResponse + nil, // 21: guest.ExecStart.EnvEntry } var file_lib_guest_guest_proto_depIdxs = []int32{ 1, // 0: guest.ExecRequest.start:type_name -> guest.ExecStart 2, // 1: guest.ExecRequest.resize:type_name -> guest.WindowSize - 19, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry + 21, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry 5, // 3: guest.CopyToGuestRequest.start:type_name -> guest.CopyToGuestStart 6, // 4: guest.CopyToGuestRequest.end:type_name -> guest.CopyToGuestEnd 10, // 5: guest.CopyFromGuestResponse.header:type_name -> guest.CopyFromGuestHeader @@ -1538,14 +1626,16 @@ var file_lib_guest_guest_proto_depIdxs = []int32{ 13, // 11: guest.GuestService.StatPath:input_type -> guest.StatPathRequest 15, // 12: guest.GuestService.Shutdown:input_type -> guest.ShutdownRequest 17, // 13: guest.GuestService.ReconfigureNetwork:input_type -> guest.ReconfigureNetworkRequest - 3, // 14: guest.GuestService.Exec:output_type -> guest.ExecResponse - 7, // 15: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse - 9, // 16: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse - 14, // 17: guest.GuestService.StatPath:output_type -> guest.StatPathResponse - 16, // 18: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse - 18, // 19: guest.GuestService.ReconfigureNetwork:output_type -> guest.ReconfigureNetworkResponse - 14, // [14:20] is the sub-list for method output_type - 8, // [8:14] is the sub-list for method input_type + 19, // 14: guest.GuestService.ArmResumeNetwork:input_type -> guest.ArmResumeNetworkRequest + 3, // 15: guest.GuestService.Exec:output_type -> guest.ExecResponse + 7, // 16: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse + 9, // 17: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse + 14, // 18: guest.GuestService.StatPath:output_type -> guest.StatPathResponse + 16, // 19: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse + 18, // 20: guest.GuestService.ReconfigureNetwork:output_type -> guest.ReconfigureNetworkResponse + 20, // 21: guest.GuestService.ArmResumeNetwork:output_type -> guest.ArmResumeNetworkResponse + 15, // [15:22] is the sub-list for method output_type + 8, // [8:15] is the sub-list for method input_type 8, // [8:8] is the sub-list for extension type_name 8, // [8:8] is the sub-list for extension extendee 0, // [0:8] is the sub-list for field type_name @@ -1583,7 +1673,7 @@ func file_lib_guest_guest_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_lib_guest_guest_proto_rawDesc), len(file_lib_guest_guest_proto_rawDesc)), NumEnums: 0, - NumMessages: 20, + NumMessages: 22, NumExtensions: 0, NumServices: 1, }, diff --git a/lib/guest/guest.proto b/lib/guest/guest.proto index 317c21b3..99a77585 100644 --- a/lib/guest/guest.proto +++ b/lib/guest/guest.proto @@ -23,6 +23,9 @@ service GuestService { // ReconfigureNetwork updates the guest network identity without spawning shell commands rpc ReconfigureNetwork(ReconfigureNetworkRequest) returns (ReconfigureNetworkResponse); + + // ArmResumeNetwork puts the guest-initiated resume network watcher into its fast reconnect loop + rpc ArmResumeNetwork(ArmResumeNetworkRequest) returns (ArmResumeNetworkResponse); } // ExecRequest represents messages from client to server @@ -169,3 +172,11 @@ message ReconfigureNetworkRequest { // ReconfigureNetworkResponse acknowledges the network reconfiguration request message ReconfigureNetworkResponse {} + +// ArmResumeNetworkRequest arms the guest-side fast reconnect loop before snapshot +message ArmResumeNetworkRequest { + uint32 poll_interval_ms = 1; // Fast reconnect interval while armed; 0 uses the guest default +} + +// ArmResumeNetworkResponse acknowledges that the fast reconnect loop is armed +message ArmResumeNetworkResponse {} diff --git a/lib/guest/guest_grpc.pb.go b/lib/guest/guest_grpc.pb.go index f93631d9..68d73811 100644 --- a/lib/guest/guest_grpc.pb.go +++ b/lib/guest/guest_grpc.pb.go @@ -25,6 +25,7 @@ const ( GuestService_StatPath_FullMethodName = "/guest.GuestService/StatPath" GuestService_Shutdown_FullMethodName = "/guest.GuestService/Shutdown" GuestService_ReconfigureNetwork_FullMethodName = "/guest.GuestService/ReconfigureNetwork" + GuestService_ArmResumeNetwork_FullMethodName = "/guest.GuestService/ArmResumeNetwork" ) // GuestServiceClient is the client API for GuestService service. @@ -45,6 +46,8 @@ type GuestServiceClient interface { Shutdown(ctx context.Context, in *ShutdownRequest, opts ...grpc.CallOption) (*ShutdownResponse, error) // ReconfigureNetwork updates the guest network identity without spawning shell commands ReconfigureNetwork(ctx context.Context, in *ReconfigureNetworkRequest, opts ...grpc.CallOption) (*ReconfigureNetworkResponse, error) + // ArmResumeNetwork puts the guest-initiated resume network watcher into its fast reconnect loop + ArmResumeNetwork(ctx context.Context, in *ArmResumeNetworkRequest, opts ...grpc.CallOption) (*ArmResumeNetworkResponse, error) } type guestServiceClient struct { @@ -130,6 +133,16 @@ func (c *guestServiceClient) ReconfigureNetwork(ctx context.Context, in *Reconfi return out, nil } +func (c *guestServiceClient) ArmResumeNetwork(ctx context.Context, in *ArmResumeNetworkRequest, opts ...grpc.CallOption) (*ArmResumeNetworkResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ArmResumeNetworkResponse) + err := c.cc.Invoke(ctx, GuestService_ArmResumeNetwork_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // GuestServiceServer is the server API for GuestService service. // All implementations must embed UnimplementedGuestServiceServer // for forward compatibility. @@ -148,6 +161,8 @@ type GuestServiceServer interface { Shutdown(context.Context, *ShutdownRequest) (*ShutdownResponse, error) // ReconfigureNetwork updates the guest network identity without spawning shell commands ReconfigureNetwork(context.Context, *ReconfigureNetworkRequest) (*ReconfigureNetworkResponse, error) + // ArmResumeNetwork puts the guest-initiated resume network watcher into its fast reconnect loop + ArmResumeNetwork(context.Context, *ArmResumeNetworkRequest) (*ArmResumeNetworkResponse, error) mustEmbedUnimplementedGuestServiceServer() } @@ -176,6 +191,9 @@ func (UnimplementedGuestServiceServer) Shutdown(context.Context, *ShutdownReques func (UnimplementedGuestServiceServer) ReconfigureNetwork(context.Context, *ReconfigureNetworkRequest) (*ReconfigureNetworkResponse, error) { return nil, status.Error(codes.Unimplemented, "method ReconfigureNetwork not implemented") } +func (UnimplementedGuestServiceServer) ArmResumeNetwork(context.Context, *ArmResumeNetworkRequest) (*ArmResumeNetworkResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ArmResumeNetwork not implemented") +} func (UnimplementedGuestServiceServer) mustEmbedUnimplementedGuestServiceServer() {} func (UnimplementedGuestServiceServer) testEmbeddedByValue() {} @@ -276,6 +294,24 @@ func _GuestService_ReconfigureNetwork_Handler(srv interface{}, ctx context.Conte return interceptor(ctx, in, info, handler) } +func _GuestService_ArmResumeNetwork_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ArmResumeNetworkRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GuestServiceServer).ArmResumeNetwork(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: GuestService_ArmResumeNetwork_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GuestServiceServer).ArmResumeNetwork(ctx, req.(*ArmResumeNetworkRequest)) + } + return interceptor(ctx, in, info, handler) +} + // GuestService_ServiceDesc is the grpc.ServiceDesc for GuestService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -295,6 +331,10 @@ var GuestService_ServiceDesc = grpc.ServiceDesc{ MethodName: "ReconfigureNetwork", Handler: _GuestService_ReconfigureNetwork_Handler, }, + { + MethodName: "ArmResumeNetwork", + Handler: _GuestService_ArmResumeNetwork_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 202660e2..d74a75a9 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -81,7 +81,9 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR // the source data directory. Restore the fork while source remains standby and // under lock, then restore the source. if forkErr == nil && targetState == StateRunning { - restoredFork, err := m.applyForkTargetState(ctx, forked.Id, StateRunning) + restoredFork, err := m.applyForkTargetState(ctx, forked.Id, StateRunning, restoreInstanceOptions{ + WaitForGuestNetwork: req.WaitForNetwork, + }) if err != nil { forkErr = fmt.Errorf("restore forked instance before source restore: %w", err) if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { @@ -93,7 +95,7 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR } log.InfoContext(ctx, "restoring source instance after running fork", "source_instance_id", id) - _, restoreErr := m.restoreInstance(ctx, id) + _, restoreErr := m.restoreInstance(ctx, id, restoreInstanceOptions{}) if restoreErr != nil { if forkErr != nil { @@ -393,7 +395,7 @@ func resolveForkTargetState(requested State, sourceState State) (State, error) { return requested, nil } -func (m *manager) applyForkTargetState(ctx context.Context, forkID string, target State) (*Instance, error) { +func (m *manager) applyForkTargetState(ctx context.Context, forkID string, target State, restoreOpts restoreInstanceOptions) (*Instance, error) { lock := m.getInstanceLock(forkID) lock.Lock() defer lock.Unlock() @@ -403,6 +405,9 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe return nil, err } if inst != nil && (inst.State == StateRunning || inst.State == StateInitializing) { + if guestInitiatedResumeNetworkMailbox(&inst.StoredMetadata) { + return inst, nil + } if err := ensureGuestAgentReadyForForkPhase(ctx, &inst.StoredMetadata, "before returning running fork instance"); err != nil { return nil, fmt.Errorf("wait for forked guest agent readiness: %w", err) } @@ -432,7 +437,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe case StateStandby: switch target { case StateRunning: - return returnWithReadiness(m.restoreInstance(ctx, forkID)) + return returnWithReadiness(m.restoreInstance(ctx, forkID, restoreOpts)) case StateStopped: if err := os.RemoveAll(m.paths.InstanceSnapshotLatest(forkID)); err != nil { return nil, fmt.Errorf("remove fork snapshot: %w", err) diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index eafcd77a..32764063 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -308,7 +308,7 @@ func TestApplyForkTargetStateStoppedRefreshesSnapshotForkCID(t *testing.T) { meta.StoredMetadata.Phases.Record(phasetracking.PhaseStandby, time.Now()) require.NoError(t, manager.saveMetadata(meta)) - inst, err := manager.applyForkTargetState(ctx, forkID, StateStopped) + inst, err := manager.applyForkTargetState(ctx, forkID, StateStopped, restoreInstanceOptions{}) require.NoError(t, err) require.Equal(t, StateStopped, inst.State) require.Equal(t, generateVsockCID(forkID), inst.VsockCID) diff --git a/lib/instances/guest_resume_network.go b/lib/instances/guest_resume_network.go new file mode 100644 index 00000000..af72fabb --- /dev/null +++ b/lib/instances/guest_resume_network.go @@ -0,0 +1,556 @@ +package instances + +import ( + "bufio" + "bytes" + "context" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "log/slog" + stdnet "net" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/logger" + "go.opentelemetry.io/otel/attribute" + "golang.org/x/sys/unix" +) + +const guestResumeNetworkPortEnv = "HYPEMAN_RESUME_NETWORK_PORT" +const guestResumeNetworkPrearmEnv = "HYPEMAN_RESUME_NETWORK_PREARM" +const guestResumeNetworkStartArmedEnv = "HYPEMAN_RESUME_NETWORK_START_ARMED" +const guestResumeNetworkArmedPollIntervalEnv = "HYPEMAN_RESUME_NETWORK_ARMED_POLL_INTERVAL_MS" +const guestResumeNetworkPrearmSettleEnv = "HYPEMAN_RESUME_NETWORK_PREARM_SETTLE_MS" +const guestResumeNetworkMailboxEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX" +const guestResumeNetworkMailboxTokenEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX_TOKEN" +const guestResumeNetworkAckPortEnv = "HYPEMAN_RESUME_NETWORK_ACK_PORT" +const firecrackerRestoreMetadataFile = "firecracker-config.json" +const firecrackerSnapshotMemoryFile = "memory" + +const guestResumeNetworkMailboxSeqOffset = 64 +const guestResumeNetworkMailboxLengthOffset = 68 +const guestResumeNetworkMailboxPayloadOffset = 72 + +var guestResumeNetworkMailboxMagic = []byte("HYPEMAN_RESUME_NETWORK_MAILBOX_V1\x00") +var guestResumeNetworkMailboxOffsets sync.Map + +type guestResumeNetworkPayload struct { + InterfaceName string `json:"interface_name"` + MAC string `json:"mac"` + IPv4 string `json:"ipv4"` + Prefix uint32 `json:"prefix"` + Gateway string `json:"gateway"` + AckPort uint32 `json:"ack_port,omitempty"` +} + +type guestResumeNetworkResult struct { + ack string + elapsed time.Duration + err error +} + +type guestResumeNetworkUDPAck struct { + received time.Time + text string +} + +type guestResumeNetworkUDPWaiter struct { + conn *stdnet.UDPConn + ch chan guestResumeNetworkUDPAck +} + +type guestResumeNetworkServer struct { + path string + payload *guestResumeNetworkPayload + + listener stdnet.Listener + done chan guestResumeNetworkResult + armed chan struct{} + close chan struct{} + once sync.Once +} + +func guestInitiatedResumeNetworkPort(stored *StoredMetadata) int { + if stored == nil || stored.HypervisorType != hypervisor.TypeFirecracker { + return 0 + } + rawPort := strings.TrimSpace(stored.Env[guestResumeNetworkPortEnv]) + if rawPort == "" { + return 0 + } + port, err := strconv.Atoi(rawPort) + if err != nil || port <= 0 { + return 0 + } + return port +} + +func guestInitiatedResumeNetworkPrearm(stored *StoredMetadata) bool { + return guestInitiatedResumeNetworkPort(stored) != 0 && strings.TrimSpace(stored.Env[guestResumeNetworkPrearmEnv]) == "1" +} + +func guestInitiatedResumeNetworkMailbox(stored *StoredMetadata) bool { + return guestInitiatedResumeNetworkPrearm(stored) && strings.TrimSpace(stored.Env[guestResumeNetworkMailboxEnv]) == "1" +} + +func guestInitiatedResumeNetworkMailboxToken(stored *StoredMetadata) string { + if stored == nil { + return "" + } + return strings.TrimSpace(stored.Env[guestResumeNetworkMailboxTokenEnv]) +} + +func guestInitiatedResumeNetworkAckPort(stored *StoredMetadata) uint32 { + if stored == nil { + return 0 + } + raw := strings.TrimSpace(stored.Env[guestResumeNetworkAckPortEnv]) + if raw == "" { + return 0 + } + port, err := strconv.Atoi(raw) + if err != nil || port <= 0 || port > 65535 { + return 0 + } + return uint32(port) +} + +func guestInitiatedResumeNetworkStartArmed(stored *StoredMetadata) bool { + return guestInitiatedResumeNetworkPrearm(stored) && strings.TrimSpace(stored.Env[guestResumeNetworkStartArmedEnv]) == "1" +} + +func guestInitiatedResumeNetworkArmedPollIntervalMS(stored *StoredMetadata) uint32 { + if stored == nil { + return 0 + } + raw := strings.TrimSpace(stored.Env[guestResumeNetworkArmedPollIntervalEnv]) + if raw == "" { + return 0 + } + ms, err := strconv.Atoi(raw) + if err != nil || ms <= 0 { + return 0 + } + return uint32(ms) +} + +func guestInitiatedResumeNetworkPrearmSettle(stored *StoredMetadata) time.Duration { + if stored == nil { + return 0 + } + raw := strings.TrimSpace(stored.Env[guestResumeNetworkPrearmSettleEnv]) + if raw == "" { + return 0 + } + ms, err := strconv.Atoi(raw) + if err != nil || ms <= 0 { + return 0 + } + return time.Duration(ms) * time.Millisecond +} + +func guestInitiatedResumeNetworkSocket(stored *StoredMetadata) string { + if stored == nil || stored.HypervisorType != hypervisor.TypeFirecracker { + return "" + } + + sourceDataDir := firecrackerSnapshotSourceDataDir(stored.DataDir) + if sourceDataDir == "" || filepath.Clean(sourceDataDir) == filepath.Clean(stored.DataDir) { + return stored.VsockSocket + } + return filepath.Join(sourceDataDir, filepath.Base(stored.VsockSocket)) +} + +func firecrackerSnapshotSourceDataDir(dataDir string) string { + if dataDir == "" { + return "" + } + + data, err := os.ReadFile(filepath.Join(dataDir, firecrackerRestoreMetadataFile)) + if err != nil { + return "" + } + var meta struct { + SnapshotSourceDataDir string `json:"snapshot_source_data_dir"` + } + if err := json.Unmarshal(data, &meta); err != nil { + return "" + } + return strings.TrimSpace(meta.SnapshotSourceDataDir) +} + +func startGuestResumeNetworkServer(ctx context.Context, vsockSocket string, port int, payload *guestResumeNetworkPayload) (*guestResumeNetworkServer, error) { + path := fmt.Sprintf("%s_%d", vsockSocket, port) + _ = os.Remove(path) + + listener, err := stdnet.Listen("unix", path) + if err != nil { + return nil, fmt.Errorf("listen on guest-initiated vsock socket %s: %w", path, err) + } + + s := &guestResumeNetworkServer{ + path: path, + payload: payload, + listener: listener, + done: make(chan guestResumeNetworkResult, 1), + armed: make(chan struct{}, 1), + close: make(chan struct{}), + } + + go func() { + <-ctx.Done() + s.Close() + }() + go s.acceptLoop() + + return s, nil +} + +func (s *guestResumeNetworkServer) Close() { + s.once.Do(func() { + close(s.close) + _ = s.listener.Close() + _ = os.Remove(s.path) + }) +} + +func (s *guestResumeNetworkServer) WaitArmed(ctx context.Context) error { + select { + case <-s.armed: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *guestResumeNetworkServer) WaitApplied(ctx context.Context) (time.Duration, string, error) { + select { + case result := <-s.done: + return result.elapsed, result.ack, result.err + case <-ctx.Done(): + return 0, "", ctx.Err() + } +} + +func (s *guestResumeNetworkServer) acceptLoop() { + for { + conn, err := s.listener.Accept() + if err != nil { + select { + case <-s.close: + return + default: + s.reportDone(guestResumeNetworkResult{err: err}) + return + } + } + go s.handleConn(conn) + } +} + +func (s *guestResumeNetworkServer) handleConn(conn stdnet.Conn) { + defer conn.Close() + + _ = conn.SetDeadline(time.Now().Add(10 * time.Second)) + reader := bufio.NewReader(conn) + line, err := reader.ReadString('\n') + if err != nil { + s.reportDone(guestResumeNetworkResult{err: fmt.Errorf("read resume network command: %w", err)}) + return + } + + switch strings.TrimSpace(line) { + case "HELLO": + slog.Info("guest resume network control connection armed", "socket", s.path) + select { + case s.armed <- struct{}{}: + default: + } + _ = conn.SetDeadline(time.Time{}) + _, _ = io.Copy(io.Discard, reader) + case "FETCH": + slog.Info("guest resume network config fetch received", "socket", s.path) + s.handleFetch(conn, reader) + default: + s.reportDone(guestResumeNetworkResult{err: fmt.Errorf("unexpected resume network command %q", strings.TrimSpace(line))}) + } +} + +func (s *guestResumeNetworkServer) handleFetch(conn stdnet.Conn, reader *bufio.Reader) { + start := time.Now() + if s.payload == nil { + s.reportDone(guestResumeNetworkResult{err: fmt.Errorf("resume network fetch received without payload")}) + return + } + + if err := json.NewEncoder(conn).Encode(s.payload); err != nil { + s.reportDone(guestResumeNetworkResult{err: fmt.Errorf("write resume network payload: %w", err)}) + return + } + + ack, err := reader.ReadString('\n') + elapsed := time.Since(start) + if err != nil { + s.reportDone(guestResumeNetworkResult{elapsed: elapsed, err: fmt.Errorf("read resume network ack: %w", err)}) + return + } + ack = strings.TrimSpace(ack) + if !strings.HasPrefix(ack, "OK") { + s.reportDone(guestResumeNetworkResult{elapsed: elapsed, ack: ack, err: fmt.Errorf("resume network failed: %s", ack)}) + return + } + s.reportDone(guestResumeNetworkResult{elapsed: elapsed, ack: ack}) +} + +func (s *guestResumeNetworkServer) reportDone(result guestResumeNetworkResult) { + select { + case s.done <- result: + go s.Close() + default: + } +} + +func newGuestResumeNetworkPayload(stored *StoredMetadata, allocConfig *guestNetworkConfig) guestResumeNetworkPayload { + return guestResumeNetworkPayload{ + InterfaceName: "eth0", + MAC: allocConfig.mac, + IPv4: allocConfig.ip, + Prefix: uint32(allocConfig.prefix), + Gateway: allocConfig.gateway, + AckPort: guestInitiatedResumeNetworkAckPort(stored), + } +} + +func startGuestResumeNetworkUDPWaiter() (*guestResumeNetworkUDPWaiter, error) { + conn, err := stdnet.ListenUDP("udp4", &stdnet.UDPAddr{IP: stdnet.IPv4zero, Port: 0}) + if err != nil { + return nil, fmt.Errorf("listen for guest resume network UDP ack: %w", err) + } + + w := &guestResumeNetworkUDPWaiter{ + conn: conn, + ch: make(chan guestResumeNetworkUDPAck, 128), + } + go w.readLoop() + return w, nil +} + +func (w *guestResumeNetworkUDPWaiter) Port() uint32 { + if w == nil || w.conn == nil { + return 0 + } + return uint32(w.conn.LocalAddr().(*stdnet.UDPAddr).Port) +} + +func (w *guestResumeNetworkUDPWaiter) Close() { + if w == nil || w.conn == nil { + return + } + _ = w.conn.Close() +} + +func (w *guestResumeNetworkUDPWaiter) readLoop() { + buf := make([]byte, 1024) + for { + n, _, err := w.conn.ReadFromUDP(buf) + if err != nil { + return + } + w.ch <- guestResumeNetworkUDPAck{ + received: time.Now(), + text: strings.TrimSpace(string(buf[:n])), + } + } +} + +func (w *guestResumeNetworkUDPWaiter) WaitApplied(ctx context.Context, mac, ip string) (time.Duration, string, error) { + if w == nil { + return 0, "", fmt.Errorf("guest resume network UDP waiter is nil") + } + + start := time.Now() + wantMAC := "mac=" + strings.ToLower(mac) + wantIP := "ip=" + ip + for { + select { + case ack := <-w.ch: + text := strings.ToLower(ack.text) + if strings.Contains(text, "stage=applied") && strings.Contains(text, wantMAC) && strings.Contains(text, wantIP) { + return ack.received.Sub(start), ack.text, nil + } + case <-ctx.Done(): + return 0, "", ctx.Err() + } + } +} + +func patchGuestResumeNetworkMailbox(snapshotDir, token string, payload *guestResumeNetworkPayload) error { + if token == "" { + return fmt.Errorf("resume network mailbox token is empty") + } + if payload == nil { + return fmt.Errorf("resume network mailbox payload is nil") + } + + payloadBytes, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("marshal resume network mailbox payload: %w", err) + } + if len(payloadBytes) > 4096-guestResumeNetworkMailboxPayloadOffset { + return fmt.Errorf("resume network mailbox payload too large: %d bytes", len(payloadBytes)) + } + + memoryPath := filepath.Join(snapshotDir, firecrackerSnapshotMemoryFile) + file, err := os.OpenFile(memoryPath, os.O_RDWR, 0) + if err != nil { + return fmt.Errorf("open snapshot memory for resume network mailbox: %w", err) + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return fmt.Errorf("stat snapshot memory for resume network mailbox: %w", err) + } + if info.Size() <= 0 { + return fmt.Errorf("snapshot memory file is empty") + } + if info.Size() > int64(int(info.Size())) { + return fmt.Errorf("snapshot memory file too large to map: %d bytes", info.Size()) + } + + marker := make([]byte, 0, len(guestResumeNetworkMailboxMagic)+len(token)) + marker = append(marker, guestResumeNetworkMailboxMagic...) + marker = append(marker, token...) + + idx, err := findGuestResumeNetworkMailbox(file, info.Size(), marker, token) + if err != nil { + return err + } + if idx+int64(guestResumeNetworkMailboxPayloadOffset)+int64(len(payloadBytes)) > info.Size() { + return fmt.Errorf("resume network mailbox marker is too close to end of memory file") + } + + if _, err := file.WriteAt(payloadBytes, idx+int64(guestResumeNetworkMailboxPayloadOffset)); err != nil { + return fmt.Errorf("write resume network mailbox payload: %w", err) + } + var u32 [4]byte + binary.LittleEndian.PutUint32(u32[:], uint32(len(payloadBytes))) + if _, err := file.WriteAt(u32[:], idx+int64(guestResumeNetworkMailboxLengthOffset)); err != nil { + return fmt.Errorf("write resume network mailbox payload length: %w", err) + } + binary.LittleEndian.PutUint32(u32[:], 1) + if _, err := file.WriteAt(u32[:], idx+int64(guestResumeNetworkMailboxSeqOffset)); err != nil { + return fmt.Errorf("write resume network mailbox sequence: %w", err) + } + return nil +} + +func findGuestResumeNetworkMailbox(file *os.File, size int64, marker []byte, token string) (int64, error) { + if cached, ok := guestResumeNetworkMailboxOffsets.Load(token); ok { + idx := cached.(int64) + if idx >= 0 && idx+int64(len(marker)) <= size { + buf := make([]byte, len(marker)) + if _, err := file.ReadAt(buf, idx); err == nil && bytes.Equal(buf, marker) { + return idx, nil + } + } + } + + data, err := unix.Mmap(int(file.Fd()), 0, int(size), unix.PROT_READ, unix.MAP_SHARED) + if err != nil { + return 0, fmt.Errorf("mmap snapshot memory for resume network mailbox: %w", err) + } + defer unix.Munmap(data) + + idx := bytes.Index(data, marker) + if idx < 0 { + return 0, fmt.Errorf("resume network mailbox marker not found") + } + guestResumeNetworkMailboxOffsets.Store(token, int64(idx)) + return int64(idx), nil +} + +func (m *manager) startGuestInitiatedResumeNetwork(ctx context.Context, stored *StoredMetadata, vsockSocket string, allocConfig *guestNetworkConfig) (*guestResumeNetworkServer, context.CancelFunc, error) { + port := guestInitiatedResumeNetworkPort(stored) + if port == 0 { + return nil, nil, nil + } + if vsockSocket == "" { + vsockSocket = stored.VsockSocket + } + + payload := newGuestResumeNetworkPayload(stored, allocConfig) + serverCtx, cancel := context.WithCancel(ctx) + server, err := startGuestResumeNetworkServer(serverCtx, vsockSocket, port, &payload) + if err != nil { + cancel() + return nil, nil, err + } + return server, cancel, nil +} + +func (m *manager) armGuestInitiatedResumeNetwork(ctx context.Context, stored *StoredMetadata) error { + if !guestInitiatedResumeNetworkPrearm(stored) { + return nil + } + + dialer, err := hypervisor.NewVsockDialer(stored.HypervisorType, stored.VsockSocket, stored.VsockCID) + if err != nil { + return fmt.Errorf("create vsock dialer: %w", err) + } + + return guest.ArmResumeNetworkInInstance(ctx, dialer, guest.ArmResumeNetworkOptions{ + PollIntervalMS: guestInitiatedResumeNetworkArmedPollIntervalMS(stored), + WaitForAgent: 120 * time.Second, + }) +} + +func (m *manager) waitForGuestInitiatedResumeNetwork(ctx context.Context, server *guestResumeNetworkServer, stored *StoredMetadata) error { + log := logger.FromContext(ctx) + waitCtx, waitSpanEnd := m.startLifecycleStep(ctx, "guest.resume_network.wait", + attribute.String("instance_id", stored.Id), + attribute.String("hypervisor", string(stored.HypervisorType)), + attribute.String("operation", "guest_resume_network_wait"), + ) + waitCtx, cancel := context.WithTimeout(waitCtx, 2*time.Second) + defer cancel() + + elapsed, ack, err := server.WaitApplied(waitCtx) + waitSpanEnd(err) + if err != nil { + return err + } + log.InfoContext(ctx, "guest-initiated resume network applied", "instance_id", stored.Id, "elapsed", elapsed, "ack", ack) + return nil +} + +func (m *manager) waitForGuestResumeNetworkUDPAck(ctx context.Context, waiter *guestResumeNetworkUDPWaiter, stored *StoredMetadata, cfg *guestNetworkConfig) error { + if waiter == nil || cfg == nil { + return nil + } + + log := logger.FromContext(ctx) + waitCtx, waitSpanEnd := m.startLifecycleStep(ctx, "guest.resume_network.udp_ack_wait", + attribute.String("instance_id", stored.Id), + attribute.String("hypervisor", string(stored.HypervisorType)), + attribute.String("operation", "guest_resume_network_udp_ack_wait"), + ) + waitCtx, cancel := context.WithTimeout(waitCtx, 2*time.Second) + defer cancel() + + elapsed, ack, err := waiter.WaitApplied(waitCtx, cfg.mac, cfg.ip) + waitSpanEnd(err) + if err != nil { + return err + } + log.InfoContext(ctx, "guest resume network UDP ack received", "instance_id", stored.Id, "elapsed", elapsed, "ack", ack) + return nil +} diff --git a/lib/instances/manager.go b/lib/instances/manager.go index abdd2165..c55ba93a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -389,14 +389,16 @@ func (m *manager) ForkInstance(ctx context.Context, id string, req ForkInstanceR return nil, err } - inst, err := m.applyForkTargetState(ctx, forked.Id, targetState) + inst, err := m.applyForkTargetState(ctx, forked.Id, targetState, restoreInstanceOptions{ + WaitForGuestNetwork: req.WaitForNetwork, + }) if err != nil { if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { return nil, fmt.Errorf("apply fork target state: %w; additionally failed to cleanup forked instance %s: %v", err, forked.Id, cleanupErr) } return nil, fmt.Errorf("apply fork target state: %w", err) } - if inst.State == StateRunning { + if inst.State == StateRunning && !guestInitiatedResumeNetworkMailbox(&inst.StoredMetadata) { if err := ensureGuestAgentReadyForForkPhase(ctx, &inst.StoredMetadata, "before returning running fork instance"); err != nil { if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { return nil, fmt.Errorf("wait for fork guest agent readiness: %w; additionally failed to cleanup forked instance %s: %v", err, forked.Id, cleanupErr) @@ -449,7 +451,7 @@ func (m *manager) RestoreInstance(ctx context.Context, id string) (*Instance, er if current.State == StateRunning || current.State == StateInitializing { return current, nil } - inst, err := m.restoreInstance(ctx, id) + inst, err := m.restoreInstance(ctx, id, restoreInstanceOptions{}) if err == nil { m.notifyLifecycleEvent(ctx, LifecycleEventRestore, inst) } diff --git a/lib/instances/reconfigure_guest_network_perf_test.go b/lib/instances/reconfigure_guest_network_perf_test.go index fcda012c..389dd28f 100644 --- a/lib/instances/reconfigure_guest_network_perf_test.go +++ b/lib/instances/reconfigure_guest_network_perf_test.go @@ -5,6 +5,7 @@ package instances import ( "context" "fmt" + "net" "os" "strconv" "strings" @@ -24,6 +25,21 @@ import ( ) const reconfigureGuestNetworkPerfEnv = "HYPEMAN_RUN_RECONFIGURE_GUEST_NETWORK_PERF" +const guestInitiatedResumeNetworkPerfEnv = "HYPEMAN_RECONFIGURE_GUEST_NETWORK_GUEST_INITIATED" +const guestInitiatedResumeNetworkPollPerfEnv = "HYPEMAN_RECONFIGURE_GUEST_NETWORK_GUEST_POLL_MS" +const guestInitiatedResumeNetworkTriggerPerfEnv = "HYPEMAN_RECONFIGURE_GUEST_NETWORK_GUEST_TRIGGER" +const guestInitiatedResumeNetworkPrearmPerfEnv = "HYPEMAN_RECONFIGURE_GUEST_NETWORK_GUEST_PREARM" +const guestInitiatedResumeNetworkStartArmedPerfEnv = "HYPEMAN_RECONFIGURE_GUEST_NETWORK_GUEST_START_ARMED" +const guestInitiatedResumeNetworkArmedPollPerfEnv = "HYPEMAN_RECONFIGURE_GUEST_NETWORK_GUEST_ARMED_POLL_MS" +const guestInitiatedResumeNetworkPrearmSettlePerfEnv = "HYPEMAN_RECONFIGURE_GUEST_NETWORK_GUEST_PREARM_SETTLE_MS" +const guestInitiatedResumeNetworkMailboxPerfEnv = "HYPEMAN_RECONFIGURE_GUEST_NETWORK_GUEST_MAILBOX" +const guestInitiatedResumeNetworkWaitAckPerfEnv = "HYPEMAN_RECONFIGURE_GUEST_NETWORK_GUEST_WAIT_ACK" +const reconfigureGuestNetworkPerfVCPUsEnv = "HYPEMAN_RECONFIGURE_GUEST_NETWORK_PERF_VCPUS" + +type resumeNetworkAck struct { + received time.Time + text string +} func TestReconfigureGuestNetworkPerf(t *testing.T) { if os.Getenv(reconfigureGuestNetworkPerfEnv) != "1" { @@ -57,15 +73,54 @@ func TestReconfigureGuestNetworkPerf(t *testing.T) { require.NoError(t, systemManager.EnsureSystemFiles(ctx)) require.NoError(t, mgr.networkManager.Initialize(ctx, nil)) + var ackCh <-chan resumeNetworkAck + expectMailboxAck := false + waitAckInRestore := os.Getenv(guestInitiatedResumeNetworkWaitAckPerfEnv) == "1" + env := map[string]string(nil) + if os.Getenv(guestInitiatedResumeNetworkPerfEnv) == "1" { + env = map[string]string{guestResumeNetworkPortEnv: "2224"} + if !waitAckInRestore { + ackPort, ch := startResumeNetworkAckListener(t) + env[guestResumeNetworkAckPortEnv] = strconv.Itoa(ackPort) + ackCh = ch + } + if os.Getenv(guestInitiatedResumeNetworkMailboxPerfEnv) == "1" { + env["HYPEMAN_RESUME_NETWORK_MAILBOX"] = "1" + env["HYPEMAN_RESUME_NETWORK_MAILBOX_TOKEN"] = fmt.Sprintf("perf-%d", time.Now().UnixNano()) + expectMailboxAck = true + } + if trigger := strings.TrimSpace(os.Getenv(guestInitiatedResumeNetworkTriggerPerfEnv)); trigger != "" { + env["HYPEMAN_RESUME_NETWORK_TRIGGER"] = trigger + } + if os.Getenv(guestInitiatedResumeNetworkPrearmPerfEnv) == "1" { + env["HYPEMAN_RESUME_NETWORK_PREARM"] = "1" + env["HYPEMAN_RESUME_NETWORK_SLOW_POLL_INTERVAL_MS"] = "1000" + env["HYPEMAN_RESUME_NETWORK_ARMED_POLL_INTERVAL_MS"] = "1" + if armedPollMS := strings.TrimSpace(os.Getenv(guestInitiatedResumeNetworkArmedPollPerfEnv)); armedPollMS != "" { + env["HYPEMAN_RESUME_NETWORK_ARMED_POLL_INTERVAL_MS"] = armedPollMS + } + if os.Getenv(guestInitiatedResumeNetworkStartArmedPerfEnv) == "1" { + env["HYPEMAN_RESUME_NETWORK_START_ARMED"] = "1" + } + if settleMS := strings.TrimSpace(os.Getenv(guestInitiatedResumeNetworkPrearmSettlePerfEnv)); settleMS != "" { + env["HYPEMAN_RESUME_NETWORK_PREARM_SETTLE_MS"] = settleMS + } + } + if pollMS := strings.TrimSpace(os.Getenv(guestInitiatedResumeNetworkPollPerfEnv)); pollMS != "" { + env["HYPEMAN_RESUME_NETWORK_POLL_INTERVAL_MS"] = pollMS + } + } + source, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ Name: "fc-reconfigure-perf-src", Image: imageName, Size: 1024 * 1024 * 1024, OverlaySize: 1024 * 1024 * 1024, - Vcpus: 1, + Vcpus: perfVCPUsFromEnv(t, 1), NetworkEnabled: true, Hypervisor: hypervisor.TypeFirecracker, Cmd: []string{"sleep", "infinity"}, + Env: env, }) require.NoError(t, err) sourceID := source.Id @@ -75,6 +130,21 @@ func TestReconfigureGuestNetworkPerf(t *testing.T) { require.NoError(t, err) require.NoError(t, waitForExecAgent(ctx, mgr, sourceID, 45*time.Second)) + var sourceResumeNetworkServer *guestResumeNetworkServer + if os.Getenv(guestInitiatedResumeNetworkPerfEnv) == "1" && + os.Getenv(guestInitiatedResumeNetworkPollPerfEnv) == "" && + os.Getenv(guestInitiatedResumeNetworkPrearmPerfEnv) != "1" && + strings.TrimSpace(os.Getenv(guestInitiatedResumeNetworkTriggerPerfEnv)) == "" { + port := guestInitiatedResumeNetworkPort(&source.StoredMetadata) + require.NotZero(t, port) + sourceResumeNetworkServer, err = startGuestResumeNetworkServer(ctx, source.VsockSocket, port, nil) + require.NoError(t, err) + t.Cleanup(sourceResumeNetworkServer.Close) + armCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + require.NoError(t, sourceResumeNetworkServer.WaitArmed(armCtx)) + cancel() + } + snapshot, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ Kind: SnapshotKindStandby, Name: "fc-reconfigure-perf-snap", @@ -84,22 +154,110 @@ func TestReconfigureGuestNetworkPerf(t *testing.T) { for i := 1; i <= iterations; i++ { before := len(recorder.Ended()) + waitForNetwork := waitAckInRestore start := time.Now() fork, err := mgr.ForkSnapshot(ctx, snapshot.Id, ForkSnapshotRequest{ - Name: fmt.Sprintf("fc-reconfigure-perf-fork-%02d", i), - TargetState: StateRunning, + Name: fmt.Sprintf("fc-reconfigure-perf-fork-%02d", i), + TargetState: StateRunning, + WaitForNetwork: &waitForNetwork, }) forkElapsed := time.Since(start) require.NoError(t, err) require.Equal(t, StateRunning, fork.State) + var mailboxAck *resumeNetworkAck + var appliedAck *resumeNetworkAck + if ackCh != nil { + if expectMailboxAck { + mailboxAck = waitForResumeNetworkAck(t, ackCh, fork, "mailbox", 2*time.Second) + } + appliedAck = waitForResumeNetworkAck(t, ackCh, fork, "applied", 2*time.Second) + } require.NoError(t, waitForExecAgent(ctx, mgr, fork.Id, 45*time.Second)) + requireGuestNetworkIdentity(t, fork) spans := append([]sdktrace.ReadOnlySpan(nil), recorder.Ended()[before:]...) t.Log(formatReconfigurePerfLine(i, forkElapsed, spans)) + if mailboxAck != nil { + t.Log(formatResumeNetworkAckLine(i, start, *mailboxAck, spans)) + } + if appliedAck != nil { + t.Log(formatResumeNetworkAckLine(i, start, *appliedAck, spans)) + } _ = mgr.DeleteInstance(ctx, fork.Id) } } +func requireGuestNetworkIdentity(t *testing.T, inst *Instance) { + t.Helper() + + output, exitCode, err := execInInstance(context.Background(), inst, "sh", "-c", "printf 'mac='; cat /sys/class/net/eth0/address; printf 'cidrs='; ip -4 -o addr show dev eth0 scope global | awk '{print $4}'") + require.NoError(t, err) + require.Equal(t, 0, exitCode, "guest network identity command output: %s", output) + require.Contains(t, strings.ToLower(output), "mac="+strings.ToLower(inst.MAC)) + require.Contains(t, output, inst.IP+"/") +} + +func startResumeNetworkAckListener(t *testing.T) (int, <-chan resumeNetworkAck) { + t.Helper() + + conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0}) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + ch := make(chan resumeNetworkAck, 128) + go func() { + buf := make([]byte, 1024) + for { + n, _, err := conn.ReadFromUDP(buf) + if err != nil { + return + } + ch <- resumeNetworkAck{ + received: time.Now(), + text: strings.TrimSpace(string(buf[:n])), + } + } + }() + return conn.LocalAddr().(*net.UDPAddr).Port, ch +} + +func waitForResumeNetworkAck(t *testing.T, ch <-chan resumeNetworkAck, inst *Instance, stage string, timeout time.Duration) *resumeNetworkAck { + t.Helper() + + timer := time.NewTimer(timeout) + defer timer.Stop() + wantStage := "stage=" + stage + wantMAC := "mac=" + strings.ToLower(inst.MAC) + wantIP := "ip=" + inst.IP + for { + select { + case ack := <-ch: + text := strings.ToLower(ack.text) + if strings.Contains(text, wantStage) && strings.Contains(text, wantMAC) && strings.Contains(text, wantIP) { + return &ack + } + case <-timer.C: + require.Failf(t, "timed out waiting for guest resume network ack", "stage=%s instance=%s mac=%s ip=%s", stage, inst.Id, inst.MAC, inst.IP) + return nil + } + } +} + +func formatResumeNetworkAckLine(iter int, forkStart time.Time, ack resumeNetworkAck, spans []sdktrace.ReadOnlySpan) string { + resume := lastSpanNamed(spans, "resume_vm") + afterResumeMS := int64(-1) + if resume != nil { + afterResumeMS = ack.received.Sub(resume.EndTime()).Milliseconds() + } + return fmt.Sprintf( + "PERF_ACK iter=%d guest_network_ack_after_fork_start_ms=%d guest_network_ack_after_resume_ms=%d ack=%q", + iter, + ack.received.Sub(forkStart).Milliseconds(), + afterResumeMS, + ack.text, + ) +} + func perfIterationsFromEnv(t *testing.T, fallback int) int { t.Helper() raw := strings.TrimSpace(os.Getenv("HYPEMAN_RECONFIGURE_GUEST_NETWORK_PERF_ITERS")) @@ -112,6 +270,18 @@ func perfIterationsFromEnv(t *testing.T, fallback int) int { return n } +func perfVCPUsFromEnv(t *testing.T, fallback int) int { + t.Helper() + raw := strings.TrimSpace(os.Getenv(reconfigureGuestNetworkPerfVCPUsEnv)) + if raw == "" { + return fallback + } + n, err := strconv.Atoi(raw) + require.NoError(t, err) + require.Positive(t, n) + return n +} + func formatReconfigurePerfLine(iter int, forkElapsed time.Duration, spans []sdktrace.ReadOnlySpan) string { reconfigure := lastSpanNamed(spans, "reconfigure_guest_network") if reconfigure == nil { @@ -121,6 +291,8 @@ func formatReconfigurePerfLine(iter int, forkElapsed time.Duration, spans []sdkt desc := descendantSpans(spans, reconfigure) exec := lastSpanNamed(desc, "guest.exec") networkRPC := lastSpanNamed(desc, "guest.reconfigure_network") + resumeNetworkWait := lastSpanNamed(desc, "guest.resume_network.wait") + resumeNetworkUDPAckWait := lastSpanNamed(desc, "guest.resume_network.udp_ack_wait") getConn := lastSpanNamed(desc, "guest.exec.get_conn") openStream := lastSpanNamed(desc, "guest.exec.open_stream") sendStart := lastSpanNamed(desc, "guest.exec.send_start") @@ -131,12 +303,14 @@ func formatReconfigurePerfLine(iter int, forkElapsed time.Duration, spans []sdkt vsockReadOK := lastSpanNamed(desc, "hypervisor.vsock.read_ok") return fmt.Sprintf( - "PERF iter=%d fork_total_ms=%d reconfigure_guest_network_ms=%d guest_exec_ms=%d guest_network_rpc_ms=%d get_conn_ms=%d open_stream_ms=%d send_start_ms=%d recv_until_exit_ms=%d vsock_dial_ms=%d vsock_unix_dial_ms=%d vsock_write_connect_ms=%d vsock_read_ok_ms=%d attempts=%d retryable_attempts=%d network_rpc_attempts=%d network_rpc_retryable_attempts=%d first_retryable_error=%s last_retryable_error=%s wait_elapsed_ms=%d open_stream_attempts=%s recv_attempts=%s unary_attempts=%s vsock_dial_attempts=%s vsock_unix_dial_attempts=%s vsock_write_connect_attempts=%s vsock_read_ok_attempts=%s", + "PERF iter=%d fork_total_ms=%d reconfigure_guest_network_ms=%d guest_exec_ms=%d guest_network_rpc_ms=%d guest_resume_network_wait_ms=%d guest_resume_network_udp_ack_wait_ms=%d get_conn_ms=%d open_stream_ms=%d send_start_ms=%d recv_until_exit_ms=%d vsock_dial_ms=%d vsock_unix_dial_ms=%d vsock_write_connect_ms=%d vsock_read_ok_ms=%d attempts=%d retryable_attempts=%d network_rpc_attempts=%d network_rpc_retryable_attempts=%d first_retryable_error=%s last_retryable_error=%s wait_elapsed_ms=%d open_stream_attempts=%s recv_attempts=%s unary_attempts=%s guest_resume_network_wait_attempts=%s guest_resume_network_udp_ack_wait_attempts=%s vsock_dial_attempts=%s vsock_unix_dial_attempts=%s vsock_write_connect_attempts=%s vsock_read_ok_attempts=%s", iter, forkElapsed.Milliseconds(), spanDurationMS(reconfigure), spanDurationMS(exec), spanDurationMS(networkRPC), + spanDurationMS(resumeNetworkWait), + spanDurationMS(resumeNetworkUDPAckWait), spanDurationMS(getConn), spanDurationMS(openStream), spanDurationMS(sendStart), @@ -155,6 +329,8 @@ func formatReconfigurePerfLine(iter int, forkElapsed time.Duration, spans []sdkt spanDurationsByName(desc, "guest.exec.open_stream"), spanDurationsByName(desc, "guest.exec.recv_until_exit"), spanDurationsByName(desc, "guest.reconfigure_network.rpc"), + spanDurationsByName(desc, "guest.resume_network.wait"), + spanDurationsByName(desc, "guest.resume_network.udp_ack_wait"), spanDurationsByName(desc, "hypervisor.vsock.dial"), spanDurationsByName(desc, "hypervisor.vsock.unix_dial"), spanDurationsByName(desc, "hypervisor.vsock.write_connect"), diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 28f88007..7d65f44c 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -20,12 +20,20 @@ import ( "google.golang.org/grpc/status" ) +type restoreInstanceOptions struct { + WaitForGuestNetwork *bool +} + +func (o restoreInstanceOptions) waitForGuestNetwork() bool { + return o.WaitForGuestNetwork == nil || *o.WaitForGuestNetwork +} + // RestoreInstance restores an instance from standby // Multi-hop orchestration: Standby → Paused → Running func (m *manager) restoreInstance( ctx context.Context, - id string, + opts restoreInstanceOptions, ) (_ *Instance, retErr error) { start := time.Now() log := logger.FromContext(ctx) @@ -46,6 +54,7 @@ func (m *manager) restoreInstance( inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata + waitForGuestNetwork := opts.waitForGuestNetwork() ctx = enrichInstancesTrace(ctx, attribute.String("hypervisor", string(stored.HypervisorType))) log.DebugContext(ctx, "loaded instance", "instance_id", id, "state", inst.State, "has_snapshot", inst.HasSnapshot) @@ -229,6 +238,72 @@ func (m *manager) restoreInstance( proxyRegistered = true } + resumeNetworkVsockSocket := guestInitiatedResumeNetworkSocket(stored) + var resumeNetworkServer *guestResumeNetworkServer + var resumeNetworkAckWaiter *guestResumeNetworkUDPWaiter + var cancelResumeNetworkServer context.CancelFunc + var resumeNetworkAckCfg *guestNetworkConfig + resumeNetworkMailboxPatched := false + if stored.NetworkEnabled && !stored.SkipGuestAgent { + var resumeNetworkCfg *guestNetworkConfig + if allocatedNet != nil { + resumeNetworkCfg, err = guestNetworkReconfigureConfig(allocatedNet) + } else if guestInitiatedResumeNetworkPrearm(stored) { + if alloc, allocErr := m.networkManager.GetAllocation(ctx, id); allocErr != nil { + log.WarnContext(ctx, "failed to get network allocation for guest-initiated resume network disarm", "instance_id", id, "error", allocErr) + } else { + resumeNetworkCfg, err = guestNetworkReconfigureConfig(alloc) + } + } + if err != nil { + log.WarnContext(ctx, "failed to build guest-initiated resume network payload", "instance_id", id, "error", err) + err = nil + } + if resumeNetworkCfg != nil && guestInitiatedResumeNetworkMailbox(stored) { + payload := newGuestResumeNetworkPayload(stored, resumeNetworkCfg) + if allocatedNet != nil && waitForGuestNetwork { + resumeNetworkAckWaiter, err = startGuestResumeNetworkUDPWaiter() + if err != nil { + log.ErrorContext(ctx, "failed to start guest resume network UDP ack waiter", "instance_id", id, "error", err) + releaseNetwork() + return nil, fmt.Errorf("start guest resume network UDP ack waiter: %w", err) + } else { + resumeNetworkAckCfg = resumeNetworkCfg + payload.AckPort = resumeNetworkAckWaiter.Port() + } + } + err = patchGuestResumeNetworkMailbox(snapshotDir, guestInitiatedResumeNetworkMailboxToken(stored), &payload) + if err != nil { + log.WarnContext(ctx, "failed to patch guest resume network mailbox; falling back to guest-initiated server", "instance_id", id, "error", err) + err = nil + if resumeNetworkAckWaiter != nil { + resumeNetworkAckWaiter.Close() + resumeNetworkAckWaiter = nil + resumeNetworkAckCfg = nil + } + } else { + resumeNetworkMailboxPatched = true + } + } + if resumeNetworkCfg != nil && !resumeNetworkMailboxPatched { + resumeNetworkServer, cancelResumeNetworkServer, err = m.startGuestInitiatedResumeNetwork(ctx, stored, resumeNetworkVsockSocket, resumeNetworkCfg) + if err != nil { + log.WarnContext(ctx, "failed to start guest-initiated resume network server; falling back to host-initiated reconfigure", "instance_id", id, "error", err) + resumeNetworkServer = nil + cancelResumeNetworkServer = nil + } + } + } + if cancelResumeNetworkServer != nil { + defer cancelResumeNetworkServer() + } + if resumeNetworkServer != nil { + defer resumeNetworkServer.Close() + } + if resumeNetworkAckWaiter != nil { + defer resumeNetworkAckWaiter.Close() + } + // 5. Transition: Standby → Paused (start hypervisor + restore) restoreCtx, restoreSpanEnd := m.startLifecycleStep(ctx, "restore_from_snapshot", attribute.String("instance_id", id), @@ -283,7 +358,19 @@ func (m *manager) restoreInstance( attribute.String("hypervisor", string(stored.HypervisorType)), attribute.String("operation", "reconfigure_guest_network"), ) - if err := reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet); err != nil { + var err error + if resumeNetworkMailboxPatched { + err = m.waitForGuestResumeNetworkUDPAck(reconfigureCtx, resumeNetworkAckWaiter, stored, resumeNetworkAckCfg) + } else if resumeNetworkServer != nil { + err = m.waitForGuestInitiatedResumeNetwork(reconfigureCtx, resumeNetworkServer, stored) + if err != nil { + log.WarnContext(ctx, "guest-initiated resume network failed; falling back to host-initiated reconfigure", "instance_id", id, "error", err) + err = reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet) + } + } else { + err = reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet) + } + if err != nil { reconfigureSpanEnd(err) log.ErrorContext(ctx, "failed to configure guest network after restore", "instance_id", id, "error", err) _ = hv.Shutdown(ctx) @@ -293,6 +380,13 @@ func (m *manager) restoreInstance( } reconfigureSpanEnd(nil) } + if allocatedNet == nil && resumeNetworkMailboxPatched && guestInitiatedResumeNetworkPrearm(stored) { + log.InfoContext(ctx, "guest resume network mailbox patched for source restore", "instance_id", id) + } else if allocatedNet == nil && resumeNetworkServer != nil && guestInitiatedResumeNetworkPrearm(stored) { + if err := m.waitForGuestInitiatedResumeNetwork(ctx, resumeNetworkServer, stored); err != nil { + log.WarnContext(ctx, "guest-initiated resume network source disarm failed", "instance_id", id, "error", err) + } + } // 8. Delete snapshot after successful restore unless the hypervisor is keeping it // as the base for the next standby snapshot. diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 05d8084e..523917b4 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -127,7 +127,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps } if restoreSource { - _, restoreErr := m.restoreInstance(ctx, id) + _, restoreErr := m.restoreInstance(ctx, id, restoreInstanceOptions{}) if restoreErr != nil { if copyErr != nil { return nil, fmt.Errorf("snapshot copy failed: %v; additionally failed to restore source: %w", copyErr, restoreErr) @@ -337,7 +337,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str } return m.getInstance(ctx, id) case StateRunning: - inst, err := m.restoreInstance(ctx, id) + inst, err := m.restoreInstance(ctx, id, restoreInstanceOptions{}) if err != nil { return nil, err } @@ -486,7 +486,9 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS } cu.Release() - inst, err := m.applyForkTargetState(ctx, forkID, targetState) + inst, err := m.applyForkTargetState(ctx, forkID, targetState, restoreInstanceOptions{ + WaitForGuestNetwork: req.WaitForNetwork, + }) if err != nil { if cleanupErr := m.cleanupForkInstanceOnError(ctx, forkID); cleanupErr != nil { return nil, fmt.Errorf("apply snapshot fork target state: %w; additionally failed to cleanup forked instance %s: %v", err, forkID, cleanupErr) diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 03b27410..fda3a805 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -110,6 +110,31 @@ func (m *manager) standbyInstance( return nil, fmt.Errorf("hypervisor %s does not support standby (snapshots)", stored.HypervisorType) } + if guestInitiatedResumeNetworkPrearm(stored) && !guestInitiatedResumeNetworkStartArmed(stored) && !stored.SkipGuestAgent { + armCtx, armSpanEnd := m.startLifecycleStep(ctx, "guest.resume_network.arm", + attribute.String("instance_id", id), + attribute.String("hypervisor", string(stored.HypervisorType)), + attribute.String("operation", "guest_resume_network_arm"), + ) + if err := m.armGuestInitiatedResumeNetwork(armCtx, stored); err != nil { + armSpanEnd(err) + return nil, fmt.Errorf("arm guest-initiated resume network: %w", err) + } + if settle := guestInitiatedResumeNetworkPrearmSettle(stored); settle > 0 { + timer := time.NewTimer(settle) + select { + case <-timer.C: + case <-armCtx.Done(): + if !timer.Stop() { + <-timer.C + } + armSpanEnd(armCtx.Err()) + return nil, armCtx.Err() + } + } + armSpanEnd(nil) + } + // 6. Transition: Running → Paused log.DebugContext(ctx, "pausing VM", "instance_id", id) pauseCtx, pauseSpanEnd := m.startLifecycleStep(ctx, "pause_vm", diff --git a/lib/instances/types.go b/lib/instances/types.go index 8c031fc9..e38f5499 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -268,9 +268,10 @@ type UpdateInstanceRequest struct { // ForkInstanceRequest is the domain request for forking an instance. type ForkInstanceRequest struct { - Name string // Required: name for the new forked instance - FromRunning bool // Optional: allow forking from Running by auto standby/fork/restore - TargetState State // Optional: desired final state of forked instance (Stopped, Standby, Running). Empty means inherit source state. + Name string // Required: name for the new forked instance + FromRunning bool // Optional: allow forking from Running by auto standby/fork/restore + TargetState State // Optional: desired final state of forked instance (Stopped, Standby, Running). Empty means inherit source state. + WaitForNetwork *bool // Optional: wait for guest networking before returning a Running fork. Nil defaults to true. } // SnapshotKind determines how snapshot data is captured and restored. @@ -314,6 +315,7 @@ type ForkSnapshotRequest struct { Name string // Required: name for the new instance TargetState State // Optional TargetHypervisor hypervisor.Type // Optional, allowed only for Stopped snapshots + WaitForNetwork *bool // Optional: wait for guest networking before returning a Running fork. Nil defaults to true. } // SnapshotPolicy defines default snapshot behavior for an instance. diff --git a/lib/instances/vmgenid_resume_signal_test.go b/lib/instances/vmgenid_resume_signal_test.go new file mode 100644 index 00000000..e5148830 --- /dev/null +++ b/lib/instances/vmgenid_resume_signal_test.go @@ -0,0 +1,92 @@ +//go:build linux + +package instances + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + snapshottest "github.com/kernel/hypeman/lib/snapshot/testsupport" + "github.com/kernel/hypeman/lib/system" + "github.com/stretchr/testify/require" +) + +const vmgenidResumeSignalEnv = "HYPEMAN_RUN_VMGENID_RESUME_SIGNAL" + +func TestFirecrackerVMGenIDResumeSignal(t *testing.T) { + if strings.TrimSpace(os.Getenv(vmgenidResumeSignalEnv)) != "1" { + t.Skipf("set %s=1 to run Firecracker VMGenID resume signal probe", vmgenidResumeSignalEnv) + } + requireFirecrackerIntegrationPrereqs(t) + + mgr, tmpDir := setupTestManagerForFirecrackerNoNetwork(t) + ctx := context.Background() + p := paths.New(tmpDir) + + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + imageName := integrationTestImageRef(t, "docker.io/library/alpine:latest") + snapshottest.EnsureImageReady(t, ctx, p, imageManager, imageName) + + systemManager := system.NewManager(p) + require.NoError(t, systemManager.EnsureSystemFiles(ctx)) + + source, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ + Name: "fc-vmgenid-src", + Image: imageName, + Size: 1024 * 1024 * 1024, + OverlaySize: 1024 * 1024 * 1024, + Vcpus: 1, + Hypervisor: hypervisor.TypeFirecracker, + Cmd: []string{"sleep", "infinity"}, + }) + require.NoError(t, err) + sourceID := source.Id + t.Cleanup(func() { _ = mgr.DeleteInstance(context.Background(), sourceID) }) + + source, err = waitForInstanceState(ctx, mgr, sourceID, StateRunning, integrationTestTimeout(45*time.Second)) + require.NoError(t, err) + require.NoError(t, waitForExecAgent(ctx, mgr, sourceID, 45*time.Second)) + + sourceOut, _, err := execCommandWithRetry(ctx, source, 20*time.Second, "sh", "-c", vmgenidProbeCommand()) + require.NoError(t, err) + t.Logf("source vmgenid probe:\n%s", sourceOut) + require.Contains(t, sourceOut, "CONFIG_VMGENID=y") + + snapshot, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStandby, + Name: "fc-vmgenid-snap", + }) + require.NoError(t, err) + t.Cleanup(func() { _ = mgr.DeleteSnapshot(context.Background(), snapshot.Id) }) + + fork, err := mgr.ForkSnapshot(ctx, snapshot.Id, ForkSnapshotRequest{ + Name: "fc-vmgenid-fork", + TargetState: StateRunning, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = mgr.DeleteInstance(context.Background(), fork.Id) }) + require.Equal(t, StateRunning, fork.State) + require.NoError(t, waitForExecAgent(ctx, mgr, fork.Id, 45*time.Second)) + + forkOut, _, err := execCommandWithRetry(ctx, fork, 20*time.Second, "sh", "-c", vmgenidProbeCommand()) + require.NoError(t, err) + t.Logf("fork vmgenid probe:\n%s", forkOut) + require.Contains(t, forkOut, "CONFIG_VMGENID=y") + require.Contains(t, forkOut, "crng reseeded due to virtual machine fork") +} + +func vmgenidProbeCommand() string { + return strings.Join([]string{ + "uname -r", + "zcat /proc/config.gz | grep -E '^CONFIG_(VIRT_DRIVERS|VMGENID)='", + "ls -ld /sys/devices/platform/FCVMGID:00 2>/dev/null || true", + "dmesg | grep -E 'crng reseeded due to virtual machine fork|vmgenid|VMGenID' || true", + }, "; ") +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 1c053f62..add4df50 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -732,6 +732,10 @@ type ForkInstanceRequest struct { // TargetState Target state for the forked instance after fork completes TargetState *ForkTargetState `json:"target_state,omitempty"` + + // WaitForNetwork When the fork result is Running, wait for guest networking to be applied before returning. + // Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. + WaitForNetwork *bool `json:"wait_for_network,omitempty"` } // ForkSnapshotRequest defines model for ForkSnapshotRequest. @@ -745,6 +749,10 @@ type ForkSnapshotRequest struct { // TargetState Target state when restoring or forking from a snapshot TargetState *SnapshotTargetState `json:"target_state,omitempty"` + + // WaitForNetwork When the fork result is Running, wait for guest networking to be applied before returning. + // Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. + WaitForNetwork *bool `json:"wait_for_network,omitempty"` } // ForkSnapshotRequestTargetHypervisor Optional hypervisor override. Allowed only when forking from a Stopped snapshot. @@ -15953,213 +15961,215 @@ var swaggerSpec = []string{ "oWvlHKxmPrBPTXxg59CSG37TplMvIgsSFZnAHF0mp1cQlPGJWbTSrChb4IiGY8qS1MsSjaR8lQrQRU2j", "CE94qsw1o00NzzuBeGuwPaZaXLezc19xcbk2clWfxFkG/Fqv0CE40qfWVQOnOEb2a5ccUVD6sutAc2lq", "n0v0znxhPET5z0laxtPpQk/Wk8SQIFJxkKTWYWibaatd+vUWcJa66BHTXy47Hyl0pjc10Qb3a2GLGQHk", - "B7VWY9Gc8h7eP4fXWwfC6w/XOlJa0J2Rq8cgOmQK9DTb9iTDycNQfFUsW+ZryF+CU1jQkPQR7C4IqnGZ", - "iZWddq54kpAw8//0R8xGkmc/SXODoj80dFBzQgXigs5oueOyg+0hg+JuwoqOm27NjsUP6xoqPITwjeZN", - "j6fKoDxcumQtUsycsovQ6XbOM0wMK4nKpHmX4YrUKJIHedaG+Prsw01D2xLBp9SHdASxEPaptcxc0Ndv", - "u4Pz3vD/MQGcmt9ARaPMxE/EPKxAWNj32508r88+nDWNKQOVQMXR1eaURbysgtVyFLGXSvZW0lowjv31", - "wZJ1kuveL3y67FTgmEzS6ZSIcexxrr3Sz5F5wYQ2UYZOfy7rs1pvbms1n5UWB8zmKQ4sJkA76nsccpVp", - "dAvU/ORfrnfEHMNNmYR6qYR9xyYT9tGbDMYDvT77IFEepeTx1JWXtzFS/2y+lDTAkWnRJAZTVnSwAXO2", - "1pDP8g+tK9KjJ/vRX9xGQBuLWZLCNjx/1zt5+3ErDsmiWxoTRBbNeUT0uDcL0mLh8gnztIKSkFg0eToM", - "Y8i2G6hAq2wHtyZSYb96qKO4wtFYRtwXrPFeP0TwEG18fGXyvfQIuigpLaX+vUCFEn/ve3eMlkhN3Z5D", - "h1WXaWmDe23HMg6nca8Uplfq1LdVTJB9XcepQy/xy/JC88v1cD+mkeZ+j1weQMWpbRVJZNIFEKQLuPtn", - "ZD41XmvrGpEkwQIrEi2NZpEdfRGdkmAZRGaPk/pVIrkmwQ0SEV7q17+aPOJUkLGaCyLnPCpfQO9067Bv", - "EoIgF8TCb5g5FRzviqMYi0s4GJ0ijVJmKFBOOthZB2k5Vyq5waR+ef/+zFjXioiFiSgrBvnK2tXqMYnw", - "Ek2IuiKEualgiTB6zTOck2pWjmyAQBBqnBBBeZmGnR1Pv+cmMBPNBA4IMl85FEa7JBJOzbaktL14oLyC", - "gEjZsL7DVetrP52mUbs19g1ruBa0NLjJAr8/OnOp/BkqoSPzdp3KZ0T0zJZz8ISrl3ZbrgaVcF0xzki9", - "M8EnBFAmbOJNMUXIBZYAiob+vJSWUxAOspgPY/uBXWBI1TX7/FMrZa+63X0X6TFmoQ/d0UQ8m9zHWRrr", - "BdFDFincHdHQRJ2YdEyjlhcDtwXBIWVEykriWJCKqNPt9KZ2VgdbWxEPcDTnUh3s7gyfb62O31sZuGnj", - "VMYhXWXfuWgWE+/g8qsMyiJMuswSWzhJWnjADB3XnA8gnuqBYoDoqM+2gobnIocHg1oO3DUOlAPGAZdY", - "6coTF7dtolmyNB9oMIMV3nvxorg/B9476BzV27H/Vo339cxMMJnmEeNuqNDRMPlnr0bFhQ/9hAtlM1wm", - "xEWbZeehi+Wylyqlzp4Pnhdn2Qo8GYRNZZvbfeeZqnm7nPVGCuS2+9c2AHExZZ3DbenVWGmaLmt4SkvE", - "GkutJihPCLsRPfd2d7ZvRs+2EzlxSXkVueTLuT86PTY6UcCZwpQRgWKisEWKLwgZ8CVpKaMN/BCTGPIU", - "pv+xWrQ0xC8Uk+gbb8CPanBrD3L73QAT9M7Eb4YoxoxOtUC2bxZ7lnO8vbd/YMDMQjLd3dvv9/s3TS1+", - "mecSt1qKLZM+Wcgy7sv53dbhATKI28zlS+fs8P0vWpClUphDa0tOKDso/Dv7Z/4A/jD/nFDmzTxuhX9H", - "pzXcu3I8mDb4ze8HBahyp/e0giP2O4MhLBTgDrwwMQrPtG5jOO6ueDC3RozLYUtVASmumE3TAjWOfl59", - "jey8SvCO7TNlikY5oF79AvlWkIhyJWpUDTEqISzDiYoi81fA2ULvCh9oVOkkcs/uFHyxUvX6e13jWr/f", - "nOK1hm39XrZM/rUFy7OQNp6T6JtL/dsEKJV7fzv7zz/+X3n27B/DP377+PG/Fq//8/gN/a+P0dnbO2Wv", - "r0Yz+qaQRPeGQgRROSUoorasdIpV4PFGaUOngcL2ibGtVTDvoyPwmh+MWA/9RhURODpAo04lv2rUQRsE", - "bAL4Sit2uimbJrqpPz4zd2f64y9O4ftabSO0+aDCLkiWRS7TSchjTNnmiI2YbQu5iUjQgPVfIQpwosBx", - "QRnSlt4STQRUY7F3G3nnXfQFJ8nXzRGD6wFyrYSeQYKFyuDXXA/AFHZUJuDSvk5Ch+djrhdGLDuXMjgf", - "c8HVz9RcCGyopqv4ibLaUrE2wvOBD/gIQub1QkZUKqNsZ5yt2SiL5UfPB5t1y2WNNp3x0Ar2g51QL9Pk", - "mLLFXjIMDF0bwT12zrg1gQhaNpk9gsBWUhz+e45cQzktsiU2HnKTQCHNBauKZCF1YrPjBYmH1W05IXPD", - "CJ9FLVKuX5rcmve/nSNFROyyHTcCTc4pDfT8IHaSSplqVqQYHR6dvtzst6gzBbTNxr9iHd9nM6xmxtob", - "x6aL1NywwzHpopNjyG2yOzRX4CAm+RUXKDICJt/XB+iDJBUbEbLXICTSrGS0zK8tzQkw6my6FpOqpDhA", - "7zK9EWdDKdW2Kt+E5vsSmrVRKyZgutZ6t1Y1Rji7yIo2CI/GKsuw0ydusyho76iwFIc9XzGrb7y3izfJ", - "jUZzYe3vG+zu/tWdnZupO65wQTLH0sfd8+JVCLy0oloQrfh3RfO9f6nftQV7su6gBpE++oqf+0qt7PWG", - "w/fD3Zvb/DfFKSsDehSwZDKosvYYYw+B1VW3f6+pGjdGhCL92MZ/Oivv4ymaY8n+ouBhxdYb7jxrhTqv", - "e20bS1mMouRTM6RMSjl0kCwG0OCkXNIoMqG1ks4YjtALtHF+8vrXk99+20Q99PbtaXUpVn3hW58WkGVO", - "VLw++wDXaViOXThScwYOzrPYyDWVStbBU1pF9d0FIs182g5W3E3StJHji6/GWfulhIXmRcPZvEeANBeK", - "WSPjY0Cffcscl+8Pdm0lUNpd0c6s8fJAYGeNwt0HFFaW8+bn+4Ute5DhrC3SVzzrXQLirXHCuh3qSb46", - "lFoEkxCdnOVQ5bmT0TVfmZOteDkcDPrDQRuXa4yDFX2fHh6173ywbVSLAzw5CMIDMr2Dy9cytlHGcXSF", - "lxKNnLk06hj7rGCYFbatNalaXU/X4dhuh75WVWj8ctoodu7eX9ryNk3KTYvsmirYS5xGJoGzWBKnXuVT", - "JgbZzGACZ7rsiMEAuxZhJavMiYNApLk/w5VcM5pvmli+HzFBZMKZNIUY++hXspQopnCHkHUPkUMSZUHc", - "4YhtCBfwn0X2JziVJNQ/QDRt10Vt6qFRBdjH+oMRk/MUKsht9tERZzKNibCuHjSh4IfeRDI1xh2MF6gB", - "9UwlDYkYMf2aBzvtS6aoH+wPBoNBVtmuc7Cj/z3wcdMd8fPc5+1UDvt5rmv4eXgd3F47HL27gpetKodz", - "Xi6E09qiu0O1yFbB5k6ns2Hm9qvxTe7KCAp4GoXaTJjog8F4cUhonU2SqLzGEJwlH9gl0+xcmroNclMc", - "/ZESsUQfT09LF2yCTG0JlRYThw3VsA48udEybK8xrNeO5paQdo8BY1c91AvK1L2D1hU9/C7dzXBoC09/", - "blx5Q5QpM0uj+WTFnCo+2pAsxmnq09n1I5fk/uHDyXGJOTDeHz4fPH/Rez4Z7vd2w8Gwh4c7+73tPTyY", - "7gTPdhqKmLVPUbh91oHXQvNVpXKxh2MXA+ktPNwQgVo5Im1U3RVlIb9qVbI7692GUK3rvh4g2XoI3rBq", - "qHQMLTVIidNCPWMTVlipWdTgedp/Pxiu8Ty1K8PcIH/fi5QFBn0KJHHm0y0WYC4uVr0I683FKQzIhS+v", - "o1ax8/ZEGxzsvTjYuyvRXAjuujFW2ekRF7cp4MBBGFZifF2eScF/UahgDPqGcbPakOBOt5NFLcPfcNBW", - "IuKyx61C8Zs2bNcvRlbJ74aMtJOS2gy3sAbJKDzQWkCWzDRJFcoSHbV6cRTxNEQF348BdoGLkZOCCq2b", - "gXsK6xoyCF0mpFar2gAJCYDElGlBDBdCuhGbvnaAXsO78AjHxrqwg8AsrNyF4HBp7oL1/nJdG11/9ZDP", - "rZoP32idH0FhcT1tTQbrIlzdhNF8DtAbDt9kRgfjVV+jeR20/frrVb/khs0rc/nH0JlV4w7Qq0x1y5Q/", - "q+xtSGL/HFuBlaf9b5aSL+2KdzS35CtXyCvsdgxFO92OIxTkH9YzET/kXF/bf0VW9AVJEBzBXs4zvVJF", - "I4tyCTOhUFrbhgDrxW3SLywgPAnHxjBpCnky6UPWeMk+curLx1O0AXhWf0PWkar/tZmFR5XOuu0Xuy/2", - "n22/2G+FWpEPcL3aeQTJbfXBrdVBgyQdu6q1DVM/OvtgTPDAGLfgmbdzLyQJJ4Jr0aNnnpfBzTt/0X9R", - "BOsIeWpKgtshWWSfr4XC9ytrFjfE+PxBowWdTtkfn4PL7X8IGg+v9+X2xOvQzCvse70/J8Xb3pqrlEx6", - "poiHH08BGErIRsiRd0TCDNA5UQj4p4dwAKZDlpNmWc4Bk1iKexlrd2dn5/mzve1WfGVHV9g4Y/AFeQ5l", - "O4LCFoM30ca783O0VWA406ZL1AWAUGbNSv8+Q7YU2KCskPaHgx0flzQc3DnX2LYXcSPJP1rTzE7KEh1S", - "6zKzrbbLvdTe2Rk82917vtduG1sv5Vhcr5YwLvDckMfi2BZXfgO0yfeHZwjSuqY4KPtNhts7u3v7z57f", - "aFTqRqMCDGaDnXqDgT1/tr+3u7M9bIed44sCsKhQpQ1bll2eTedhCs9qeEhRF73dptPCp04ZBntHggjT", - "+DBwEbyV08dgpI6FeS1fhDYHg3WM1w6uFt+2chxVSk4b1YALlLIMmbu//grwdjd6zWLanAfrxXjdso8w", - "0+SyIA+mFMctaJcIsqA8lffQEFcm1WkacS5u9G2ThfKOyDRS5tqNSvTx9C8gRDRzIalIUjaaLPutgMK4", - "5eRutIFLPOHn6iZitVqNNku/asLdhm3aXZUHXdr+jYgzoRZVKVsffneEoyAF8HmcraeeFWBHQCZnkkRL", - "E6gaRZwzFMwxmxEoI2hKXbAZwmjOo7DvDR7UT8ZT77U9v0IRN1iZl4QkFpfdDEJ/pnUWuiBoo5BJigwr", - "Vcoz7cVGqljk7TI37sX+QkBY+pIfsgxGTU+seAHG0XxS8jFGfCbBClQQgtuvogcnWJjIWsxMnYFFbIzH", - "MvTOtj7tPUOsSG/fEWqOTj61Fq3VMSA/0FASB4JLiUhEZ4Bp//G0kna2IociSz5bH1JXHmwL1jUXaZ6z", - "C8402bocie9A9ASn3+VIBB6GHJQVwWrOGxljlgJSe4GRyXVChWGPdgFpcy7VOIMTueFgpRoDCncqSI45", - "lCVLZg4g9473XHSi7TbkspGft/q6xlX+ppoG2CxTvRT1U6ub8aCPjeuAKisxXHJQmCoCyE0gf3LYZiqh", - "VVpAm0EbjKuSWCpAD2+2Cc7w26i6n5p5aqtM/bY7OG+LxrMafOcMq/kJm3JPyvYNriGd69nGCyZExBRw", - "6FFIGCWhMx6z+0jr24I8v0gSFKbEUs4opAJbgmOzvSHtmjmnGGWziqyvdtjGH2zGsBqkG/q1L7YJs5H+", - "7LD3IgVamcA4iXCeJ9YqypDKsf/+qt6wILM0wgJVEadWDFku44iyyzaty2U84RENkP6gesk85VHEr8b6", - "kfwJ5rLZanb6g3Ge5lC5NDaDs0kuZkEq/eZT+EnPcrOSYgeuly3z/RYk+reJWvLG6r6iEbGgTB8YvS4w", - "ehnFdnd70JR92dBoKe+yDuh1U8ltWda34x3W1mFWNNNzS2mibit3pWVH5NqbPgjrXpVrWnfFoA0XCOVQ", - "gst0LaD1tvKEtIssr4b8udFsSRKUe999vvdsvyVc8p18nSaF/b49m4t4hUezYaVO27jNnu89f/FiZ3fv", - "xfaNHFQuOrRhfZoiRIvrU6mNW3Ga7UEk1eBGgzLxof4hNcSIlgdUqnN76wF9XbF1m4IL8r3ZdMkZFVfS", - "3bOUPaDtfIwrtKXDkspVqAK/QaZTAkbl2NCtlw+mkiHYagwBTnBA1dLjMMFXpmRf9koF7q2NN608WA9J", - "bdsWGkhLLplO8iSKDdc5+qtxrVd44Xlr1HWZTprc+G+rvRonfu4DKl4RtbihyQtD1t0F2XyusCzFmum/", - "A4iezKv8V2NmzRuroaeqgB9wCWiLCxQiKXyQhZXzz35UXP7KchbcviUluUrxVUdo8xa8kQ3tOZE9JnSw", - "PhOmIh/sAXi7r8aTYj2ElQUnSsUT8lP35v22SPapg4VmJ9jN+yukPdzkwyowFvCjHYMled52t8QSDdxU", - "CNP1mCM8Ir0szsHG8CKZGv+q3vMWa9GThxFc8um0DPi01wwQCMh8ELLtesFKkThRXUSuwUwnYQ1dztXR", - "3pOjDuICjTrDeNSpOAG9SRAxvh7bDsq5yoNViH1ZJaDqIKWbwSTiwaWB+leCEtlHAxQTzCRKGWz+io9y", - "OFjta+t2ksLaZPh4xNwQ18QWjGlC5nhBAZbVeqhmpTgWck2VhHgbaOcAhRzs23KdIztD/ZpJUTjIJw2H", - "DmZL27BuUL/HmQsIyt8Fc2kK1ZXYZyJ41ybfaYn99u1p19z/QOSGGVgpPMRN1IxAC8isiwrGaP67P/xq", - "EpExjLuKWRnX6VhMJQM/tSCSKGlB7HJ2qDABCnjKVBXMMm4XwlmOeK8fSSmDWAl7ewaQDbZ3Wx0xJIGt", - "QFt3L5UY/RbMXQm7tJT2xV3u+FgYNgV45vye93fWvV4dgERYkGKJMtNOMSzO+FzHUnGLaZ/t6jG5DggJ", - "q3A8/lfahhraL72hhr9hm/OeVQ+zb0O4WH12/YeLPYexNlG7GBLJOOtBtq1bUpsZa1BDbO51mdFK+HuF", - "DNSxD/zI90KbvClyvZrWb8i1ApDAMI00eZtY14oqexito/itky6aNjQX64ttPkDtBBOud6vqCTbS71EK", - "KNifH6RoQm05zoly755bvmmulVmCNy55BF2ApHulfEVpeKeL7ImOhvFmhed25343iAXoapkvwnBMxokg", - "U3q9glvMC8YSLmcj5zunVE9Voo0YX6PdZyiYYyErY2d0NlfRsnx/uesBA7hTKRFBFGHqBpVn89V0H9aD", - "BexyFlv3acPnhdR9f0VcEo5XwdgdZa+569gEL8Ft0+hjfbazOxjsbA9uhWN3X4V6C+00pSAUvrP3JKWo", - "nmILWcJXvZrTlaCQN5aRSSpBcHwAgcoJDgiKyBRQXrIqemsPi1rXqwdvNSibR57xv1sou27uCqMMFJ11", - "ZSEA3TQ6LgCqDB1QfF4f9goomEzMBDVMGE+Owk5vsP9+uHOwt38wHD4E8F1GpKbo2Gefh1fPom083Y2e", - "L5/9MZw/m23HO17D6wFqQpdzPqslou0cEiKqZbqq5e0kiSgjPZlFlK9P61ghC0yQxtr9fzPHvpnBSmXh", - "vDzJos6AVU6cEmc9Ek6GHf3K24nq8E+OVw/7ViHa1YH4Gaw6FOCndoMBINbhXVE/U9by3PlQeLH1ybMy", - "bWDd2ePL8oSt7V3lBor7+LkkGEs7bNWJXT/VPN7RGRdUzePVx0P2WoYhCHFmn6UKy7gMfXQyY1CUr/hz", - "FlZQNJP0x51uJ/q8W94z9vf2CB0WMy9jQLvURTWgxbU71HxcTQV4JTcthIn809a4HvNPw97wBQS/RZ93", - "fxr0XvTR3wtBeF1DrSL5hu7t0q+DNjQsVrpwCOnDFzeKUHP0XMVBv1JfnYb8ILZoepbH84po7qxwGUml", - "Bc4f19a4AiPQqHHeVbWzp9m4qCWFJMJLXx3rgitWVuzDIpOhCZlRJtt4ZncGmWt2Lx51+ujQYlCCtZoX", - "vCw1D6UOC3xC45iEVCuVxrhvjvjcbultqxoPNwMmdl951LO+Xz97sT6HdF2A+rpjsn+HhKU7mbvtTNxV", - "6c3gOXM2KZRAgRe7iE4RZpU6PLbqrM00hMwRQJg5cIAqOctaGSBzxc95QrpoxhXKcwxbetRS1uz5y8ZP", - "rsGjuiKp2DDE9r1kjGfoJXSV+Do5RongYRrkCTYRDDpPiRZpBcxxhVa/PoTpIR0akLk25QKtd2g0eTDa", - "eSCb1rvifdQM27zUw8H6pX4QL0i3kybhehlmXmonwW6ENbomZcPjkymTvaIJFibzqYVEf1ekYN3INd7i", - "QKtEaeKuUDRP1TnJc6EClwg+2L1jEhF9TNUbQTwK86hSKnMpul6kDvefz5suMeHOqT6QXwlJtK0CABHQ", - "X4zZ0juwapFVtDFwlbOkudLqGaxyS63y4J6t1cQal6p9vdqKV9sklBfLA2egm/dbrNZ+ubaU+EP44b6l", - "kvbWXi5U4NUc+F8GV+r6NxkzAM/Iosp5veu7gPexxXtrGTdBxlWzZopu5sPefxu3Mhr3D7Z++tv/3fv0", - "V697uWI3SyJ6IZlCKNElWfYA7x5pG71fBkwDrF6tTNva/orgGJxGwSUxTqoYXxfHuzfIhMbyDY5rU4AY", - "rJiy7N9rJ/S3f2uOYCqQ8QPIybUse2co64eoE6S4O442YiJmrpKxC7zf7I8YxPJfkqVEhWIEVqVxjPoX", - "mX2iVXRwWuIIXRg1sE/Y4gJNKNR0kSOmrVocBCTR1oQFZaemGB8H6SMIjort2KIILlHOXjmaiAGCPp7W", - "0Pbefnj/89sPb47Hb89evjk8Gf/68r8giOOqZ3oIe5r3dvf2bQm+IiWHniW+A/DvnVD8fOxmsMA8/AVJ", - "KVDX0KMwUwkppS6goPAy2iBxopau2JDLbdm8GTbZYdagN5ztnlHYBy/uo+jMh5VVZhY86mmNugFE1+vA", - "NLTwhmNDUybMvdPk2J55yoCfW2/ijM6wx5ftrUB6H8Vh3IDWgsbV1r+xtIM/OP64ijBspIEhVQURt2KX", - "StVrjp2PtSI1zms9liMyUmbTS2ghYKucSxIztWWLOPlSWkN98q5OKMp3mUMs6sFH6/NkVqryhZkVRtK8", - "NqdOY63o1CsIdKZJczUnghQWAj7IkVtvSDKb7NEiUdqUWkmIyAMhXaYI1D4XFLJHMmeDI0GWEFT3wK5G", - "5j3F11kP4L3HsnbHBfPIMfKHr38edTb76J0rVUqnrgkYRsWe8AOhlrloFU0cV9UXo8hV9Xmb970bz8qq", - "FdKvaW9VmDPvo8SaPn78O6bqFRdggTSnJT84nipYNyERgMtSRUttBTVKYxKOs4LNTfvf1Wg2OclZOey8", - "jJOztjAwsRZy60vtuMTZfAx1SmtykCAVVC3PobCriRAmWBBxmJoN7+rD2p/zjqEq0tev4KecerIQXhNG", - "BA3Q4dkJ7McYM1DS0cfTQiUTU9SmhqEG6uXboxNr4ToYPrBYqALWc8F8h2cnnW5nQYSx8jqD/nZ/AJs5", - "IQwntHPQ2ekP+4OOqegLU9yCIorwp00wzCylk9DqQT+bV/RXAsdEESE7B797EvUgkA1eBn0XzwoWS4Kp", - "sCZLEkH6oGEVqr8FYF13lB6Y89gW5G3toJNqaZMpSPLWLusnUCdh18AUtwcDCzOq7MELqSAm/nzrHzYY", - "Me+3lT4H5PGgzNYsCqdTWpJ/7XZ2B8MbjWfVMGDH+rr9wHCq5lzQzwSGuXdDItyq0xNmMryQwQqzgTbF", - "fQYsVNxhv3/S6yXTOMZi6ciV0yrhskkZJhJhxMiVrQj6Dz7pI3v9AGVj5JynkZYmyKSvOUeDwqI/+4yw", - "COZ0QUbMntNxGimaYAFuhBjp89kYTOWtYbo2q58BC/zMw2WFullzW7q5nnM65wSupiVIMgYc4nFTud/c", - "3UwZ02ISS2JraWR1L+vRPFpcjmXAfflE7wnDTPVkQgI6pQGCl/XutR5tb4OtoPm0wINlIQIQs5yHZnvT", - "n5MKVT/86dzH2TNkyVtWJxhcAwVRGuY6l0uTwmKCo8iL3TSL+ARHY0OfS+JRUV/DG5YoxQIpTrlhPCSm", - "2EWyVHPOzN/pJGUqNX9PBL+SRGgVyBYxs7QmoSlbZlj3CrA+YygkZkqk6j63zBC3vlyS5df+iB2GsSt/", - "K80nOJJcn5q26KDNCzBb2vCuvyxLQ1zJUSoVjy1LZRUY82HyVCWpsnfqkihbeQ1epxIlqZyTcMQUR18E", - "mVGpxPLr1pe8x69guxAcaj4pvGKmtPWFhl+bRi3HWM9+DK96rD8CBBh19Oky6ui/ZwJr2yWVc3CiSHCc", - "zIpLupHh6Wi9cLNK4QAzlPDEYBEBU82xZrlSGxCPjqMIKdhK7lutbcJKNszHphfHk8bcYpMMWtlGlKHT", - "nwubabD73L+fJAkE8Tk4/vP87RsER5VeA/Na7rAyl9pMn6IoTEGTh977I/YSB3Nk9CbAmx11aDjqZNZF", - "uAljTaWN0O71QMX9SQ/tJ9NNl4Y/9fu6KaM9H6Dfv5hWDvReSuKx4peEjTpfu6jwYEbVPJ1kzz75CdqU", - "onleEgRow8j+TVeDGKCi8mPQnBuYhYhbWRstEUa5BCr6USaUYbGygLKH9JaC2pTHM1kkxpcR+G5HnYOR", - "896OOt1Rh7AF/GZdvKPOVz8FrBLdDG5qakg7XTtjov3BYHM9doKlr0eFLr2ot9/Xmva1fW+Kh1W66oqH", - "mZxDZtYraKqBG3XrETSfn3Ho6kv+UPHWqHjWc1FQ3uD74jlg2DcixsCtaGDano2cBrbSOjFsAQUTwOJw", - "SCfG4KBOg8uZt2h+VM35ulmx27TLAhhi5Phv9xH4D/rNShGafl88Vr84ApxxB1z/xNgRFssxYtdvEb8m", - "6nvguMFjiVILiv4t+fep8M9rYvW+nGgVabZFFu6+yY/nBMkm0rZiXta26jmMqXdOmEIv4de+/a+zeCBb", - "+iLis4sDZEgY8RmKKLP3gIXbIn0oWlrCRybfJPvOpp84MM0Nc37+83/+FwZF2eyf//O/Wps2f8F23zKJ", - "kwC+fzEnWKgJweriAP1KSNLDEV0QNxmAxyYLIpZoZwBqZiLgUbGkktVN5IiN2DuiUsEK96UG11LaBsH0", - "YDAfylIibb6OfpFOLeiWcTB7THi3lw0pH3VHdz2JzjCDwgT0qeh4ALJEbfk1a391/N4zM+eS/6zqK695", - "TNfLF0WuleHenhngDQUMkNi37+CBnTTaOD9/udlHYGMYrgBgNdCY82as8tz/IZPWyyQjUcoCBahsZJNJ", - "Ylvt/z2277RzANsW/0weYIu1eQMXsHF5QO66W4EftkILd7Cfbs417PPPHrsszWYH7e3nW+zCxTG1MoTv", - "b50d79Vpbp4USPYtTGC04aLhwY3IBTo7OnE1bTe/GdM/yqmhZ2pr9WVHB+IMsNcezSw74mwa0UChnhsL", - "FGaKSWaqlRnkqYiDd3bUCLt5VSGMi+fbVgmRr/Gky8D58iPv4U+PSqc3OUZymOWc136cJOtY55jKgOtv", - "C9zSC3AChHTqS7ZPi1y0ziFlguuzI2elumTF88mx25CP55qyXaesejY8glA8rgjEbygIK8XjC8DkT4mb", - "P2Sr6BApVniuvi/WHDyeFvTYXiwfmz8lN1ZYIZuWgiaou/EAfU2UCeXuPOBC2x48Ez8nwu1qV0cCZp1N", - "y3xqCquaCcGF9Grb98S80s70Ne39mSxfIM9NNBZL8h8qSgtjN6fVKgP3xJYsfzj7Fnq4kXl7f/e8lsE8", - "RIZgk4nzWJu6u1guWbD5p7rqfZTTzBD7SR5mZ2kUuRuPBREKvT06MTureAZsfYGwpPW6vdttK4+DD+9+", - "6xEWcIhDy2Ko/EqUfXLPGr5ZMDOVH2zSxiY0adHUnWdNGs4d1t+ECyIT4din/N+3X0V0IrBY/vv2Kxwl", - "lJF/3zmMsCJSbT4YswweSzQ/tsb9hJlPK9y0TDQQTQzqva/TULO3Wiqp7v0/lZ5qJn0jTTWj6w9ltY2y", - "WiTXSn3VLsWDaqymj290JZMxm4/a8MjFJ/7JNNXH9fJZjnRA0VSWrz1skT0uwM8LjyhDqSRPMICSZhxX", - "PDZauqvzDbny+HCse3LcBUJCXQRAbbIJIo/kvHbjeHTl1vb7+J7rw3hCZylPZTH3JMYqmBNpk5UiUhbA", - "T03tzo/nRsX7O+bSwWMeHY+uV//g+wfS+KsLaoS3uYFap/O7t9rq/PZ9rfObFGqbu2ahpboOdnCzIajQ", - "JVG3ZeNSrnk92NE3Lp8tgj5oQyU3FxBYEAcj9n+0/fG7Ijj+9JNLkkkHg+19+J2wxaefXJ4MO3WsQhhU", - "PAKU2MM3x3DtN4PscwCSzVPyquMwlSeA9Rx0zr+cgZTffLa3kBwX/rCQWllIBXKttpDsWjysiVSG33p0", - "G8nxm4/gFsTkh5X0GFaSTKdTGlDCVF7ztBYkZksmP8HcMmbvhwrBHaWDtrWVlG3KNQpoXhbg0QN7TnIc", - "xMc2jlwFgqcZI88TC+ltzZH8MGy2R743fhg8rnB+fDvkKbOYUfjrpEu0TukrpQ0Yk3EKZSFRjhACUZ9I", - "2PqPrsU+yitYyzRJuFDS4FSCAmyQ7OdaAfZhWpZhKn24lIDFSInsjhhUKtCPTS7/1iVZGhRKylkGOFmt", - "x+rLvSqjgH7TbXT/OpYf4rSVjvXI29iCVn87HeubiY5H0bROSrUANrKNAQblhGQ7mWfJffQzZbPNJxWB", - "aoRVNrcCnpFH1dqCSn8W13dLZtVkmw7aArSvLT37L3ji1ifp09odFm2BgCikeMa4VDQoVt4twoP+OKFb", - "n9CrKevl5qktke436F9xcdn2iPPUFXsCJ11xht+hL0EPD9DAvr1LAYxtcxpopnn0U7BWLO5bpmDQ6rkY", - "RGkIVentgehUyang8dj+aPBq9a6waKDgoghsq99a2OjeH8Fh9IYrROMkIlqLJyHqGW7Sq2lVfwc3T2Wh", - "tOLNhKHeNsWEGANGJ11pIisi4XLNLdgG3LPXl8srNSM+Ww+CkXXuEB88KBgjZuDwicPOv0CZkIXiXSQi", - "gUJXcxrMAREDCnpBRVcAq8BJcpFBYG0eoNewU4tIYND5hiRCG0IBZ5JHxABdLOL44qCO2Prx9BQ+MmAY", - "Bpv14iAruZ4dEFK/VUS4yGoevbG4HRuakwSPIrOiF9pqLMxv02Jf5BBlI+bDwWDkyjZIp+iiAIlx0YCJ", - "4QTqb3z2zbStbjOwpJmL4kgA4QxvEhZ2mi5iaORHwxgOvPVgWiJzmGE8MDBHbTC/8VkGalliZZwkbdnX", - "DhO4eBHHK3gYbRSKs0oV8lT9TaqQCAEfW+5uYm60gQPzD4UvNaPawkJZeVtgP+91o0GZ85JKC9VCFR3z", - "r0Ucd7odO54COt0NtPc1CCfVBuvXYnplCjAmP/TumwCUlIV9AaGkcnLY8v/NKvc788Kf3j9rCRX+Gbws", - "5fusfBSU5QWgBFRwd1W4nhTSASxkTRczhZF8e8TNsicLxUPbXW/Vyo5+B0bruluvrIZkVuDysa+/6iN4", - "ykkwsjabKRfV9Ph192LfPSPd35LUptqGQ37w5s3dc60YM0lX1BKFUqgS/HxQXxNwnYM557LA9hMyxwvK", - "hUVgt17XjDPBZWGsRxs9d6FZ9cL6by+sen5gfU0IFx/ZPvrwuY2583/hHuVfvCpY25nE7zqVGlAgJcJo", - "IiiZogSnkmhtKY0JMhVGLJA3wcHcVQvvj9j7OUG2PmbBgZCVU6YSXQzjiy6apApFWMzA2jEPTSSdIAGP", - "Y8JCU/N2xOYEL6g21QSKsCIsWPYkgRrIC5IXMNGmu72hNKW2syqrXeSK84KD4aJQevcCJYIAExlzmZXq", - "3I6YSNl/GORK3eyFG+gFIlLhSUTlPKsVEeCQsMALC3n+fYux+3finhNVr077Te4sbyVLv+UlZtGXmdUH", - "/y7uN59YoBYXrrJmCzG/QumVzaZhOfLxPK/I+y+4pc1c3Ry/0c1MRuJVu/j7uJIpleT/cS2j7JYMU9Md", - "KZet/9PeteR1pFNWum6xPtnbXrhklRAyMt9I5m19cX+e3MJH9p1Iwm6jYd+EuZ1P+nsQuZaqt5K538g5", - "aH1JBa/YNxTBdlDfTn3ioiDlvgsxbDZcJo2LMkcJDDYVZz+EcVUY2/CA2wpj53GtXYAXxDNlvSTCTXI5", - "r1rvF8DWIfAvGv1amV1BEH5zwZffCDyasDvJxJsReAleRhz/2e9lAi6ESei05YifDqBYwRdYuGDaAI9b", - "N5MQXZdN8vH0dLNJSgi1UkYI9YQlRLmsaRB7qjW+XRAhaOhKRx6dHtvoVSqRSFkfvY0p1HO8JCSBQjGU", - "pxJBZm5fz8+lttaL4JVyWLsdwpRYJpwytXYU+asPM5ivtyqd98hy0kIq/ukvj8EL//SEFMgOra7YCay2", - "IhVWjcF4LjiNMlPvUmtbeMJT3bqWLK7Q7gzOtimNiFxKRWITmTdNI9hEALprazLZ70xGaRdRJZHeD13I", - "wEuIiKmUlDM5Yrb8e0KE7lt/DsV/8yAjr/Ne4UxqnhnR930EsOnBmJgtrJqoBtACUAe0c9DZwkmyBeWi", - "/UFSdnh3GNIriEhDchlPeEQDFFF2KdFGRC+N0YEWEkX6j82VIW1j+O6+K07dfmdpSp+wKfcW5TA8mzHz", - "nyMJqSzW3CXikxNrr0lxszj5AwvtF2tyrVwTBEc9RWOSJb+jVNGIfjaiTjdCpaKByavJUy+hCLPNvhyx", - "U6KEfgcLggIeRSRQzrmylQgebI3SwWAnSCiglOwQGBwIvObHMfR4dPYB3jOForsjpv8BDb8/PDM3sVNs", - "fQSFgTKirri4RCdbb9cE+Z4Dmf6Fo+TMBFfmQHoX/Mf13c0zmxv3kGzYojxZZQDx5E8fxmk1uB/egqfp", - "LQBoiWw2GzOBA1CK5TxVIb9ifs/AgkdprP9h/jhZB1CicDD/CK9+N9quGc7abtwEn8SmtHMKiSka9E0u", - "KAzBnmp8qSacmwIoMaXIPe8pcKj+jNx9/075Ih2/w6tJS1FXkOu72VuPffLZMTjcrSI9nso2N5zmZqL4", - "au/TFabN3qefIx5cSpQyRaMSqIG22wAHVP+Y4zbaiz9QEyA70pUSR+Q6oQIQbCrwCIjoGUuEkSIipgxH", - "WzBn0wggUDovFl5wCknKQUQhTYyGBCU8igBl52pOGNKzAUeVa6BwTyttBYjiO8UrRsXRhAQ8Jg6Vc9Nn", - "uv0dU/WKizLE5vciF98X6K/no6eq57kGVbS5xzuhjJ7iawhrDlN7TexGtPGa5z8aV1AXwdqMOjsDOep0", - "0aizHY86egWOMLhQsUJ7KKYsVUT20bHxb0Ea6v4ASRJwFkoHDuo8eDsD2ZSUatiyIcNxH757TLXHchWQ", - "8p3txCce9HtIfw8JNmijuOHsngy7sOlCxFMFAdxuX9m3QqLAPbL56DewhT3yw7ZvI8n/brdvSUbBKmtx", - "WVh6I9kz+Mi1XjeXVDHnMkedRAFOcEDVsotwFPEg9x6kMrsd6GVDmQiCL7UN1R+xdxlwpU2EQEdnH7rO", - "aYZCKi9NC9Yv1kdvF0TIdJINDoE0MB48WAwSjpjiKMBRkEaab8l0SgLIYYhoTJVs8KtlQ3nIMoh5J56F", - "dw8z2Jqn5Uzy8wSsXs4WssJxW2aptwQJIkzjolOpShxQfeFKF9y+E90o18fwNLLXW4HgUiLbVI9EdEYn", - "kb2skX30XqscOCYjlkSYMSJQKk3ckR56LxFEytQkxugGoM6s4aguyoFOEsGVdRNHnAtpPLuawz+eIqlI", - "soLN3pmWT2HODwQTbBq3PX0jg6EyhuZjyb6C9IIYTjEE13ykj+lvEOxjBvSt4YSfysZ/L+hsRoTeFdgI", - "WXM1ara1I6fZ9KVMj0aM/PPsrXYY+VmrhWjuQqTzSqCKsXtxDAr0TW5gPZ1f0kYsE/voZtkXv+qPWvZd", - "jvL3D8I+uuMs/yylx84LwdVtkfVzDn9qIPeFkZe2ailBYT0cQeuMhIfMEGiNO/DN4AaeMsoALqUdNMEJ", - "fH+MMHjc7LjHhtl+2rxVQgkoFdZpSJVaD9/5XXDgw+B2fuPs0Fvgdn5X+UqAu/jt8ka/q0ylkh/QFQ/5", - "0yNzPlSCkoHnBBiLpgQlI/VsIMFKQ+mjfaedmWRb/DNp8Pbu+Qb6uyP7D6u/hclQIJbfZWdyox1uC4kT", - "tXSXi3xauQCU9DMkY/iAH7IYgofDW7jF9fr9sYfj08bL9R/1tB7t/j4vOnxy/PSLaBX3XOlg2dKnTg+L", - "YE4XpNnpXt7BlkSJIL2EJ3C5EhqCWXq4s0xh0Z99RrZ5i1Vl/4WogzgmIQqpIIGKlogyxUEimD7+IpHg", - "2hKA51wsfc704s59JXh8aGez5jy0e8o6w/I733jZC7HCvYWTNitcaHe4aXd321rgIcrQ65/RBrlWwiDu", - "oqm2fBCdZiQl1wEhoQSe3CwOeDho8GzSz2Q8m7QZ5Qrs5LcWmxoFqVQ8dmt/cow2oNjCjDC9FlrVn4Im", - "mwi+oKEpRJoTdcEjQ9VhA0Fv6nfVSkVWKcMZF2Zw30SHaXMgzT7TpCwWTOhC56AzoQzD4NaiFJf3lEmo", - "0v1hCmkN+d5xnNP5cYRZy2/DGTuaE7WR44ioODfQeJs/jrmnfMwVA1PdmVY67dqVimwXq9oyhPQhAHOz", - "OObHdVt//H7CK6l8kpGV1nW+yAzSJrf598WCg8c7Hx7bXf7xCYfjvybO+C64yqEB3aKPYX7jAY5QSBYk", - "4glUkTTvdrqdVESdg85cqeRgayvS7825VAfPB88Hna+fvv7/AQAA//8QadjqX44BAA==", + "B7VWY9Gc8h7eP4fXv3Y7V5gqSPwvxeGsjq5gGQ31WqWRKq21bhEIbaJrCy4/xeEywTq2LEqMICoVdnmL", + "oSjg2UTnRFmHo+L2VYSnylp2H0+NkSbTGDxlNCL1XqeUUVg9LJcsmAvOeCqjZStvYttEAU3YtY6mFnzJ", + "yNVjMCVkUvT0tu5JhpOH4chVsX6ZLyZ/CbQUQUPSRyB9IOjIZW5WJNG54klCwsw/1h8xG2mf/STNDRMw", + "KdBBzQkViAs6o+WOyw7IhwwavMlWddz0Y7s+0HYtErZu4cBDCP9pPjTMxKYmRtEk7Rcz7yyTdrqd8wxT", + "xVK9zDrvMlyaGsfkQcK1Ib4++3DT0MhE8Cn1IWVBLI19ai17FzT42+7gvDf8f0wAsN6PoOJTZuJvYh5W", + "IFDs++00l9dnH86axpSBkqDi6GpzyiKmVsGyOYrYS0l7q20tYCcetGKSdZLbbi98ttBU4JhM0umUiHHs", + "cc6+0s+RecGExlGGTn8u20Pa7mrrdTkrLQ64XaY4sJgS7ajvcehWptEtUPOTf7neEaPGNWWi6qUS9h2b", + "jNpHbzIYGPT67INEeZSbx9NbXt7GTI+z+VLSAEemRZNYTlnRQQvM2drCOss/tK5sj53lRw9yGwFtLGZJ", + "Ctvw/F3v5O3HrTgki25pTBCZNucR0ePeLEiLhctHzdNSSkJi0eQpM4wh226gAq2yHdyaSIX96qGO4gpH", + "YxlxX7DPe/0QwUO08fGVyRfUI+iipLSU+vcCFUr8ve/dMVoiNXV7Dh1WXe6lDe71PZRxXI17rjC9Uqe+", + "rWKSNOo6YB26i1+WF5pfroeLMo0093vk8kgquoA1RJBJN0GQbuLiF5D51JzQ1rUmSYIFViRaGs0rO/oi", + "OiXBMojMHif1q2hyTYIbJLK81K9/NXnoqSBjNRdEznlUDmDY6dZhAyUE0S6IhW8xcypc3CiOYiwu4WB0", + "hhhKmaFAOWllZx0k6lyp5AaT+uX9+zPjnVFELExEYjFIXNau5o9JhJdoQtQVIcxNBUuE0Wue4eRUs7pk", + "A4SGUOOECMrLNOzsePo9N4G9aCZwQJD5yil8dkkknJptSWl78UDBBQGRsmF9h6vW1346TaN2a+wb1nAt", + "6G1wkwV+f3TmoCAyVEtH5u06lc+I6Jkt5+AtVy/ttlwNSuK6YpyRemeCTwiglNjErWKKmQtMAhQW/Xkp", + "rasgHGQxn8r2A7vAkKpr9vmnVspedbv7AjFizEIfOqiJmDe5s7M01gsCFkYKd480NFFLxpYwankx8F8Q", + "HFJGpKwkHgapiDrdTm9qZ3WwtRXxAEdzLtXB7s7w+dbq+M+Vgb/WnhmHdJX966weEy/j8vMMSidMuswS", + "WzhJWnhQDR3XnA8gnuqBhoAIqs+2gobnIs8Hg1oO5TUOlANWApdq6cocF7dtolmyNB9oMIOl3nvxorg/", + "B94YhhwV3rH/Vo339cxMMCJYoeCOqdDRMPlnr0bFhQ89hwtlM6QmxEUrZuehiwW0l3Klzp4Pnhdn2Qp8", + "G4RNZZvbfeeZqnm7nDVJCuS2+9c2AHFVZZ3DbenVWHuaLmt4SkvEGkutJihPCLsRPfd2d7ZvRs+2Ezlx", + "SZ0VueTDbDg6PTY6UcCZwpQRgWKisK00UBAy4GvTUkYb+CEmMeS5TP9jtWhpiH8pgjA0RlAc1eD6HiR6", + "ogFm6p2J/w1RjBmdaoFs3yz2LOd4e2//wIDhhWS6u7ff7/dvmpr+Ms9Fb7UUWyb9tpCl3pfzu63DA2Sg", + "t5nLl87Z4ftftCBLpTCH1pacUHZQ+Hf2z/wB/GH+OaHMm7neCj+RTmu4ieV4Qm3wm98PClD3Tu9pBWft", + "d5ZDWDHAZXhhhhSead3GcNxd8YRujTiYw96qAtJgMRurBeog/bw6DMF5leAd22fKFI1yQMZ6AMKtIDXl", + "StSxGuJYQliGMxZF5q+As4XeFT7QsdJJ5J7dKXhnper197rGtX6/OcVrDdv6vWyZ/GsLtmghkTwn0TeX", + "+rcJcCv3/nb2n3/8v/Ls2T+Gf/z28eN/LV7/5/Eb+l8fo7O3d0I/WI2G9U0hre4NxQqiukpQVm1Z6RSr", + "wOON0oZOA4XtE2Nbq2DeR0fgNT8YsR76jSoicHSARp1Kft6ogzYI2ATwlVbsdFM2zXhTf3xm7hb1x1+c", + "wve12kZo84mFXZAMhUCmk5DHmLLNERsx2xZyE5GgAeu/QhTgRIHjgjKkLb0lmgio5mPvNvLOu+gLTpKv", + "myMG1wPkWgk9gwQLlcH3uR6AKeyoTMCufZ2EDg/KXC+MWHYuZXBQ5gKwn6m5EBhTTXfyE2W1pWJthOcD", + "H3AWpFzohYyoVEbZzjhbs1GWC4KeDzbrlssabTrjoRXsBzuhXubLMWWLvWQYGLo2gnvsnHFrAlm0bDJ7", + "BIGtpDj89xy5hnJaZEtsPOTm4lOaC2gVyULqzWbHW2QAVrflhMwNI3wWtUjZf2lys97/do4UEbHLlt0I", + "NDmnNNDzg9hbKmWqWZFidHh0+nKz36JOGdA2G/+KdXyfzbCaWW1vHJsuUnPDDseki06OITfO7tBcgYOY", + "9ldcoMgImHxfH6APklRsRMh+hJBas5LRMr+2NCfAqLPpWkyqkuIAvcv0RpwNpVQbrXwTmu9LaNZGPZmA", + "+1rr3VrVIeHsIivaILweqyxDU5+4zaKgvaPCUhz2fMWsvvHeLt4kNxrNhbW/b7DE+1d3dm6m7rjCF8kc", + "Sx93z4tXIfDSimpTtOLfFc33/qV+1xZ8yrqDGlb66Ct+7ivVs9cbDt8Pd29u898U564MCFPAIsqg7tpj", + "1D0E1lvd/r2matwYUYz0Yxs/7Ky8j6dojiX7i4KHFVtvuPOsVdUC3WvbWNxiFC6fmiFlUsqhy2QxpAZn", + "55JGkQnNlnTGcIReoI3zk9e/nvz22ybqobdvT6tLseoL3/q0gLxzouL12Qe4TsNy7MK1mjO4cJ4FSa6p", + "VLIOvtMqKvQuEHvm03aw9G6Spo0cn341Tt8vJSw9L5rS5j0C7LlQ3hoZHwM671vmSH1/sH0rgfbuipZn", + "jZcHAstrFO4+oLmynDc/3y/s3YMMZ22Rx+JZ7xJYb40z1+1QT/LeodQimITo5CyHus+djK75ypxsxdTh", + "YNAfDtq4XGMcrOj79PCofeeDbaNaHODJQRAekOkdXL6WsY0yjqMrvJRo5MylUcfYZwXDrLBtrUnV6nq6", + "Dud3O/S+qkLjl9NGsXP3/tKWR2pSblpkZ1XBguI0MgnAxZJK9SqxMjHIeAZTOtNlRwwG2LUIPVllVxwE", + "Is39Ga5kn9F808Ty/YgJIhPOpCnk2Ue/kqVEMYU7hKx7iBySKAtyD0dsQ7io5CwzJMGpJKH+AaJpuy5q", + "Uw+NKsDO1h+MmJynUIFws4+OOJNpTIR19aAJBT/0JpKpMe5gvEANqIcraUjEiOnXPNh7XzJF/WB/MBgM", + "ssqInYMd/e+Bj5vuiL/oPm+nctjPc13Dz8Pr4Brb4TDeFfxuVTml83IhpdYW3R2qjbYKxnc6XRaGD1+N", + "b3JXRlDA0yjUZsJEHwzGi0NC62ySROU1quAs+cAumWbn0tRtkJvi6I+UiCX6eHpaumATZGpL8LSYOGyo", + "hnXgyY2WYXuNYb12NLeERHwMGMTqoV5Qpu4d9LDo4XfpkoZDW3j6c+PKG6JMmVkazScr5lTx0YZkMU5T", + "n86uHzmQhA8fTo5LzIHx/vD54PmL3vPJcL+3Gw6GPTzc2e9t7+HBdCd4ttNQBK99isLtsw68FpqvqpmL", + "PRy7GEhv4eqGCNTKEWmj6q4oC/lVq5LvWe82hGpd9/UAydZD8IZVQ6VsaKlBSpwW6mGbsMJKzasGz9P+", + "+8FwjeepXRnvBvn7XqQsMOhlIIkzn26xgHdxsepFfG8uTmFALnx5HbWKnbcn2uBg78XB3l2J5kJw142x", + "yk6PuLhNAQcOArMS4+vyTAr+i0IFbNA3jJvVhgR3up0sahn+hoO2EhGXPW4Vit+0Ybt+MbJKfjdkpJ2U", + "1Ga4hTVIWOGB1gKyZKZJqlCWCKrVi6OIpyEq+H4MMBBcjJwUVOgDl55nXUPF9DytagOkKABaU6YFMVwI", + "6UZs+toBeg3vwiMcG+vCDgKzsHIXgsOluQvW+8t1bXT91UM+t2o+fKN1fgSF6fW0NRmsi3B1E0bzOUBv", + "OHyTGR2MV32N5nXQ9uuvV/2SGzavzOWvQ2dWjTtArzLVLVP+rLK3IYn9c2wFVg4bsVlKTrUr3tHckq9c", + "Ia+w2zEU7XQ7jlCQf1jPRPyQc31t/xVZ0RckQXAEeznP9EoVjSxKKsyEQml2GwKsF7dJv7AFBUg4NoZJ", + "U8iTSR+yxkv2kVNfPp6iDcBD+xuyjlT9r80sPKp01m2/2H2x/2z7xX4r1JN8gOvVziNIbqsPbq0OGiTp", + "2FU9bpj60dkHY4IHxrgFz7ydeyGJOhFcix4987yMct75i/6LIthLyFNTUt4OySJDQc5Km5rXDTE+f9Bo", + "QadT9sfn4HL7H4LGw+t9uT3xOjSzjvzen5PibW/NVUomPVMExo/HAQwlZCNkzTsiYQaQtgz800M4ANMh", + "y0mzLOeAbVwas4+xdnd2dp4/29tuxVd2dIWNMwZfkOdQtiMobDF4E228Oz9HWwWGM226RF0AmGXWrPTv", + "M2RLyQ3KCml/ONjxcUnDwZ1zjW17ETeS/KM1zeykLNEhtS4z22q73EvtnZ3Bs92953vttrH1Uo7F9WoJ", + "4wLPDXksDnJx5TdAm3x/eIYgrWuKg7LfZLi9s7u3/+z5jUalbjQqwPA22Ls3GNjzZ/t7uzvbw3bYS74o", + "AIsqVtqwZdnl2XQepvCshocUddHbbTotfOqUYbB3JIgwjQ8DF8FbOX0MtsFYmNfyRWhzMFjHeO3gavFt", + "K8dRpWS5UQ24QCnLkN37668Ab3ej1yymzXmwXozXLfsIM00uC4JhSrncgnaJIAvKU3kPDXFlUp2mEefi", + "Rt82WSjvDL4GXLtRiT6e/gWEiGYuJBVJykaTZb8VUCG3nNyNNnCJJ/xc3USsVqvRZulXTbjbsE27q/Kg", + "S9u/EZEn1KIqZevD745wFKRQvABn66lnBdgRkMmZJNHSBKpGEecMBXPMZgTKUJpSKWyGMJrzKOx7gwf1", + "k/HUe23Pr1DEDR7LJSGJxfU3g9CfaZ2FLgjaKGSSIsNKlfJee7GRKha5vcyNe7G/kBSWvuSHLINR0xMr", + "XoABNZ+UfIwRn0mwAhWE4Par6NMJFiayFjNTp2IRG+OxDE20rU97zxAr0tt3hJqjk0+tRWt1DMgPNJTE", + "geBSIhLRGdRE+HhaSTtbkUORJZ+tD6krD7YF65qLNM/ZBWeabF3OxncgeoLT73IkAg9DDsqKYDXnjYwx", + "SwHpv8DI5DqhwrBHu4C0OZdqnMGJ3HCwUo0BxT0VJMdkypIlMweQe8d7LjrRdhty2cjPW31d4yp/U00D", + "bJapXor6qdXNeNDHxnVAlZUYLjkoTBUB5CaQPznsN5XQKi2gzaANxlVJLBWgqzfbBGf4bVTdT808tVXK", + "ftsdnLdF41kNvnOG1fyETbknZfsG15DO9WzjBRMiYgp1DFBIGCWhMx6z+0jr24I8v0gSFKbEUs4opAJb", + "gmOzvSHtmjmnGGWziqyvdtjGH2zGsBrkHfq1L7YJs5H+7LD3IgVamcA4iXCeJ9YqypDKsf/+qt6wILM0", + "wgJVEadWDFku44iyyzaty2U84RENkP6gesk85VHEr8b6kfwJ5rLZanb6g3Ge5lC5NDaDs0kuZkEq/eZT", + "+EnPcrOSYgeuly3z/RYk+reJWvLG6r6iEbGgTB8YvS4wehkFeXd70JR92dBoKe+yDuh1U8ltWda34x3W", + "1mFWdNVzS2mibit3pWVH5NqbPgjrXpVrWnfFoA0XCOVQpst0LaA9t/KEtIssr4b8udFsSRKUe999vvds", + "vyXc9p18nSaF/b49m4t4hUezYaVO27jNnu89f/FiZ3fvxfaNHFQuOrRhfZoiRIvrU6mtXHGa7UEk1eBG", + "gzLxof4hNcSIlgdUqpN86wF9XbF1m4IL8r3ZdMkZFVfS3bOUPaDtfIwrtKXDksqV13ZGG2Q6JWBUjg3d", + "evlgKhmCrcYQ4AQHVC09DhN8ZUo+Zq9U4N7aeNPKg/WQ1LZtoYG05JLpJE+i2HCdo78a13qFF563Ru2X", + "6aTJjf+22qtx4uc+oOIVUYsbmrywaN1dkM3nCstSrJn+O4DoSRcnXo+ZNW+shp6qAn7AJaAtTlGIpPBB", + "FlbOP/tRcfkry1lw+5aU5CrFVx2hzVvwRja050T2mNDB+kyYinywB+DtvhpPivU0VhYsKRXfyE/dm/fb", + "ItmnDhaanWA376+Q9nCTD6vAWMCPdgyW5Hnb3RJLNHBTIUzXY47wiPSyOAcbw4tkavyres9brEVPHkZw", + "yafTMuDTXjNAICDzQci26wUrReJEdRG5BjOdhDV0OVeHfU+OOogLNOoM41Gn4gT0JkHE+HpsOyjnKg9W", + "IfZllaSqg5RuBpOIB5emVIQSlMg+GqCYYCZRymDzV3yUw8FqX1u3kxTWJsPHI+aGuCa2YEwTMscLCrCs", + "1kM1K8WxkGuqJMTbQDsHKORg35brZNkZ6tdMisJBPmk4dDBb2oZ1g/o9zlxAUP4umEtTqM7FPhPBuzb5", + "Tkvst29Pu+b+ByI3zMBK4SFuomYEWkBmXVQwRvPf/eFXk4iMYdxVzMq4TsdiKhn4qQWRREkLYpezQ4UJ", + "UMBTpqpglnG7EM5yxHv9SEoZxErY2zOAbLC92+qaIQlsBeO6e6nE6Ldg7krYpaW0L+5yx8fCsCnAM+f3", + "vL+z7vXqACTCghRL3Jl2imFxxuc6lopbzP9sV4/JdUBIWIXj8b/SNtTQfukNNfwN25z3rPqcfRvCxeqz", + "6z9c7DmMtYnaxZBIxlkPsm3dktrMWIMaYnOvy4xWwt8rZKCOfeBHvhfa5E2R69W0fkOuFYAEhmmkydvE", + "ulZU2cNoHcVvnXTRtKG5WF+s9QFqS5hwvVtVl7CRfo9SYML+fP9FJXzLcU6Ue/fc8k1zrdUSvHHJI+gC", + "JN0r5StKwztdZE90NIw3Kzy3O/e7QSxAV8t8EYZjMk4EmdLrFdxiXjCWcDkbOd85pXq8Em3E+BrtPkPB", + "HAtZGTujs7mKluX7y10PGMCdSq0IoghTN6hcnK+m+7AeLGCXs9i6Txs+L6Tu+ysqk3C8CsbuKHvNXccm", + "eAlum0Yf67Od3cFgZ3twKxy7+yr0XGinKQWh8J29JylF9RRbyBK+6tXArgSFvLGMTFIJguMDCFROcEBQ", + "RKaA8pJVYVx7WNS6Xj14q0HZPPKM/91C2XVzVxhloOisKwsB6KbRcQFQZeiA4vP6sFdAwWRiJqhhwnhy", + "FHZ6g/33w52Dvf2D4fAhgO8yIjVFxz77PLx6Fm3j6W70fPnsj+H82Ww73vEaXg9QU7yc81ktMW7nkBBR", + "LfNWLY8oSUQZ6cksonx9WscKWWCCNNbu/5s59s0MVioL5+VJFnUGrHLilDjrkXAy7OhX3k5Uh39yvHrY", + "twrRrg7Ez2DVoQA/tRsMALEO74r6mbKW586HwoutT56VaQPrzh5flidsbe8qN1Dcx88lwVjaYatO7Pqp", + "5vGOzrigah6vPh6y1zIMQYgz+yxVWMZl6KOTGYOijsWfs7CCopmkP+50O9Hn3fKesb+3R+iwmHkZA9ql", + "LqoBLa7doWboairAK7lpIUzkn7bG9Zh/GvaGLyD4Lfq8+9Og96KP/l4IwusaahXJN3Rvl34dtKFhsdKF", + "Q0gfvrhRhJqj5yoO+pX66jTkB7FF07M8nldEc2eFy0gqLXD+uLbGFRiBRo3zrqqdPc3GRS0pJBFe+uqg", + "F1yxsmIfFpkMTciMMtnGM7szyFyze/Go00eHFoMSrNW8YGqpeSgFWeATGsckpFqpNMZ9c8TndktvW9V4", + "uBkwsfvKo571/frZi/U5pOsC1Ncdk/07JCzdydxtZ+KuSm8Gz5mzSaEECrzYRXSKMKvU4bFVi22mIWSO", + "AMLMgQNUyVnWygCZK37OE9JFM65QnmPY0qOWsmbPXzZ+cg0e1RVJxYYhtu8lYzxDL6GrxNfJMUoED9Mg", + "T7CJYNB5SrRIK2COK7T69SFMD+nQgMw1qO+51qHR5MFo54FsWu+K91EzbPNSDwfrl/pBvCDdTpqE62WY", + "eamdBLsR1uialA2PT6ZM9oomWJjMpxYS/V2RgnUj13iLA60SpYm7QtE8Veckz4UKXCL4YPeOSUT0MVVv", + "BPEozKNKqcyl6HqROtx/Pm+6xIQ7p/pAfiUk0bYKAERAfzFmS+/AqkVW0cbAVc6S5kqrZ7DKLbXKg3u2", + "VhNrXKr29WorXm2TUF4sn5yBbt5vsVr75dpS9A/hh/uWStpbe7lQgVdz4H8ZXKnr32TMADwjiyrn9a7v", + "At7HFu+tZdwEGVfNmim6mQ97/23cymjcP9j66W//d+/TX73u5YrdLInohWQKoUSXZNkDvHukbfR+GTAN", + "sHq1Mj2zrEJwDE6j4JIYJ1WMr4vj3RtkQmP5Bse1KUAMVkxZ9u+1E/rbvzVHMBXI+AHk5FqWvTOU9UPU", + "CVLcHUcbMREzV8nYBd5v9kcMYvkvyVKiQjECq9I4Rv2LzD7RKjo4LXGELowa2CdscYEmFGq6yBHTVi0O", + "ApJoa8KCslNTjI+D9BEER8V2bFEElyhnrxxNxABBH09raHtvP7z/+e2HN8fjt2cv3xyejH99+V8QxHHV", + "Mz2EPc17u3v7tgRfkZJDzxLfAfj3Tih+PnYzWGAe/oKkFKhr6FGYqYSUUhdQUHgZbZA4UUtXbMjltmze", + "DJvsMGvQG852zyjsgxf3UXTmw8oqMwse9bRG3QCi63VgGlp4w7GhKRPm3mlybM88ZcDPrTdxRmfY48v2", + "ViC9j+IwbkBrQeNq699Y2sEfHH9cRRg20sCQqoKIW7FLpeo1x87HWpEa57UeyxEZKbPpJbQQsFXOJYmZ", + "2rJFnHwpraE+eVcnFOW7zCEW9eCj9XkyK1X5wswKI2lem1OnsVZ06hUEOtOkuZoTQQoLAR/kyK03JJlN", + "9miRKG1KrSRE5IGQLlMEap8LCtkjmbPBkSBLCKp7YFcj857i66wH8N5jWbvjgnnkGPnD1z+POpt99M6V", + "KqVT1wQMo2JP+IFQy1y0iiaOq+qLUeSq+rzN+96NZ2XVCunXtLcqzJn3UWJNHz/+HVP1iguwQJrTkh8c", + "TxWsm5AIwGWpoqW2ghqlMQnHWcHmpv3vajSbnOSsHHZexslZWxiYWAu59aV2XOJsPoY6pTU5SJAKqpbn", + "UNjVRAgTLIg4TM2Gd/Vh7c95x1AV6etX8FNOPVkIrwkjggbo8OwE9mOMGSjp6ONpoZKJKWpTw1AD9fLt", + "0Ym1cB0MH1gsVAHruWC+w7OTTrezIMJYeZ1Bf7s/gM2cEIYT2jno7PSH/UHHVPSFKW5BEUX40yYYZpbS", + "SWj1oJ/NK/orgWOiiJCdg989iXoQyAYvg76LZwWLJcFUWJMliSB90LAK1d8CsK47Sg/MeWwL8rZ20Em1", + "tMkUJHlrl/UTqJOwa2CK24OBhRlV9uCFVBATf771DxuMmPfbSp8D8nhQZmsWhdMpLcm/dju7g+GNxrNq", + "GLBjfd1+YDhVcy7oZwLD3LshEW7V6QkzGV7IYIXZQJviPgMWKu6w3z/p9ZJpHGOxdOTKaZVw2aQME4kw", + "YuTKVgT9B5/0kb1+gLIxcs7TSEsTZNLXnKNBYdGffUZYBHO6ICNmz+k4jRRNsAA3Qoz0+WwMpvLWMF2b", + "1c+ABX7m4bJC3ay5Ld1czzmdcwJX0xIkGQMO8bip3G/ubqaMaTGJJbG1NLK6l/VoHi0uxzLgvnyi94Rh", + "pnoyIQGd0gDBy3r3Wo+2t8FW0Hxa4MGyEAGIWc5Ds73pz0mFqh/+dO7j7Bmy5C2rEwyugYIoDXOdy6VJ", + "YTHBUeTFbppFfIKjsaHPJfGoqK/hDUuUYoEUp9wwHhJT7CJZqjln5u90kjKVmr8ngl9JIrQKZIuYWVqT", + "0JQtM6x7BVifMRQSMyVSdZ9bZohbXy7J8mt/xA7D2JW/leYTHEmuT01bdNDmBZgtbXjXX5alIa7kKJWK", + "x5alsgqM+TB5qpJU2Tt1SZStvAavU4mSVM5JOGKKoy+CzKhUYvl160ve41ewXQgONZ8UXjFT2vpCw69N", + "o5ZjrGc/hlc91h8BAow6+nQZdfTfM4G17ZLKOThRJDhOZsUl3cjwdLReuFmlcIAZSnhisIiAqeZYs1yp", + "DYhHx1GEFGwl963WNmElG+Zj04vjSWNusUkGrWwjytDpz4XNNNh97t9PkgSC+Bwc/3n+9g2Co0qvgXkt", + "d1iZS22mT1EUpqDJQ+/9EXuJgzkyehPgzY46NBx1Musi3ISxptJGaPd6oOL+pIf2k+mmS8Of+n3dlNGe", + "D9DvX0wrB3ovJfFY8UvCRp2vXVR4MKNqnk6yZ5/8BG1K0TwvCQK0YWT/pqtBDFBR+TFozg3MQsStrI2W", + "CKNcAhX9KBPKsFhZQNlDektBbcrjmSwS48sIfLejzsHIeW9Hne6oQ9gCfrMu3lHnq58CVoluBjc1NaSd", + "rp0x0f5gsLkeO8HS16NCl17U2+9rTfvavjfFwypddcXDTM4hM+sVNNXAjbr1CJrPzzh09SV/qHhrVDzr", + "uSgob/B98Rww7BsRY+BWNDBtz0ZOA1tpnRi2gIIJYHE4pBNjcFCnweXMWzQ/quZ83azYbdplAQwxcvy3", + "+wj8B/1mpQhNvy8eq18cAc64A65/YuwIi+UYseu3iF8T9T1w3OCxRKkFRf+W/PtU+Oc1sXpfTrSKNNsi", + "C3ff5MdzgmQTaVsxL2tb9RzG1DsnTKGX8Gvf/tdZPJAtfRHx2cUBMiSM+AxFlNl7wMJtkT4ULS3hI5Nv", + "kn1n008cmOaGOT//+T//C4OibPbP//lfrU2bv2C7b5nESQDfv5gTLNSEYHVxgH4lJOnhiC6ImwzAY5MF", + "EUu0MwA1MxHwqFhSyeomcsRG7B1RqWCF+1KDayltg2B6MJgPZSmRNl9Hv0inFnTLOJg9Jrzby4aUj7qj", + "u55EZ5hBYQL6VHQ8AFmitvyatb86fu+ZmXPJf1b1ldc8puvliyLXynBvzwzwhgIGSOzbd/DAThptnJ+/", + "3OwjsDEMVwCwGmjMeTNWee7/kEnrZZKRKGWBAlQ2sskksa32/x7bd9o5gG2LfyYPsMXavIEL2Lg8IHfd", + "rcAPW6GFO9hPN+ca9vlnj12WZrOD9vbzLXbh4phaGcL3t86O9+o0N08KJPsWJjDacNHw4EbkAp0dnbia", + "tpvfjOkf5dTQM7W1+rKjA3EG2GuPZpYdcTaNaKBQz40FCjPFJDPVygzyVMTBOztqhN28qhDGxfNtq4TI", + "13jSZeB8+ZH38KdHpdObHCM5zHLOaz9OknWsc0xlwPW3BW7pBTgBQjr1JdunRS5a55AywfXZkbNSXbLi", + "+eTYbcjHc03ZrlNWPRseQSgeVwTiNxSEleLxBWDyp8TNH7JVdIgUKzxX3xdrDh5PC3psL5aPzZ+SGyus", + "kE1LQRPU3XiAvibKhHJ3HnChbQ+eiZ8T4Xa1qyMBs86mZT41hVXNhOBCerXte2JeaWf6mvb+TJYvkOcm", + "Gosl+Q8VpYWxm9NqlYF7YkuWP5x9Cz3cyLy9v3tey2AeIkOwycR5rE3dXSyXLNj8U131PsppZoj9JA+z", + "szSK3I3HggiF3h6dmJ1VPAO2vkBY0nrd3u22lcfBh3e/9QgLOMShZTFUfiXKPrlnDd8smJnKDzZpYxOa", + "tGjqzrMmDecO62/CBZGJcOxT/u/bryI6EVgs/337FY4Sysi/7xxGWBGpNh+MWQaPJZofW+N+wsynFW5a", + "JhqIJgb13tdpqNlbLZVU9/6fSk81k76RpprR9Yey2kZZLZJrpb5ql+JBNVbTxze6ksmYzUdteOTiE/9k", + "murjevksRzqgaCrL1x62yB4X4OeFR5ShVJInGEBJM44rHhst3dX5hlx5fDjWPTnuAiGhLgKgNtkEkUdy", + "XrtxPLpya/t9fM/1YTyhs5Snsph7EmMVzIm0yUoRKQvgp6Z258dzo+L9HXPp4DGPjkfXq3/w/QNp/NUF", + "NcLb3ECt0/ndW211fvu+1vlNCrXNXbPQUl0HO7jZEFTokqjbsnEp17we7Ogbl88WQR+0oZKbCwgsiIMR", + "+z/a/vhdERx/+sklyaSDwfY+/E7Y4tNPLk+GnTpWIQwqHgFK7OGbY7j2m0H2OQDJ5il51XGYyhPAeg46", + "51/OQMpvPttbSI4Lf1hIrSykArlWW0h2LR7WRCrDbz26jeT4zUdwC2Lyw0p6DCtJptMpDShhKq95WgsS", + "syWTn2BuGbP3Q4XgjtJB29pKyjblGgU0Lwvw6IE9JzkO4mMbR64CwdOMkeeJhfS25kh+GDbbI98bPwwe", + "Vzg/vh3ylFnMKPx10iVap/SV0gaMyTiFspAoRwiBqE8kbP1H12If5RWsZZokXChpcCpBATZI9nOtAPsw", + "LcswlT5cSsBipER2RwwqFejHJpd/65IsDQol5SwDnKzWY/XlXpVRQL/pNrp/HcsPcdpKx3rkbWxBq7+d", + "jvXNRMejaFonpVoAG9nGAINyQrKdzLPkPvqZstnmk4pANcIqm1sBz8ijam1BpT+L67sls2qyTQdtAdrX", + "lp79Fzxx65P0ae0Oi7ZAQBRSPGNcKhoUK+8W4UF/nNCtT+jVlPVy89SWSPcb9K+4uGx7xHnqij2Bk644", + "w+/Ql6CHB2hg396lAMa2OQ000zz6KVgrFvctUzBo9VwMojSEqvT2QHSq5FTweGx/NHi1eldYNFBwUQS2", + "1W8tbHTvj+AwesMVonESEa3FkxD1DDfp1bSqv4Obp7JQWvFmwlBvm2JCjAGjk640kRWRcLnmFmwD7tnr", + "y+WVmhGfrQfByDp3iA8eFIwRM3D4xGHnX6BMyELxLhKRQKGrOQ3mgIgBBb2goiuAVeAkucggsDYP0GvY", + "qUUkMOh8QxKhDaGAM8kjYoAuFnF8cVBHbP14egofGTAMg816cZCVXM8OCKnfKiJcZDWP3ljcjg3NSYJH", + "kVnRC201Fua3abEvcoiyEfPhYDByZRukU3RRgMS4aMDEcAL1Nz77ZtpWtxlY0sxFcSSAcIY3CQs7TRcx", + "NPKjYQwH3nowLZE5zDAeGJijNpjf+CwDtSyxMk6StuxrhwlcvIjjFTyMNgrFWaUKear+JlVIhICPLXc3", + "MTfawIH5h8KXmlFtYaGsvC2wn/e60aDMeUmlhWqhio751yKOO92OHU8Bne4G2vsahJNqg/VrMb0yBRiT", + "H3r3TQBKysK+gFBSOTls+f9mlfudeeFP75+1hAr/DF6W8n1WPgrK8gJQAiq4uypcTwrpABaypouZwki+", + "PeJm2ZOF4qHtrrdqZUe/A6N13a1XVkMyK3D52Ndf9RE85SQYWZvNlItqevy6e7HvnpHub0lqU23DIT94", + "8+buuVaMmaQraolCKVQJfj6orwm4zsGcc1lg+wmZ4wXlwiKwW69rxpngsjDWo42eu9CsemH9txdWPT+w", + "viaEi49sH3343Mbc+b9wj/IvXhWs7Uzid51KDSiQEmE0EZRMUYJTSbS2lMYEmQojFsib4GDuqoX3R+z9", + "nCBbH7PgQMjKKVOJLobxRRdNUoUiLGZg7ZiHJpJOkIDHMWGhqXk7YnOCF1SbagJFWBEWLHuSQA3kBckL", + "mGjT3d5QmlLbWZXVLnLFecHBcFEovXuBEkGAiYy5zEp1bkdMpOw/DHKlbvbCDfQCEanwJKJyntWKCHBI", + "WOCFhTz/vsXY/Ttxz4mqV6f9JneWt5Kl3/ISs+jLzOqDfxf3m08sUIsLV1mzhZhfofTKZtOwHPl4nlfk", + "/Rfc0maubo7f6GYmI/GqXfx9XMmUSvL/uJZRdkuGqemOlMvW/2nvWvI60ikrXbdYn+xtL1yySggZmW8k", + "87a+uD9PbuEj+04kYbfRsG/C3M4n/T2IXEvVW8ncb+QctL6kglfsG4pgO6hvpz5xUZBy34UYNhsuk8ZF", + "maMEBpuKsx/CuCqMbXjAbYWx87jWLsAL4pmyXhLhJrmcV633C2DrEPgXjX6tzK4gCL+54MtvBB5N2J1k", + "4s0IvAQvI47/7PcyARfCJHTacsRPB1Cs4AssXDBtgMetm0mIrssm+Xh6utkkJYRaKSOEesISolzWNIg9", + "1RrfLogQNHSlI49Oj230KpVIpKyP3sYU6jleEpJAoRjKU4kgM7ev5+dSW+tF8Eo5rN0OYUosE06ZWjuK", + "/NWHGczXW5XOe2Q5aSEV//SXx+CFf3pCCmSHVlfsBFZbkQqrxmA8F5xGmal3qbUtPOGpbl1LFldodwZn", + "25RGRC6lIrGJzJumEWwiAN21NZnsdyajtIuokkjvhy5k4CVExFRKypkcMVv+PSFC960/h+K/eZCR13mv", + "cCY1z4zo+z4C2PRgTMwWVk1UA2gBqAPaOehs4STZgnLR/iApO7w7DOkVRKQhuYwnPKIBiii7lGgjopfG", + "6EALiSL9x+bKkLYxfHffFaduv7M0pU/YlHuLchiezZj5z5GEVBZr7hLxyYm116S4WZz8gYX2izW5Vq4J", + "gqOeojHJkt9RqmhEPxtRpxuhUtHA5NXkqZdQhNlmX47YKVFCv4MFQQGPIhIo51zZSgQPtkbpYLATJBRQ", + "SnYIDA4EXvPjGHo8OvsA75lC0d0R0/+Aht8fnpmb2Cm2PoLCQBlRV1xcopOtt2uCfM+BTP/CUXJmgitz", + "IL0L/uP67uaZzY17SDZsUZ6sMoB48qcP47Qa3A9vwdP0FgC0RDabjZnAASjFcp6qkF8xv2dgwaM01v8w", + "f5ysAyhROJh/hFe/G23XDGdtN26CT2JT2jmFxBQN+iYXFIZgTzW+VBPOTQGUmFLknvcUOFR/Ru6+f6d8", + "kY7f4dWkpagryPXd7K3HPvnsGBzuVpEeT2WbG05zM1F8tffpCtNm79PPEQ8uJUqZolEJ1EDbbYADqn/M", + "cRvtxR+oCZAd6UqJI3KdUAEINhV4BET0jCXCSBERU4ajLZizaQQQKJ0XCy84hSTlIKKQJkZDghIeRYCy", + "czUnDOnZgKPKNVC4p5W2AkTxneIVo+JoQgIeE4fKuekz3f6OqXrFRRli83uRi+8L9Nfz0VPV81yDKtrc", + "451QRk/xNYQ1h6m9JnYj2njN8x+NK6iLYG1GnZ2BHHW6aNTZjkcdvQJHGFyoWKE9FFOWKiL76Nj4tyAN", + "dX+AJAk4C6UDB3UevJ2BbEpKNWzZkOG4D989ptpjuQpI+c524hMP+j2kv4cEG7RR3HB2T4Zd2HQh4qmC", + "AG63r+xbIVHgHtl89BvYwh75Ydu3keR/t9u3JKNglbW4LCy9kewZfORar5tLqphzmaNOogAnOKBq2UU4", + "iniQew9Smd0O9LKhTATBl9qG6o/Yuwy40iZCoKOzD13nNEMhlZemBesX66O3CyJkOskGh0AaGA8eLAYJ", + "R0xxFOAoSCPNt2Q6JQHkMEQ0pko2+NWyoTxkGcS8E8/Cu4cZbM3Tcib5eQJWL2cLWeG4LbPUW4IEEaZx", + "0alUJQ6ovnClC27fiW6U62N4GtnrrUBwKZFtqkciOqOTyF7WyD56r1UOHJMRSyLMGBEolSbuSA+9lwgi", + "ZWoSY3QDUGfWcFQX5UAnieDKuokjzoU0nl3N4R9PkVQkWcFm70zLpzDnB4IJNo3bnr6RwVAZQ/OxZF9B", + "ekEMpxiCaz7Sx/Q3CPYxA/rWcMJPZeO/F3Q2I0LvCmyErLkaNdvakdNs+lKmRyNG/nn2VjuM/KzVQjR3", + "IdJ5JVDF2L04BgX6Jjewns4vaSOWiX10s+yLX/VHLfsuR/n7B2Ef3XGWf5bSY+eF4Oq2yPo5hz81kPvC", + "yEtbtZSgsB6OoHVGwkNmCLTGHfhmcANPGWUAl9IOmuAEvj9GGDxudtxjw2w/bd4qoQSUCus0pEqth+/8", + "LjjwYXA7v3F26C1wO7+rfCXAXfx2eaPfVaZSyQ/oiof86ZE5HypBycBzAoxFU4KSkXo2kGClofTRvtPO", + "TLIt/pk0eHv3fAP93ZH9h9XfwmQoEMvvsjO50Q63hcSJWrrLRT6tXABK+hmSMXzAD1kMwcPhLdziev3+", + "2MPxaePl+o96Wo92f58XHT45fvpFtIp7rnSwbOlTp4dFMKcL0ux0L+9gS6JEkF7CE7hcCQ3BLD3cWaaw", + "6M8+I9u8xaqy/0LUQRyTEIVUkEBFS0SZ4iARTB9/kUhwbQnAcy6WPmd6cee+Ejw+tLNZcx7aPWWdYfmd", + "b7zshVjh3sJJmxUutDvctLu7bS3wEGXo9c9og1wrYRB30VRbPohOM5KS64CQUAJPbhYHPBw0eDbpZzKe", + "TdqMcgV28luLTY2CVCoeu7U/OUYbUGxhRpheC63qT0GTTQRf0NAUIs2JuuCRoeqwgaA39btqpSKrlOGM", + "CzO4b6LDtDmQZp9pUhYLJnShc9CZUIZhcGtRist7yiRU6f4whbSGfO84zun8OMKs5bfhjB3NidrIcURU", + "nBtovM0fx9xTPuaKganuTCuddu1KRbaLVW0ZQvoQgLlZHPPjuq0/fj/hlVQ+ychK6zpfZAZpk9v8+2LB", + "weOdD4/tLv/4hMPxXxNnfBdc5dCAbtHHML/xAEcoJAsS8QSqSJp3O91OKqLOQWeuVHKwtRXp9+ZcqoPn", + "g+eDztdPX///AAAA//9DeBEen5ABAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/system/guest_agent/main.go b/lib/system/guest_agent/main.go index 46ee3d7a..704274be 100644 --- a/lib/system/guest_agent/main.go +++ b/lib/system/guest_agent/main.go @@ -22,6 +22,7 @@ const ( // guestServer implements the gRPC GuestService type guestServer struct { pb.UnimplementedGuestServiceServer + resumeNetwork *resumeNetworkController } func main() { @@ -53,8 +54,10 @@ func main() { } // Create gRPC server + guestSvc := &guestServer{} + startResumeNetworkWatcher(guestSvc) grpcServer := grpc.NewServer() - pb.RegisterGuestServiceServer(grpcServer, &guestServer{}) + pb.RegisterGuestServiceServer(grpcServer, guestSvc) // Serve gRPC over vsock if err := grpcServer.Serve(l); err != nil { diff --git a/lib/system/guest_agent/resume_network.go b/lib/system/guest_agent/resume_network.go new file mode 100644 index 00000000..299dd779 --- /dev/null +++ b/lib/system/guest_agent/resume_network.go @@ -0,0 +1,570 @@ +//go:build linux + +package main + +import ( + "bufio" + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net" + "os" + "strconv" + "strings" + "sync/atomic" + "time" + "unsafe" + + pb "github.com/kernel/hypeman/lib/guest" + "github.com/mdlayher/socket" + "golang.org/x/sys/unix" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const resumeNetworkPortEnv = "HYPEMAN_RESUME_NETWORK_PORT" +const resumeNetworkPollIntervalEnv = "HYPEMAN_RESUME_NETWORK_POLL_INTERVAL_MS" +const resumeNetworkTriggerEnv = "HYPEMAN_RESUME_NETWORK_TRIGGER" +const resumeNetworkPrearmEnv = "HYPEMAN_RESUME_NETWORK_PREARM" +const resumeNetworkStartArmedEnv = "HYPEMAN_RESUME_NETWORK_START_ARMED" +const resumeNetworkSlowPollIntervalEnv = "HYPEMAN_RESUME_NETWORK_SLOW_POLL_INTERVAL_MS" +const resumeNetworkArmedPollIntervalEnv = "HYPEMAN_RESUME_NETWORK_ARMED_POLL_INTERVAL_MS" +const resumeNetworkMailboxEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX" +const resumeNetworkMailboxTokenEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX_TOKEN" +const resumeNetworkTriggerVMGenID = "vmgenid" +const resumeNetworkHostCID = 2 +const vmgenIDKmsgSignal = "crng reseeded due to virtual machine fork" +const defaultResumeNetworkSlowPollInterval = time.Second +const defaultResumeNetworkArmedPollInterval = time.Millisecond +const resumeNetworkMailboxSize = 4096 +const resumeNetworkMailboxSeqOffset = 64 +const resumeNetworkMailboxLengthOffset = 68 +const resumeNetworkMailboxPayloadOffset = 72 + +var resumeNetworkMailboxMagic = []byte("HYPEMAN_RESUME_NETWORK_MAILBOX_V1\x00") + +type resumeNetworkPayload struct { + InterfaceName string `json:"interface_name"` + MAC string `json:"mac"` + IPv4 string `json:"ipv4"` + Prefix uint32 `json:"prefix"` + Gateway string `json:"gateway"` + AckPort uint32 `json:"ack_port,omitempty"` +} + +type resumeNetworkArmRequest struct { + interval time.Duration + ready chan error +} + +type vmGenIDResumeWaiter struct { + file *os.File + reader *bufio.Reader +} + +type resumeNetworkController struct { + s *guestServer + port uint32 + trigger string + startArmed bool + mailbox []byte + slowInterval time.Duration + fastInterval time.Duration + arm chan resumeNetworkArmRequest +} + +func startResumeNetworkWatcher(s *guestServer) { + rawPort := strings.TrimSpace(os.Getenv(resumeNetworkPortEnv)) + if rawPort == "" { + return + } + + port, err := strconv.ParseUint(rawPort, 10, 32) + if err != nil || port == 0 { + log.Printf("[guest-agent] ignoring invalid %s=%q", resumeNetworkPortEnv, rawPort) + return + } + + if resumeNetworkPrearmEnabled() { + controller := newResumeNetworkController(s, uint32(port)) + s.resumeNetwork = controller + go controller.run() + return + } + + if resumeNetworkTrigger() == resumeNetworkTriggerVMGenID { + go resumeNetworkVMGenIDLoop(s, uint32(port)) + return + } + + if interval, ok := resumeNetworkPollInterval(); ok { + go resumeNetworkPollLoop(s, uint32(port), interval) + return + } + + go resumeNetworkLoop(s, uint32(port)) +} + +func resumeNetworkPrearmEnabled() bool { + return strings.TrimSpace(os.Getenv(resumeNetworkPrearmEnv)) == "1" +} + +func newResumeNetworkController(s *guestServer, port uint32) *resumeNetworkController { + mailbox := []byte(nil) + if strings.TrimSpace(os.Getenv(resumeNetworkMailboxEnv)) == "1" { + mailbox = newResumeNetworkMailbox() + } + return &resumeNetworkController{ + s: s, + port: port, + trigger: resumeNetworkTrigger(), + startArmed: strings.TrimSpace(os.Getenv(resumeNetworkStartArmedEnv)) == "1", + mailbox: mailbox, + slowInterval: resumeNetworkIntervalFromEnv(resumeNetworkSlowPollIntervalEnv, defaultResumeNetworkSlowPollInterval), + fastInterval: resumeNetworkIntervalFromEnv(resumeNetworkArmedPollIntervalEnv, defaultResumeNetworkArmedPollInterval), + arm: make(chan resumeNetworkArmRequest), + } +} + +func newResumeNetworkMailbox() []byte { + token := strings.TrimSpace(os.Getenv(resumeNetworkMailboxTokenEnv)) + if token == "" { + log.Printf("[guest-agent] resume network mailbox disabled: missing %s", resumeNetworkMailboxTokenEnv) + return nil + } + + buf := make([]byte, resumeNetworkMailboxSize) + copy(buf, resumeNetworkMailboxMagic) + copy(buf[len(resumeNetworkMailboxMagic):resumeNetworkMailboxSeqOffset], token) + if err := unix.Mlock(buf); err != nil { + log.Printf("[guest-agent] resume network mailbox mlock failed: %v", err) + } + log.Printf("[guest-agent] resume network mailbox armed token=%s", token) + return buf +} + +func resumeNetworkIntervalFromEnv(name string, fallback time.Duration) time.Duration { + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + return fallback + } + ms, err := strconv.Atoi(raw) + if err != nil || ms <= 0 { + log.Printf("[guest-agent] ignoring invalid %s=%q", name, raw) + return fallback + } + return time.Duration(ms) * time.Millisecond +} + +func resumeNetworkTrigger() string { + return strings.ToLower(strings.TrimSpace(os.Getenv(resumeNetworkTriggerEnv))) +} + +func resumeNetworkLoop(s *guestServer, port uint32) { + for { + if err := armResumeNetworkConnection(port); err != nil { + log.Printf("[guest-agent] resume network arm failed: %v", err) + time.Sleep(100 * time.Millisecond) + continue + } + + start := time.Now() + if err := fetchAndApplyResumeNetwork(s, port, 50*time.Millisecond); err != nil { + log.Printf("[guest-agent] resume network apply failed after wake: %v", err) + time.Sleep(25 * time.Millisecond) + continue + } + log.Printf("[guest-agent] resume network applied in %s", time.Since(start)) + } +} + +func resumeNetworkPollInterval() (time.Duration, bool) { + raw := strings.TrimSpace(os.Getenv(resumeNetworkPollIntervalEnv)) + if raw == "" { + return 0, false + } + ms, err := strconv.Atoi(raw) + if err != nil || ms <= 0 { + log.Printf("[guest-agent] ignoring invalid %s=%q", resumeNetworkPollIntervalEnv, raw) + return 0, false + } + return time.Duration(ms) * time.Millisecond, true +} + +func resumeNetworkPollLoop(s *guestServer, port uint32, interval time.Duration) { + log.Printf("[guest-agent] resume network polling host port %d every %s", port, interval) + dialTimeout := resumeNetworkDialTimeout(interval) + for { + start := time.Now() + if err := fetchAndApplyResumeNetwork(s, port, dialTimeout); err == nil { + log.Printf("[guest-agent] resume network poll applied in %s", time.Since(start)) + time.Sleep(100 * time.Millisecond) + continue + } + if remaining := interval - time.Since(start); remaining > 0 { + time.Sleep(remaining) + } + } +} + +func (c *resumeNetworkController) run() { + log.Printf("[guest-agent] resume network prearm loop host port %d trigger=%s slow=%s fast=%s start_armed=%t", c.port, c.trigger, c.slowInterval, c.fastInterval, c.startArmed) + armed := c.startArmed + fastInterval := c.fastInterval + + applyArm := func(req resumeNetworkArmRequest) { + if req.interval > 0 { + fastInterval = req.interval + } + armed = true + } + + for { + var armReady chan error + if !armed { + select { + case req := <-c.arm: + applyArm(req) + armReady = req.ready + } + } + + if c.mailbox != nil { + var vmgenIDWaiter *vmGenIDResumeWaiter + if c.trigger == resumeNetworkTriggerVMGenID { + var err error + vmgenIDWaiter, err = newVMGenIDResumeWaiter() + if armReady != nil { + armReady <- err + } + if err != nil { + log.Printf("[guest-agent] resume network VMGenID prepare failed: %v", err) + armed = false + continue + } + } else if armReady != nil { + armReady <- nil + } + + start := time.Now() + if c.trigger == resumeNetworkTriggerVMGenID { + if err := vmgenIDWaiter.Wait(); err != nil { + log.Printf("[guest-agent] resume network VMGenID wait failed: %v", err) + vmgenIDWaiter.Close() + armed = false + continue + } + vmgenIDWaiter.Close() + } + if err := c.waitAndApplyMailbox(); err != nil { + log.Printf("[guest-agent] resume network mailbox apply failed: %v", err) + armed = false + continue + } + log.Printf("[guest-agent] resume network mailbox applied in %s", time.Since(start)) + armed = false + continue + } + + if armReady != nil { + armReady <- nil + } + interval := fastInterval + start := time.Now() + err := fetchAndApplyResumeNetwork(c.s, c.port, resumeNetworkDialTimeout(interval)) + if err == nil { + log.Printf("[guest-agent] resume network prearm loop applied in %s", time.Since(start)) + armed = false + continue + } + + sleep := interval - time.Since(start) + if sleep <= 0 { + continue + } + + timer := time.NewTimer(sleep) + select { + case req := <-c.arm: + if !timer.Stop() { + <-timer.C + } + applyArm(req) + req.ready <- nil + case <-timer.C: + } + } +} + +func (c *resumeNetworkController) waitAndApplyMailbox() error { + buf := c.mailbox + for { + seq := atomicLoadUint32(buf[resumeNetworkMailboxSeqOffset:]) + if seq == 0 { + continue + } + + payloadLen := binary.LittleEndian.Uint32(buf[resumeNetworkMailboxLengthOffset:]) + if payloadLen == 0 || int(payloadLen) > len(buf)-resumeNetworkMailboxPayloadOffset { + return fmt.Errorf("invalid mailbox payload length %d", payloadLen) + } + + var payload resumeNetworkPayload + if err := json.Unmarshal(buf[resumeNetworkMailboxPayloadOffset:resumeNetworkMailboxPayloadOffset+int(payloadLen)], &payload); err != nil { + return fmt.Errorf("decode mailbox payload: %w", err) + } + sendResumeNetworkAck(payload, "mailbox") + + _, err := c.s.ReconfigureNetwork(context.Background(), &pb.ReconfigureNetworkRequest{ + InterfaceName: payload.InterfaceName, + Mac: payload.MAC, + Ipv4: payload.IPv4, + Prefix: payload.Prefix, + Gateway: payload.Gateway, + }) + if err != nil { + return err + } + sendResumeNetworkAck(payload, "applied") + atomicStoreUint32(buf[resumeNetworkMailboxSeqOffset:], 0) + return nil + } +} + +func atomicLoadUint32(buf []byte) uint32 { + return atomic.LoadUint32((*uint32)(unsafe.Pointer(&buf[0]))) +} + +func atomicStoreUint32(buf []byte, value uint32) { + atomic.StoreUint32((*uint32)(unsafe.Pointer(&buf[0])), value) +} + +func resumeNetworkDialTimeout(interval time.Duration) time.Duration { + if interval <= 0 { + return time.Millisecond + } + if interval > 10*time.Millisecond { + return 10 * time.Millisecond + } + return interval +} + +func (c *resumeNetworkController) Arm(ctx context.Context, interval time.Duration) error { + req := resumeNetworkArmRequest{ + interval: interval, + ready: make(chan error, 1), + } + select { + case c.arm <- req: + case <-ctx.Done(): + return ctx.Err() + } + select { + case err := <-req.ready: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *guestServer) ArmResumeNetwork(ctx context.Context, req *pb.ArmResumeNetworkRequest) (*pb.ArmResumeNetworkResponse, error) { + if s.resumeNetwork == nil { + return nil, status.Error(codes.FailedPrecondition, "resume network prearm loop is not enabled") + } + + var interval time.Duration + if req.PollIntervalMs > 0 { + interval = time.Duration(req.PollIntervalMs) * time.Millisecond + } + if err := s.resumeNetwork.Arm(ctx, interval); err != nil { + return nil, err + } + return &pb.ArmResumeNetworkResponse{}, nil +} + +func resumeNetworkVMGenIDLoop(s *guestServer, port uint32) { + log.Printf("[guest-agent] resume network waiting for VMGenID signal on host port %d", port) + for { + if err := waitForVMGenIDResumeSignal(); err != nil { + log.Printf("[guest-agent] resume network VMGenID wait failed: %v", err) + time.Sleep(100 * time.Millisecond) + continue + } + + start := time.Now() + if err := fetchAndApplyResumeNetwork(s, port, 50*time.Millisecond); err != nil { + log.Printf("[guest-agent] resume network VMGenID apply failed: %v", err) + continue + } + log.Printf("[guest-agent] resume network VMGenID applied in %s", time.Since(start)) + } +} + +func waitForVMGenIDResumeSignal() error { + waiter, err := newVMGenIDResumeWaiter() + if err != nil { + return err + } + defer waiter.Close() + return waiter.Wait() +} + +func newVMGenIDResumeWaiter() (*vmGenIDResumeWaiter, error) { + f, err := os.Open("/dev/kmsg") + if err != nil { + return nil, fmt.Errorf("open /dev/kmsg: %w", err) + } + + if _, err := f.Seek(0, io.SeekEnd); err != nil { + log.Printf("[guest-agent] warning: failed to seek /dev/kmsg to end: %v", err) + } + + return &vmGenIDResumeWaiter{ + file: f, + reader: bufio.NewReader(f), + }, nil +} + +func (w *vmGenIDResumeWaiter) Close() { + if w == nil || w.file == nil { + return + } + _ = w.file.Close() +} + +func (w *vmGenIDResumeWaiter) Wait() error { + for { + line, err := w.reader.ReadString('\n') + if err != nil { + return fmt.Errorf("read /dev/kmsg: %w", err) + } + if strings.Contains(line, vmgenIDKmsgSignal) { + return nil + } + } +} + +func armResumeNetworkConnection(port uint32) error { + conn, err := dialHost(port, 5*time.Second) + if err != nil { + return fmt.Errorf("dial host port %d: %w", port, err) + } + defer conn.Close() + + if _, err := conn.Write([]byte("HELLO\n")); err != nil { + return fmt.Errorf("write hello: %w", err) + } + log.Printf("[guest-agent] resume network watcher armed on host port %d", port) + + var buf [1]byte + _, err = conn.Read(buf[:]) + if err != nil { + log.Printf("[guest-agent] resume network watcher woke: %v", err) + return nil + } + return nil +} + +func fetchAndApplyResumeNetwork(s *guestServer, port uint32, dialTimeout time.Duration) error { + conn, err := dialHost(port, dialTimeout) + if err != nil { + return fmt.Errorf("dial host config port %d: %w", port, err) + } + defer conn.Close() + + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + if _, err := conn.Write([]byte("FETCH\n")); err != nil { + return fmt.Errorf("write fetch: %w", err) + } + + var payload resumeNetworkPayload + if err := json.NewDecoder(conn).Decode(&payload); err != nil { + return fmt.Errorf("decode payload: %w", err) + } + + applyStart := time.Now() + _, err = s.ReconfigureNetwork(context.Background(), &pb.ReconfigureNetworkRequest{ + InterfaceName: payload.InterfaceName, + Mac: payload.MAC, + Ipv4: payload.IPv4, + Prefix: payload.Prefix, + Gateway: payload.Gateway, + }) + applyElapsed := time.Since(applyStart) + if err != nil { + _, _ = fmt.Fprintf(conn, "ERR %s\n", err) + return err + } + sendResumeNetworkAck(payload, "applied") + + writer := bufio.NewWriter(conn) + _, _ = fmt.Fprintf(writer, "OK apply_ms=%d\n", applyElapsed.Milliseconds()) + _ = writer.Flush() + return nil +} + +func dialHost(port uint32, timeout time.Duration) (*socket.Conn, error) { + fd, err := unix.Socket(unix.AF_VSOCK, unix.SOCK_STREAM|unix.SOCK_CLOEXEC|unix.SOCK_NONBLOCK, 0) + if err != nil { + return nil, err + } + + if err := unix.Connect(fd, &unix.SockaddrVM{CID: resumeNetworkHostCID, Port: port}); err != nil { + if !errors.Is(err, unix.EINPROGRESS) && !errors.Is(err, unix.EAGAIN) { + _ = unix.Close(fd) + return nil, err + } + if err := waitVsockConnect(fd, timeout); err != nil { + _ = unix.Close(fd) + return nil, err + } + } + + return socket.New(fd, "vsock") +} + +func sendResumeNetworkAck(payload resumeNetworkPayload, stage string) { + if payload.AckPort == 0 || payload.Gateway == "" { + return + } + + addr := net.JoinHostPort(payload.Gateway, strconv.FormatUint(uint64(payload.AckPort), 10)) + conn, err := net.DialTimeout("udp4", addr, 100*time.Millisecond) + if err != nil { + log.Printf("[guest-agent] resume network ack dial failed: %v", err) + return + } + defer conn.Close() + + _, _ = fmt.Fprintf(conn, "stage=%s mac=%s ip=%s\n", stage, payload.MAC, payload.IPv4) +} + +func waitVsockConnect(fd int, timeout time.Duration) error { + timeoutMS := int(timeout.Milliseconds()) + if timeout > 0 && timeoutMS <= 0 { + timeoutMS = 1 + } + ready, err := unix.Poll([]unix.PollFd{{ + Fd: int32(fd), + Events: unix.POLLOUT, + }}, timeoutMS) + if err != nil { + return err + } + if ready == 0 { + return context.DeadlineExceeded + } + + errno, err := unix.GetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_ERROR) + if err != nil { + return err + } + if errno != 0 { + return unix.Errno(errno) + } + return nil +} diff --git a/lib/system/guest_agent/resume_network_other.go b/lib/system/guest_agent/resume_network_other.go new file mode 100644 index 00000000..dfa74e07 --- /dev/null +++ b/lib/system/guest_agent/resume_network_other.go @@ -0,0 +1,7 @@ +//go:build !linux + +package main + +type resumeNetworkController struct{} + +func startResumeNetworkWatcher(_ *guestServer) {} diff --git a/lib/system/versions.go b/lib/system/versions.go index aa07108d..7743add0 100644 --- a/lib/system/versions.go +++ b/lib/system/versions.go @@ -20,14 +20,18 @@ const ( // Kernel_202603301 is the current kernel version with expanded nftables/raw support for Docker bridge networking Kernel_202603301 KernelVersion = "ch-6.12.8-kernel-1.6-202603301" + + // Kernel_202605291 is the current kernel version with VMGenID support for snapshot resume detection + Kernel_202605291 KernelVersion = "ch-6.12.8-kernel-3.0-202605291" ) var ( // DefaultKernelVersion is the kernel version used for new instances - DefaultKernelVersion = Kernel_202603301 + DefaultKernelVersion = Kernel_202605291 // SupportedKernelVersions lists all supported kernel versions SupportedKernelVersions = []KernelVersion{ + Kernel_202605291, Kernel_202603301, Kernel_202603091, Kernel_202602101, @@ -37,6 +41,10 @@ var ( // KernelDownloadURLs maps kernel versions and architectures to download URLs var KernelDownloadURLs = map[KernelVersion]map[string]string{ + Kernel_202605291: { + "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-3.0-202605291/vmlinux-x86_64", + "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-3.0-202605291/Image-arm64", + }, Kernel_202603301: { "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.6-202603301/vmlinux-x86_64", "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.6-202603301/Image-arm64", @@ -58,6 +66,10 @@ var KernelDownloadURLs = map[KernelVersion]map[string]string{ // KernelHeaderURLs maps kernel versions and architectures to kernel header tarball URLs // These tarballs contain kernel headers needed for DKMS to build out-of-tree modules (e.g., NVIDIA vGPU drivers) var KernelHeaderURLs = map[KernelVersion]map[string]string{ + Kernel_202605291: { + "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-3.0-202605291/kernel-headers-x86_64.tar.gz", + "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-3.0-202605291/kernel-headers-aarch64.tar.gz", + }, Kernel_202603301: { "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.6-202603301/kernel-headers-x86_64.tar.gz", "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.6-202603301/kernel-headers-aarch64.tar.gz", diff --git a/openapi.yaml b/openapi.yaml index c6bce872..e6d03230 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -690,6 +690,13 @@ components: Optional final state for the forked instance. Default is the source instance state at fork time. For example, forking from Running defaults the fork result to Running. + wait_for_network: + type: boolean + description: | + When the fork result is Running, wait for guest networking to be applied before returning. + Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. + default: true + example: true ForkTargetState: type: string @@ -875,6 +882,13 @@ components: Optional hypervisor override. Allowed only when forking from a Stopped snapshot. Standby snapshots must fork with their original hypervisor. example: cloud-hypervisor + wait_for_network: + type: boolean + description: | + When the fork result is Running, wait for guest networking to be applied before returning. + Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. + default: true + example: true SnapshotScheduleRetention: type: object