From 87821dd59c49cb78357a52214a93c682c73a94b4 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sat, 30 May 2026 19:29:47 +0000 Subject: [PATCH 01/16] Add mailbox resume network handoff --- cmd/api/api/instances.go | 7 +- cmd/api/api/instances_test.go | 6 +- cmd/api/api/snapshots.go | 5 +- cmd/api/api/snapshots_test.go | 61 +++ lib/forkvm/README.md | 18 + lib/guest/client.go | 107 +++++ lib/guest/guest.pb.go | 188 ++++++-- lib/guest/guest.proto | 15 + lib/guest/guest_grpc.pb.go | 52 ++- lib/instances/fork.go | 13 +- lib/instances/fork_test.go | 2 +- lib/instances/guest_resume_network.go | 246 ++++++++++ lib/instances/manager.go | 8 +- lib/instances/restore.go | 126 +++++- lib/instances/restore_egress_test.go | 64 ++- lib/instances/snapshot.go | 8 +- lib/instances/types.go | 8 +- lib/oapi/oapi.go | 424 +++++++++--------- lib/system/guest_agent/main.go | 4 +- lib/system/guest_agent/network.go | 91 ++++ lib/system/guest_agent/resume_network.go | 201 +++++++++ .../guest_agent/resume_network_other.go | 7 + lib/system/versions.go | 14 +- openapi.yaml | 14 + 24 files changed, 1401 insertions(+), 288 deletions(-) create mode 100644 lib/instances/guest_resume_network.go create mode 100644 lib/system/guest_agent/network.go create mode 100644 lib/system/guest_agent/resume_network.go create mode 100644 lib/system/guest_agent/resume_network_other.go diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 2c85284b..9d59515d 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -657,9 +657,10 @@ func (s *ApiService) ForkInstance(ctx context.Context, request oapi.ForkInstance } result, err := s.InstanceManager.ForkInstance(ctx, inst.Id, instances.ForkInstanceRequest{ - Name: request.Body.Name, - FromRunning: request.Body.FromRunning != nil && *request.Body.FromRunning, - TargetState: targetState, + Name: request.Body.Name, + FromRunning: request.Body.FromRunning != nil && *request.Body.FromRunning, + TargetState: targetState, + WaitForNetwork: request.Body.WaitForNetwork, }) if err != nil { switch { diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 4e7863aa..896a0632 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -1165,13 +1165,15 @@ func TestForkInstance_Success(t *testing.T) { result: forked, } svc.InstanceManager = mockMgr + waitForNetwork := false resp, err := svc.ForkInstance( mw.WithResolvedInstance(ctx(), source.Id, source), oapi.ForkInstanceRequestObject{ Id: source.Id, Body: &oapi.ForkInstanceRequest{ - Name: "forked-instance", + Name: "forked-instance", + WaitForNetwork: &waitForNetwork, }, }, ) @@ -1185,6 +1187,8 @@ func TestForkInstance_Success(t *testing.T) { assert.Equal(t, "forked-instance", mockMgr.lastReq.Name) assert.False(t, mockMgr.lastReq.FromRunning) assert.Equal(t, instances.State(""), mockMgr.lastReq.TargetState) + require.NotNil(t, mockMgr.lastReq.WaitForNetwork) + assert.False(t, *mockMgr.lastReq.WaitForNetwork) } func TestForkInstance_NotSupported(t *testing.T) { diff --git a/cmd/api/api/snapshots.go b/cmd/api/api/snapshots.go index 620d9574..ffe35474 100644 --- a/cmd/api/api/snapshots.go +++ b/cmd/api/api/snapshots.go @@ -175,7 +175,10 @@ func (s *ApiService) ForkSnapshot(ctx context.Context, request oapi.ForkSnapshot return oapi.ForkSnapshot400JSONResponse{Code: "invalid_request", Message: "request body is required"}, nil } - domainReq := instances.ForkSnapshotRequest{Name: request.Body.Name} + domainReq := instances.ForkSnapshotRequest{ + Name: request.Body.Name, + WaitForNetwork: request.Body.WaitForNetwork, + } if request.Body.TargetState != nil { domainReq.TargetState = instances.State(*request.Body.TargetState) } diff --git a/cmd/api/api/snapshots_test.go b/cmd/api/api/snapshots_test.go index d0730133..f87e9148 100644 --- a/cmd/api/api/snapshots_test.go +++ b/cmd/api/api/snapshots_test.go @@ -1,14 +1,35 @@ package api import ( + "context" "testing" "time" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/oapi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +type captureForkSnapshotManager struct { + instances.Manager + lastID string + lastReq *instances.ForkSnapshotRequest + result *instances.Instance + err error +} + +func (m *captureForkSnapshotManager) ForkSnapshot(ctx context.Context, snapshotID string, req instances.ForkSnapshotRequest) (*instances.Instance, error) { + reqCopy := req + m.lastID = snapshotID + m.lastReq = &reqCopy + if m.err != nil { + return nil, m.err + } + return m.result, nil +} + func TestSnapshotScheduleToOAPIPreservesZeroMaxCount(t *testing.T) { t.Parallel() @@ -30,3 +51,43 @@ func TestSnapshotScheduleToOAPIPreservesZeroMaxCount(t *testing.T) { require.NotNil(t, out.Retention.MaxAge) assert.Equal(t, "24h0m0s", *out.Retention.MaxAge) } + +func TestForkSnapshotMapsWaitForNetwork(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + forked := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "forked-instance", + Name: "forked-instance", + Image: "docker.io/library/alpine:latest", + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeFirecracker, + }, + State: instances.StateRunning, + } + mockMgr := &captureForkSnapshotManager{ + Manager: svc.InstanceManager, + result: forked, + } + svc.InstanceManager = mockMgr + waitForNetwork := false + + resp, err := svc.ForkSnapshot(ctx(), oapi.ForkSnapshotRequestObject{ + SnapshotId: "snap-123", + Body: &oapi.ForkSnapshotRequest{ + Name: "forked-instance", + WaitForNetwork: &waitForNetwork, + }, + }) + require.NoError(t, err) + + created, ok := resp.(oapi.ForkSnapshot201JSONResponse) + require.True(t, ok, "expected 201 response") + assert.Equal(t, "forked-instance", created.Name) + assert.Equal(t, "snap-123", mockMgr.lastID) + require.NotNil(t, mockMgr.lastReq) + assert.Equal(t, "forked-instance", mockMgr.lastReq.Name) + require.NotNil(t, mockMgr.lastReq.WaitForNetwork) + assert.False(t, *mockMgr.lastReq.WaitForNetwork) +} diff --git a/lib/forkvm/README.md b/lib/forkvm/README.md index bb6567c7..88474c72 100644 --- a/lib/forkvm/README.md +++ b/lib/forkvm/README.md @@ -14,6 +14,24 @@ to work across implementations. For networked forks, the fork gets a fresh host/guest identity (IP, MAC, TAP) instead of reusing the source identity. +## Resume network handoff + +Networked standby/running forks need a new host-side allocation, but the guest +memory snapshot still contains the source VM's old interface state. On restore, +Hypeman prepares the fork's TAP/IP/MAC before the VM resumes, then hands the new +guest network config to the guest through a small mailbox embedded in snapshot +memory. After resume, VMGenID tells the guest-agent that this is a restored VM; +the guest-agent reads the mailbox and applies the new MAC, address, route, and +neighbor state with netlink. + +For API calls that return a running fork, `wait_for_network` defaults to true. +In that mode Hypeman waits for a guest UDP "applied" ack before returning, so +the fast path still avoids making host-initiated guest RPC/vsock contact as the +first post-resume dependency. If `wait_for_network=false`, the API returns after +resume once the mailbox has been patched and the guest finishes the network +handoff asynchronously. If the mailbox path is unavailable, restore falls back +to the older host-initiated guest network reconfigure path. + ## Fork data copy behavior - Guest directory copy is **sparse-only** for regular files. diff --git a/lib/guest/client.go b/lib/guest/client.go index 20ff8d0b..945f082c 100644 --- a/lib/guest/client.go +++ b/lib/guest/client.go @@ -82,8 +82,12 @@ func GetOrCreateConn(ctx context.Context, dialer hypervisor.VsockDialer) (*grpc. } // Create new connection using the VsockDialer + traceCtx := ctx conn, err := grpc.Dial("passthrough:///vsock", grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + if span := trace.SpanFromContext(traceCtx); span.SpanContext().IsValid() { + ctx = trace.ContextWithSpan(ctx, span) + } netConn, err := dialer.DialVsock(ctx, vsockGuestPort) if err != nil { return nil, &AgentVSockDialError{Err: err} @@ -146,6 +150,109 @@ type ExecOptions struct { ResizeChan <-chan *WindowSize // Optional: channel to receive resize events (pointer to avoid copying mutex) } +type ReconfigureNetworkOptions struct { + InterfaceName string + MAC string + IPv4 string + Prefix uint32 + Gateway string + WaitForAgent time.Duration +} + +func ReconfigureNetworkInInstance(ctx context.Context, dialer hypervisor.VsockDialer, opts ReconfigureNetworkOptions) error { + if opts.WaitForAgent == 0 { + return reconfigureNetworkOnce(ctx, dialer, opts) + } + + ctx, span := otel.Tracer("hypeman/guest").Start(ctx, "guest.reconfigure_network", trace.WithAttributes( + attribute.Bool("wait_for_agent", true), + attribute.Int64("wait_for_agent_ms", opts.WaitForAgent.Milliseconds()), + )) + defer span.End() + + deadline := time.Now().Add(opts.WaitForAgent) + start := time.Now() + attempts := 0 + retryableAttempts := 0 + firstRetryableErrorType := "" + lastRetryableErrorType := "" + lastRetryInterval := time.Duration(0) + + for { + attempts++ + err := reconfigureNetworkOnce(ctx, dialer, opts) + if err == nil { + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.SetStatus(otelcodes.Ok, "") + return nil + } + if !isRetryableConnectionError(err) { + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.RecordError(err) + span.SetStatus(otelcodes.Error, err.Error()) + return err + } + + retryableAttempts++ + errType := retryableConnectionErrorType(err) + if firstRetryableErrorType == "" { + firstRetryableErrorType = errType + } + lastRetryableErrorType = errType + CloseConn(dialer.Key()) + + if time.Now().After(deadline) { + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.RecordError(err) + span.SetStatus(otelcodes.Error, err.Error()) + return err + } + + retryInterval := guestExecRetryInterval(time.Since(start)) + lastRetryInterval = retryInterval + select { + case <-ctx.Done(): + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.RecordError(ctx.Err()) + span.SetStatus(otelcodes.Error, ctx.Err().Error()) + return ctx.Err() + case <-time.After(retryInterval): + } + } +} + +func reconfigureNetworkOnce(ctx context.Context, dialer hypervisor.VsockDialer, opts ReconfigureNetworkOptions) error { + grpcConn, err := GetOrCreateConn(ctx, dialer) + if err != nil { + return fmt.Errorf("get grpc connection: %w", err) + } + client := NewGuestServiceClient(grpcConn) + + _, span := otel.Tracer("hypeman/guest").Start(ctx, "guest.reconfigure_network.rpc") + _, err = client.ReconfigureNetwork(ctx, &ReconfigureNetworkRequest{ + InterfaceName: opts.InterfaceName, + Mac: opts.MAC, + Ipv4: opts.IPv4, + Prefix: opts.Prefix, + Gateway: opts.Gateway, + }) + finishGuestNetworkStepSpan(span, err) + if err != nil { + return fmt.Errorf("reconfigure network rpc: %w", err) + } + return nil +} + +func finishGuestNetworkStepSpan(span trace.Span, err error) { + if err != nil { + span.RecordError(err) + span.SetStatus(otelcodes.Error, err.Error()) + } else { + span.SetStatus(otelcodes.Ok, "") + } + span.End() +} + // ExecIntoInstance executes command in instance via vsock using gRPC. // The dialer is a hypervisor-specific VsockDialer that knows how to connect to the guest. // If WaitForAgent is set, it will retry on connection errors until the timeout. diff --git a/lib/guest/guest.pb.go b/lib/guest/guest.pb.go index 0de1628e..a239fc97 100644 --- a/lib/guest/guest.pb.go +++ b/lib/guest/guest.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.10 -// protoc v3.21.12 +// protoc v6.33.4 // source: lib/guest/guest.proto package guest @@ -1266,6 +1266,120 @@ func (*ShutdownResponse) Descriptor() ([]byte, []int) { return file_lib_guest_guest_proto_rawDescGZIP(), []int{16} } +// ReconfigureNetworkRequest updates a guest network interface after snapshot restore +type ReconfigureNetworkRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InterfaceName string `protobuf:"bytes,1,opt,name=interface_name,json=interfaceName,proto3" json:"interface_name,omitempty"` // Interface to reconfigure, defaults to eth0 + Mac string `protobuf:"bytes,2,opt,name=mac,proto3" json:"mac,omitempty"` // New MAC address + Ipv4 string `protobuf:"bytes,3,opt,name=ipv4,proto3" json:"ipv4,omitempty"` // New IPv4 address without prefix + Prefix uint32 `protobuf:"varint,4,opt,name=prefix,proto3" json:"prefix,omitempty"` // IPv4 prefix length + Gateway string `protobuf:"bytes,5,opt,name=gateway,proto3" json:"gateway,omitempty"` // Default gateway IPv4 address + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReconfigureNetworkRequest) Reset() { + *x = ReconfigureNetworkRequest{} + mi := &file_lib_guest_guest_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReconfigureNetworkRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReconfigureNetworkRequest) ProtoMessage() {} + +func (x *ReconfigureNetworkRequest) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReconfigureNetworkRequest.ProtoReflect.Descriptor instead. +func (*ReconfigureNetworkRequest) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{17} +} + +func (x *ReconfigureNetworkRequest) GetInterfaceName() string { + if x != nil { + return x.InterfaceName + } + return "" +} + +func (x *ReconfigureNetworkRequest) GetMac() string { + if x != nil { + return x.Mac + } + return "" +} + +func (x *ReconfigureNetworkRequest) GetIpv4() string { + if x != nil { + return x.Ipv4 + } + return "" +} + +func (x *ReconfigureNetworkRequest) GetPrefix() uint32 { + if x != nil { + return x.Prefix + } + return 0 +} + +func (x *ReconfigureNetworkRequest) GetGateway() string { + if x != nil { + return x.Gateway + } + return "" +} + +// ReconfigureNetworkResponse acknowledges the network reconfiguration request +type ReconfigureNetworkResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReconfigureNetworkResponse) Reset() { + *x = ReconfigureNetworkResponse{} + mi := &file_lib_guest_guest_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReconfigureNetworkResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReconfigureNetworkResponse) ProtoMessage() {} + +func (x *ReconfigureNetworkResponse) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReconfigureNetworkResponse.ProtoReflect.Descriptor instead. +func (*ReconfigureNetworkResponse) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{18} +} + var File_lib_guest_guest_proto protoreflect.FileDescriptor const file_lib_guest_guest_proto_rawDesc = "" + @@ -1358,13 +1472,21 @@ const file_lib_guest_guest_proto_rawDesc = "" + "\x05error\x18\b \x01(\tR\x05error\")\n" + "\x0fShutdownRequest\x12\x16\n" + "\x06signal\x18\x01 \x01(\x05R\x06signal\"\x12\n" + - "\x10ShutdownResponse2\xd3\x02\n" + + "\x10ShutdownResponse\"\x9a\x01\n" + + "\x19ReconfigureNetworkRequest\x12%\n" + + "\x0einterface_name\x18\x01 \x01(\tR\rinterfaceName\x12\x10\n" + + "\x03mac\x18\x02 \x01(\tR\x03mac\x12\x12\n" + + "\x04ipv4\x18\x03 \x01(\tR\x04ipv4\x12\x16\n" + + "\x06prefix\x18\x04 \x01(\rR\x06prefix\x12\x18\n" + + "\agateway\x18\x05 \x01(\tR\agateway\"\x1c\n" + + "\x1aReconfigureNetworkResponse2\xae\x03\n" + "\fGuestService\x123\n" + "\x04Exec\x12\x12.guest.ExecRequest\x1a\x13.guest.ExecResponse(\x010\x01\x12F\n" + "\vCopyToGuest\x12\x19.guest.CopyToGuestRequest\x1a\x1a.guest.CopyToGuestResponse(\x01\x12L\n" + "\rCopyFromGuest\x12\x1b.guest.CopyFromGuestRequest\x1a\x1c.guest.CopyFromGuestResponse0\x01\x12;\n" + "\bStatPath\x12\x16.guest.StatPathRequest\x1a\x17.guest.StatPathResponse\x12;\n" + - "\bShutdown\x12\x16.guest.ShutdownRequest\x1a\x17.guest.ShutdownResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" + "\bShutdown\x12\x16.guest.ShutdownRequest\x1a\x17.guest.ShutdownResponse\x12Y\n" + + "\x12ReconfigureNetwork\x12 .guest.ReconfigureNetworkRequest\x1a!.guest.ReconfigureNetworkResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" var ( file_lib_guest_guest_proto_rawDescOnce sync.Once @@ -1378,31 +1500,33 @@ func file_lib_guest_guest_proto_rawDescGZIP() []byte { return file_lib_guest_guest_proto_rawDescData } -var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 20) var file_lib_guest_guest_proto_goTypes = []any{ - (*ExecRequest)(nil), // 0: guest.ExecRequest - (*ExecStart)(nil), // 1: guest.ExecStart - (*WindowSize)(nil), // 2: guest.WindowSize - (*ExecResponse)(nil), // 3: guest.ExecResponse - (*CopyToGuestRequest)(nil), // 4: guest.CopyToGuestRequest - (*CopyToGuestStart)(nil), // 5: guest.CopyToGuestStart - (*CopyToGuestEnd)(nil), // 6: guest.CopyToGuestEnd - (*CopyToGuestResponse)(nil), // 7: guest.CopyToGuestResponse - (*CopyFromGuestRequest)(nil), // 8: guest.CopyFromGuestRequest - (*CopyFromGuestResponse)(nil), // 9: guest.CopyFromGuestResponse - (*CopyFromGuestHeader)(nil), // 10: guest.CopyFromGuestHeader - (*CopyFromGuestEnd)(nil), // 11: guest.CopyFromGuestEnd - (*CopyFromGuestError)(nil), // 12: guest.CopyFromGuestError - (*StatPathRequest)(nil), // 13: guest.StatPathRequest - (*StatPathResponse)(nil), // 14: guest.StatPathResponse - (*ShutdownRequest)(nil), // 15: guest.ShutdownRequest - (*ShutdownResponse)(nil), // 16: guest.ShutdownResponse - nil, // 17: guest.ExecStart.EnvEntry + (*ExecRequest)(nil), // 0: guest.ExecRequest + (*ExecStart)(nil), // 1: guest.ExecStart + (*WindowSize)(nil), // 2: guest.WindowSize + (*ExecResponse)(nil), // 3: guest.ExecResponse + (*CopyToGuestRequest)(nil), // 4: guest.CopyToGuestRequest + (*CopyToGuestStart)(nil), // 5: guest.CopyToGuestStart + (*CopyToGuestEnd)(nil), // 6: guest.CopyToGuestEnd + (*CopyToGuestResponse)(nil), // 7: guest.CopyToGuestResponse + (*CopyFromGuestRequest)(nil), // 8: guest.CopyFromGuestRequest + (*CopyFromGuestResponse)(nil), // 9: guest.CopyFromGuestResponse + (*CopyFromGuestHeader)(nil), // 10: guest.CopyFromGuestHeader + (*CopyFromGuestEnd)(nil), // 11: guest.CopyFromGuestEnd + (*CopyFromGuestError)(nil), // 12: guest.CopyFromGuestError + (*StatPathRequest)(nil), // 13: guest.StatPathRequest + (*StatPathResponse)(nil), // 14: guest.StatPathResponse + (*ShutdownRequest)(nil), // 15: guest.ShutdownRequest + (*ShutdownResponse)(nil), // 16: guest.ShutdownResponse + (*ReconfigureNetworkRequest)(nil), // 17: guest.ReconfigureNetworkRequest + (*ReconfigureNetworkResponse)(nil), // 18: guest.ReconfigureNetworkResponse + nil, // 19: guest.ExecStart.EnvEntry } var file_lib_guest_guest_proto_depIdxs = []int32{ 1, // 0: guest.ExecRequest.start:type_name -> guest.ExecStart 2, // 1: guest.ExecRequest.resize:type_name -> guest.WindowSize - 17, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry + 19, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry 5, // 3: guest.CopyToGuestRequest.start:type_name -> guest.CopyToGuestStart 6, // 4: guest.CopyToGuestRequest.end:type_name -> guest.CopyToGuestEnd 10, // 5: guest.CopyFromGuestResponse.header:type_name -> guest.CopyFromGuestHeader @@ -1413,13 +1537,15 @@ var file_lib_guest_guest_proto_depIdxs = []int32{ 8, // 10: guest.GuestService.CopyFromGuest:input_type -> guest.CopyFromGuestRequest 13, // 11: guest.GuestService.StatPath:input_type -> guest.StatPathRequest 15, // 12: guest.GuestService.Shutdown:input_type -> guest.ShutdownRequest - 3, // 13: guest.GuestService.Exec:output_type -> guest.ExecResponse - 7, // 14: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse - 9, // 15: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse - 14, // 16: guest.GuestService.StatPath:output_type -> guest.StatPathResponse - 16, // 17: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse - 13, // [13:18] is the sub-list for method output_type - 8, // [8:13] is the sub-list for method input_type + 17, // 13: guest.GuestService.ReconfigureNetwork:input_type -> guest.ReconfigureNetworkRequest + 3, // 14: guest.GuestService.Exec:output_type -> guest.ExecResponse + 7, // 15: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse + 9, // 16: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse + 14, // 17: guest.GuestService.StatPath:output_type -> guest.StatPathResponse + 16, // 18: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse + 18, // 19: guest.GuestService.ReconfigureNetwork:output_type -> guest.ReconfigureNetworkResponse + 14, // [14:20] is the sub-list for method output_type + 8, // [8:14] is the sub-list for method input_type 8, // [8:8] is the sub-list for extension type_name 8, // [8:8] is the sub-list for extension extendee 0, // [0:8] is the sub-list for field type_name @@ -1457,7 +1583,7 @@ func file_lib_guest_guest_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_lib_guest_guest_proto_rawDesc), len(file_lib_guest_guest_proto_rawDesc)), NumEnums: 0, - NumMessages: 18, + NumMessages: 20, NumExtensions: 0, NumServices: 1, }, diff --git a/lib/guest/guest.proto b/lib/guest/guest.proto index c42198a9..317c21b3 100644 --- a/lib/guest/guest.proto +++ b/lib/guest/guest.proto @@ -20,6 +20,9 @@ service GuestService { // Shutdown requests graceful VM shutdown by signaling init (PID 1) rpc Shutdown(ShutdownRequest) returns (ShutdownResponse); + + // ReconfigureNetwork updates the guest network identity without spawning shell commands + rpc ReconfigureNetwork(ReconfigureNetworkRequest) returns (ReconfigureNetworkResponse); } // ExecRequest represents messages from client to server @@ -154,3 +157,15 @@ message ShutdownRequest { // ShutdownResponse acknowledges the shutdown request message ShutdownResponse {} + +// ReconfigureNetworkRequest updates a guest network interface after snapshot restore +message ReconfigureNetworkRequest { + string interface_name = 1; // Interface to reconfigure, defaults to eth0 + string mac = 2; // New MAC address + string ipv4 = 3; // New IPv4 address without prefix + uint32 prefix = 4; // IPv4 prefix length + string gateway = 5; // Default gateway IPv4 address +} + +// ReconfigureNetworkResponse acknowledges the network reconfiguration request +message ReconfigureNetworkResponse {} diff --git a/lib/guest/guest_grpc.pb.go b/lib/guest/guest_grpc.pb.go index 71224327..f93631d9 100644 --- a/lib/guest/guest_grpc.pb.go +++ b/lib/guest/guest_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.6.0 -// - protoc v3.21.12 +// - protoc v6.33.4 // source: lib/guest/guest.proto package guest @@ -19,11 +19,12 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - GuestService_Exec_FullMethodName = "/guest.GuestService/Exec" - GuestService_CopyToGuest_FullMethodName = "/guest.GuestService/CopyToGuest" - GuestService_CopyFromGuest_FullMethodName = "/guest.GuestService/CopyFromGuest" - GuestService_StatPath_FullMethodName = "/guest.GuestService/StatPath" - GuestService_Shutdown_FullMethodName = "/guest.GuestService/Shutdown" + GuestService_Exec_FullMethodName = "/guest.GuestService/Exec" + GuestService_CopyToGuest_FullMethodName = "/guest.GuestService/CopyToGuest" + GuestService_CopyFromGuest_FullMethodName = "/guest.GuestService/CopyFromGuest" + GuestService_StatPath_FullMethodName = "/guest.GuestService/StatPath" + GuestService_Shutdown_FullMethodName = "/guest.GuestService/Shutdown" + GuestService_ReconfigureNetwork_FullMethodName = "/guest.GuestService/ReconfigureNetwork" ) // GuestServiceClient is the client API for GuestService service. @@ -42,6 +43,8 @@ type GuestServiceClient interface { StatPath(ctx context.Context, in *StatPathRequest, opts ...grpc.CallOption) (*StatPathResponse, error) // Shutdown requests graceful VM shutdown by signaling init (PID 1) Shutdown(ctx context.Context, in *ShutdownRequest, opts ...grpc.CallOption) (*ShutdownResponse, error) + // ReconfigureNetwork updates the guest network identity without spawning shell commands + ReconfigureNetwork(ctx context.Context, in *ReconfigureNetworkRequest, opts ...grpc.CallOption) (*ReconfigureNetworkResponse, error) } type guestServiceClient struct { @@ -117,6 +120,16 @@ func (c *guestServiceClient) Shutdown(ctx context.Context, in *ShutdownRequest, return out, nil } +func (c *guestServiceClient) ReconfigureNetwork(ctx context.Context, in *ReconfigureNetworkRequest, opts ...grpc.CallOption) (*ReconfigureNetworkResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ReconfigureNetworkResponse) + err := c.cc.Invoke(ctx, GuestService_ReconfigureNetwork_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // GuestServiceServer is the server API for GuestService service. // All implementations must embed UnimplementedGuestServiceServer // for forward compatibility. @@ -133,6 +146,8 @@ type GuestServiceServer interface { StatPath(context.Context, *StatPathRequest) (*StatPathResponse, error) // Shutdown requests graceful VM shutdown by signaling init (PID 1) Shutdown(context.Context, *ShutdownRequest) (*ShutdownResponse, error) + // ReconfigureNetwork updates the guest network identity without spawning shell commands + ReconfigureNetwork(context.Context, *ReconfigureNetworkRequest) (*ReconfigureNetworkResponse, error) mustEmbedUnimplementedGuestServiceServer() } @@ -158,6 +173,9 @@ func (UnimplementedGuestServiceServer) StatPath(context.Context, *StatPathReques func (UnimplementedGuestServiceServer) Shutdown(context.Context, *ShutdownRequest) (*ShutdownResponse, error) { return nil, status.Error(codes.Unimplemented, "method Shutdown not implemented") } +func (UnimplementedGuestServiceServer) ReconfigureNetwork(context.Context, *ReconfigureNetworkRequest) (*ReconfigureNetworkResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ReconfigureNetwork not implemented") +} func (UnimplementedGuestServiceServer) mustEmbedUnimplementedGuestServiceServer() {} func (UnimplementedGuestServiceServer) testEmbeddedByValue() {} @@ -240,6 +258,24 @@ func _GuestService_Shutdown_Handler(srv interface{}, ctx context.Context, dec fu return interceptor(ctx, in, info, handler) } +func _GuestService_ReconfigureNetwork_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReconfigureNetworkRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GuestServiceServer).ReconfigureNetwork(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: GuestService_ReconfigureNetwork_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GuestServiceServer).ReconfigureNetwork(ctx, req.(*ReconfigureNetworkRequest)) + } + return interceptor(ctx, in, info, handler) +} + // GuestService_ServiceDesc is the grpc.ServiceDesc for GuestService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -255,6 +291,10 @@ var GuestService_ServiceDesc = grpc.ServiceDesc{ MethodName: "Shutdown", Handler: _GuestService_Shutdown_Handler, }, + { + MethodName: "ReconfigureNetwork", + Handler: _GuestService_ReconfigureNetwork_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/lib/instances/fork.go b/lib/instances/fork.go index b4af9434..2c61fa25 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -81,7 +81,9 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR // the source data directory. Restore the fork while source remains standby and // under lock, then restore the source. if forkErr == nil && targetState == StateRunning { - restoredFork, err := m.applyForkTargetState(ctx, forked.Id, StateRunning) + restoredFork, err := m.applyForkTargetState(ctx, forked.Id, StateRunning, restoreInstanceOptions{ + WaitForGuestNetwork: req.WaitForNetwork, + }) if err != nil { forkErr = fmt.Errorf("restore forked instance before source restore: %w", err) if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { @@ -93,7 +95,7 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR } log.InfoContext(ctx, "restoring source instance after running fork", "source_instance_id", id) - restoredSource, restoreErr := m.restoreInstance(ctx, id) + restoredSource, restoreErr := m.restoreInstance(ctx, id, restoreInstanceOptions{}) if restoreErr != nil { if forkErr != nil { @@ -401,7 +403,7 @@ func resolveForkTargetState(requested State, sourceState State) (State, error) { return requested, nil } -func (m *manager) applyForkTargetState(ctx context.Context, forkID string, target State) (*Instance, error) { +func (m *manager) applyForkTargetState(ctx context.Context, forkID string, target State, restoreOpts restoreInstanceOptions) (*Instance, error) { lock := m.getInstanceLock(forkID) lock.Lock() defer lock.Unlock() @@ -411,6 +413,9 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe return nil, err } if inst != nil && (inst.State == StateRunning || inst.State == StateInitializing) { + if guestInitiatedResumeNetworkMailbox(&inst.StoredMetadata) { + return inst, nil + } if err := ensureGuestAgentReadyForForkPhase(ctx, &inst.StoredMetadata, "before returning running fork instance"); err != nil { return nil, fmt.Errorf("wait for forked guest agent readiness: %w", err) } @@ -440,7 +445,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe case StateStandby: switch target { case StateRunning: - return returnWithReadiness(m.restoreInstance(ctx, forkID)) + return returnWithReadiness(m.restoreInstance(ctx, forkID, restoreOpts)) case StateStopped: if err := os.RemoveAll(m.paths.InstanceSnapshotLatest(forkID)); err != nil { return nil, fmt.Errorf("remove fork snapshot: %w", err) diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 606b7671..f158a645 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -308,7 +308,7 @@ func TestApplyForkTargetStateStoppedRefreshesSnapshotForkCID(t *testing.T) { meta.StoredMetadata.Phases.Record(phasetracking.PhaseStandby, time.Now()) require.NoError(t, manager.saveMetadata(meta)) - inst, err := manager.applyForkTargetState(ctx, forkID, StateStopped) + inst, err := manager.applyForkTargetState(ctx, forkID, StateStopped, restoreInstanceOptions{}) require.NoError(t, err) require.Equal(t, StateStopped, inst.State) require.Equal(t, generateVsockCID(forkID), inst.VsockCID) diff --git a/lib/instances/guest_resume_network.go b/lib/instances/guest_resume_network.go new file mode 100644 index 00000000..354657b0 --- /dev/null +++ b/lib/instances/guest_resume_network.go @@ -0,0 +1,246 @@ +package instances + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "fmt" + stdnet "net" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/logger" + "go.opentelemetry.io/otel/attribute" + "golang.org/x/sys/unix" +) + +const guestResumeNetworkMailboxEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX" +const guestResumeNetworkMailboxTokenEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX_TOKEN" +const firecrackerSnapshotMemoryFile = "memory" + +const guestResumeNetworkMailboxSeqOffset = 64 +const guestResumeNetworkMailboxLengthOffset = 68 +const guestResumeNetworkMailboxPayloadOffset = 72 + +var guestResumeNetworkMailboxMagic = []byte("HYPEMAN_RESUME_NETWORK_MAILBOX_V1\x00") +var guestResumeNetworkMailboxOffsets sync.Map + +type guestResumeNetworkPayload struct { + InterfaceName string `json:"interface_name"` + MAC string `json:"mac"` + IPv4 string `json:"ipv4"` + Prefix uint32 `json:"prefix"` + Gateway string `json:"gateway"` + AckPort uint32 `json:"ack_port,omitempty"` +} + +type guestResumeNetworkUDPAck struct { + received time.Time + text string +} + +type guestResumeNetworkUDPWaiter struct { + conn *stdnet.UDPConn + ch chan guestResumeNetworkUDPAck +} + +func guestInitiatedResumeNetworkMailbox(stored *StoredMetadata) bool { + token := guestInitiatedResumeNetworkMailboxToken(stored) + return stored != nil && + stored.HypervisorType == hypervisor.TypeFirecracker && + strings.TrimSpace(stored.Env[guestResumeNetworkMailboxEnv]) == "1" && + token != "" && + len(token) <= guestResumeNetworkMailboxSeqOffset-len(guestResumeNetworkMailboxMagic) +} + +func guestInitiatedResumeNetworkMailboxToken(stored *StoredMetadata) string { + if stored == nil { + return "" + } + return strings.TrimSpace(stored.Env[guestResumeNetworkMailboxTokenEnv]) +} + +func newGuestResumeNetworkPayload(cfg *guestNetworkConfig) guestResumeNetworkPayload { + return guestResumeNetworkPayload{ + InterfaceName: "eth0", + MAC: cfg.mac, + IPv4: cfg.ip, + Prefix: uint32(cfg.prefix), + Gateway: cfg.gateway, + } +} + +func startGuestResumeNetworkUDPWaiter() (*guestResumeNetworkUDPWaiter, error) { + conn, err := stdnet.ListenUDP("udp4", &stdnet.UDPAddr{IP: stdnet.IPv4zero, Port: 0}) + if err != nil { + return nil, fmt.Errorf("listen for guest resume network UDP ack: %w", err) + } + + w := &guestResumeNetworkUDPWaiter{ + conn: conn, + ch: make(chan guestResumeNetworkUDPAck, 128), + } + go w.readLoop() + return w, nil +} + +func (w *guestResumeNetworkUDPWaiter) Port() uint32 { + if w == nil || w.conn == nil { + return 0 + } + return uint32(w.conn.LocalAddr().(*stdnet.UDPAddr).Port) +} + +func (w *guestResumeNetworkUDPWaiter) Close() { + if w == nil || w.conn == nil { + return + } + _ = w.conn.Close() +} + +func (w *guestResumeNetworkUDPWaiter) readLoop() { + buf := make([]byte, 1024) + for { + n, _, err := w.conn.ReadFromUDP(buf) + if err != nil { + return + } + w.ch <- guestResumeNetworkUDPAck{ + received: time.Now(), + text: strings.TrimSpace(string(buf[:n])), + } + } +} + +func (w *guestResumeNetworkUDPWaiter) WaitApplied(ctx context.Context, mac, ip string) (time.Duration, string, error) { + if w == nil { + return 0, "", fmt.Errorf("guest resume network UDP waiter is nil") + } + + start := time.Now() + wantMAC := "mac=" + strings.ToLower(mac) + wantIP := "ip=" + ip + for { + select { + case ack := <-w.ch: + text := strings.ToLower(ack.text) + if strings.Contains(text, "stage=applied") && strings.Contains(text, wantMAC) && strings.Contains(text, wantIP) { + return ack.received.Sub(start), ack.text, nil + } + case <-ctx.Done(): + return 0, "", ctx.Err() + } + } +} + +func (m *manager) waitForGuestResumeNetworkUDPAck(ctx context.Context, waiter *guestResumeNetworkUDPWaiter, stored *StoredMetadata, cfg *guestNetworkConfig) error { + if waiter == nil || cfg == nil { + return nil + } + + log := logger.FromContext(ctx) + waitCtx, waitSpanEnd := m.startLifecycleStep(ctx, "guest.resume_network.udp_ack_wait", + attribute.String("instance_id", stored.Id), + attribute.String("hypervisor", string(stored.HypervisorType)), + attribute.String("operation", "guest_resume_network_udp_ack_wait"), + ) + waitCtx, cancel := context.WithTimeout(waitCtx, 2*time.Second) + defer cancel() + + elapsed, ack, err := waiter.WaitApplied(waitCtx, cfg.mac, cfg.ip) + waitSpanEnd(err) + if err != nil { + return err + } + log.InfoContext(ctx, "guest resume network UDP ack received", "instance_id", stored.Id, "elapsed", elapsed, "ack", ack) + return nil +} + +func patchGuestResumeNetworkMailbox(snapshotDir, token string, payload *guestResumeNetworkPayload) error { + if token == "" { + return fmt.Errorf("resume network mailbox token is empty") + } + if len(token) > guestResumeNetworkMailboxSeqOffset-len(guestResumeNetworkMailboxMagic) { + return fmt.Errorf("resume network mailbox token is too long") + } + if payload == nil { + return fmt.Errorf("resume network mailbox payload is nil") + } + + payloadBytes, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("marshal resume network mailbox payload: %w", err) + } + if len(payloadBytes) > 4096-guestResumeNetworkMailboxPayloadOffset { + return fmt.Errorf("resume network mailbox payload too large: %d bytes", len(payloadBytes)) + } + + file, err := os.OpenFile(filepath.Join(snapshotDir, firecrackerSnapshotMemoryFile), os.O_RDWR, 0) + if err != nil { + return fmt.Errorf("open snapshot memory for resume network mailbox: %w", err) + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return fmt.Errorf("stat snapshot memory for resume network mailbox: %w", err) + } + if info.Size() <= 0 { + return fmt.Errorf("resume network mailbox memory file is empty") + } + + marker := make([]byte, 0, len(guestResumeNetworkMailboxMagic)+len(token)) + marker = append(marker, guestResumeNetworkMailboxMagic...) + marker = append(marker, []byte(token)...) + + idx, err := findGuestResumeNetworkMailbox(file, info.Size(), marker, token) + if err != nil { + return err + } + if idx+int64(guestResumeNetworkMailboxPayloadOffset)+int64(len(payloadBytes)) > info.Size() { + return fmt.Errorf("resume network mailbox marker is too close to end of memory file") + } + + if _, err := file.WriteAt(payloadBytes, idx+int64(guestResumeNetworkMailboxPayloadOffset)); err != nil { + return fmt.Errorf("write resume network mailbox payload: %w", err) + } + var u32 [4]byte + binary.LittleEndian.PutUint32(u32[:], uint32(len(payloadBytes))) + if _, err := file.WriteAt(u32[:], idx+int64(guestResumeNetworkMailboxLengthOffset)); err != nil { + return fmt.Errorf("write resume network mailbox payload length: %w", err) + } + binary.LittleEndian.PutUint32(u32[:], 1) + if _, err := file.WriteAt(u32[:], idx+int64(guestResumeNetworkMailboxSeqOffset)); err != nil { + return fmt.Errorf("write resume network mailbox sequence: %w", err) + } + return nil +} + +func findGuestResumeNetworkMailbox(file *os.File, size int64, marker []byte, token string) (int64, error) { + if cached, ok := guestResumeNetworkMailboxOffsets.Load(token); ok { + if offset, ok := cached.(int64); ok && offset >= 0 && offset+int64(len(marker)) <= size { + buf := make([]byte, len(marker)) + if _, err := file.ReadAt(buf, offset); err == nil && bytes.Equal(buf, marker) { + return offset, nil + } + } + } + + data, err := unix.Mmap(int(file.Fd()), 0, int(size), unix.PROT_READ, unix.MAP_SHARED) + if err != nil { + return 0, fmt.Errorf("mmap snapshot memory for resume network mailbox: %w", err) + } + defer unix.Munmap(data) + + idx := bytes.Index(data, marker) + if idx < 0 { + return 0, fmt.Errorf("resume network mailbox marker not found") + } + guestResumeNetworkMailboxOffsets.Store(token, int64(idx)) + return int64(idx), nil +} diff --git a/lib/instances/manager.go b/lib/instances/manager.go index abdd2165..c55ba93a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -389,14 +389,16 @@ func (m *manager) ForkInstance(ctx context.Context, id string, req ForkInstanceR return nil, err } - inst, err := m.applyForkTargetState(ctx, forked.Id, targetState) + inst, err := m.applyForkTargetState(ctx, forked.Id, targetState, restoreInstanceOptions{ + WaitForGuestNetwork: req.WaitForNetwork, + }) if err != nil { if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { return nil, fmt.Errorf("apply fork target state: %w; additionally failed to cleanup forked instance %s: %v", err, forked.Id, cleanupErr) } return nil, fmt.Errorf("apply fork target state: %w", err) } - if inst.State == StateRunning { + if inst.State == StateRunning && !guestInitiatedResumeNetworkMailbox(&inst.StoredMetadata) { if err := ensureGuestAgentReadyForForkPhase(ctx, &inst.StoredMetadata, "before returning running fork instance"); err != nil { if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { return nil, fmt.Errorf("wait for fork guest agent readiness: %w; additionally failed to cleanup forked instance %s: %v", err, forked.Id, cleanupErr) @@ -449,7 +451,7 @@ func (m *manager) RestoreInstance(ctx context.Context, id string) (*Instance, er if current.State == StateRunning || current.State == StateInitializing { return current, nil } - inst, err := m.restoreInstance(ctx, id) + inst, err := m.restoreInstance(ctx, id, restoreInstanceOptions{}) if err == nil { m.notifyLifecycleEvent(ctx, LifecycleEventRestore, inst) } diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 9f1a3034..714f319e 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -16,14 +16,24 @@ import ( "github.com/kernel/hypeman/lib/network" snapshotstore "github.com/kernel/hypeman/lib/snapshot" "go.opentelemetry.io/otel/attribute" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) -// RestoreInstance restores an instance from standby -// Multi-hop orchestration: Standby → Paused → Running +type restoreInstanceOptions struct { + WaitForGuestNetwork *bool +} + +func (o restoreInstanceOptions) waitForGuestNetwork() bool { + return o.WaitForGuestNetwork == nil || *o.WaitForGuestNetwork +} + +// RestoreInstance restores an instance from standby. +// Multi-hop orchestration: Standby → Paused → Running. func (m *manager) restoreInstance( ctx context.Context, - id string, + opts restoreInstanceOptions, ) (_ *Instance, retErr error) { start := time.Now() log := logger.FromContext(ctx) @@ -44,6 +54,7 @@ func (m *manager) restoreInstance( inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata + waitForGuestNetwork := opts.waitForGuestNetwork() ctx = enrichInstancesTrace(ctx, attribute.String("hypervisor", string(stored.HypervisorType))) log.DebugContext(ctx, "loaded instance", "instance_id", id, "state", inst.State, "has_snapshot", inst.HasSnapshot) @@ -227,6 +238,42 @@ func (m *manager) restoreInstance( proxyRegistered = true } + var resumeNetworkAckWaiter *guestResumeNetworkUDPWaiter + var resumeNetworkAckCfg *guestNetworkConfig + resumeNetworkMailboxPatched := false + if allocatedNet != nil && !stored.SkipGuestAgent && guestInitiatedResumeNetworkMailbox(stored) { + resumeNetworkCfg, cfgErr := guestNetworkReconfigureConfig(allocatedNet) + if cfgErr != nil { + log.WarnContext(ctx, "failed to build guest resume network mailbox payload; falling back to host-initiated reconfigure", "instance_id", id, "error", cfgErr) + } else { + payload := newGuestResumeNetworkPayload(resumeNetworkCfg) + if waitForGuestNetwork { + var waitErr error + resumeNetworkAckWaiter, waitErr = startGuestResumeNetworkUDPWaiter() + if waitErr != nil { + log.ErrorContext(ctx, "failed to start guest resume network UDP ack waiter", "instance_id", id, "error", waitErr) + releaseNetwork() + return nil, fmt.Errorf("start guest resume network UDP ack waiter: %w", waitErr) + } + resumeNetworkAckCfg = resumeNetworkCfg + payload.AckPort = resumeNetworkAckWaiter.Port() + } + if patchErr := patchGuestResumeNetworkMailbox(snapshotDir, guestInitiatedResumeNetworkMailboxToken(stored), &payload); patchErr != nil { + if resumeNetworkAckWaiter != nil { + resumeNetworkAckWaiter.Close() + resumeNetworkAckWaiter = nil + resumeNetworkAckCfg = nil + } + log.WarnContext(ctx, "failed to patch guest resume network mailbox; falling back to host-initiated reconfigure", "instance_id", id, "error", patchErr) + } else { + resumeNetworkMailboxPatched = true + } + } + } + if resumeNetworkAckWaiter != nil { + defer resumeNetworkAckWaiter.Close() + } + // 5. Transition: Standby → Paused (start hypervisor + restore) restoreCtx, restoreSpanEnd := m.startLifecycleStep(ctx, "restore_from_snapshot", attribute.String("instance_id", id), @@ -281,15 +328,22 @@ func (m *manager) restoreInstance( attribute.String("hypervisor", string(stored.HypervisorType)), attribute.String("operation", "reconfigure_guest_network"), ) - if err := reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet); err != nil { - reconfigureSpanEnd(err) - log.ErrorContext(ctx, "failed to configure guest network after restore", "instance_id", id, "error", err) + var reconfigureErr error + if resumeNetworkMailboxPatched && waitForGuestNetwork { + reconfigureErr = m.waitForGuestResumeNetworkUDPAck(reconfigureCtx, resumeNetworkAckWaiter, stored, resumeNetworkAckCfg) + } else if resumeNetworkMailboxPatched { + log.InfoContext(ctx, "guest resume network mailbox patched", "instance_id", id) + } else { + reconfigureErr = reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet) + } + reconfigureSpanEnd(reconfigureErr) + if reconfigureErr != nil { + log.ErrorContext(ctx, "failed to configure guest network after restore", "instance_id", id, "error", reconfigureErr) _ = hv.Shutdown(ctx) m.rollbackAdmissionAllocationActive(stored) releaseNetwork() - return nil, fmt.Errorf("configure guest network after restore: %w", err) + return nil, fmt.Errorf("configure guest network after restore: %w", reconfigureErr) } - reconfigureSpanEnd(nil) } // 8. Delete snapshot after successful restore unless the hypervisor is keeping it @@ -379,7 +433,7 @@ func (m *manager) restoreFromSnapshot( } func reconfigureGuestNetwork(ctx context.Context, stored *StoredMetadata, alloc *network.Allocation) error { - cmd, err := guestNetworkReconfigureCommand(alloc) + cfg, err := guestNetworkReconfigureConfig(alloc) if err != nil { return err } @@ -389,6 +443,30 @@ func reconfigureGuestNetwork(ctx context.Context, stored *StoredMetadata, alloc return fmt.Errorf("create vsock dialer: %w", err) } + err = guest.ReconfigureNetworkInInstance(ctx, dialer, guest.ReconfigureNetworkOptions{ + InterfaceName: "eth0", + MAC: cfg.mac, + IPv4: cfg.ip, + Prefix: uint32(cfg.prefix), + Gateway: cfg.gateway, + WaitForAgent: 120 * time.Second, + }) + if err != nil { + if status.Code(err) == codes.Unimplemented { + return reconfigureGuestNetworkWithExec(ctx, dialer, alloc) + } + return fmt.Errorf("reconfigure guest network: %w", err) + } + + return nil +} + +func reconfigureGuestNetworkWithExec(ctx context.Context, dialer hypervisor.VsockDialer, alloc *network.Allocation) error { + cmd, err := guestNetworkReconfigureCommand(alloc) + if err != nil { + return err + } + var stdout, stderr bytes.Buffer exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ Command: []string{"sh", "-c", cmd}, @@ -402,30 +480,44 @@ func reconfigureGuestNetwork(ctx context.Context, stored *StoredMetadata, alloc if exit.Code != 0 { return fmt.Errorf("network reconfiguration command failed (exit=%d, stdout=%q, stderr=%q)", exit.Code, strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String())) } - return nil } -func guestNetworkReconfigureCommand(alloc *network.Allocation) (string, error) { +type guestNetworkConfig struct { + ip string + mac string + gateway string + prefix int +} + +func guestNetworkReconfigureConfig(alloc *network.Allocation) (*guestNetworkConfig, error) { if alloc == nil { - return "", fmt.Errorf("missing network allocation") + return nil, fmt.Errorf("missing network allocation") } ip := strings.TrimSpace(alloc.IP) if ip == "" { - return "", fmt.Errorf("missing network allocation IP") + return nil, fmt.Errorf("missing network allocation IP") } mac := strings.ToLower(strings.TrimSpace(alloc.MAC)) if mac == "" { - return "", fmt.Errorf("missing network allocation MAC") + return nil, fmt.Errorf("missing network allocation MAC") } if _, err := net.ParseMAC(mac); err != nil { - return "", fmt.Errorf("invalid network allocation MAC %q: %w", alloc.MAC, err) + return nil, fmt.Errorf("invalid network allocation MAC %q: %w", alloc.MAC, err) } gateway := strings.TrimSpace(alloc.Gateway) if gateway == "" { - return "", fmt.Errorf("missing network allocation gateway") + return nil, fmt.Errorf("missing network allocation gateway") } prefix, err := netmaskToPrefix(alloc.Netmask) + if err != nil { + return nil, err + } + return &guestNetworkConfig{ip: ip, mac: mac, gateway: gateway, prefix: prefix}, nil +} + +func guestNetworkReconfigureCommand(alloc *network.Allocation) (string, error) { + cfg, err := guestNetworkReconfigureConfig(alloc) if err != nil { return "", err } @@ -445,7 +537,7 @@ func guestNetworkReconfigureCommand(alloc *network.Allocation) (string, error) { "ip route replace default via %s dev eth0 && "+ // Drop snapshotted ARP/neighbor entries so peers are rediscovered. "(ip neigh flush dev eth0 || true)", - mac, ip, prefix, gateway, + cfg.mac, cfg.ip, cfg.prefix, cfg.gateway, ), nil } diff --git a/lib/instances/restore_egress_test.go b/lib/instances/restore_egress_test.go index f65b0eb1..b5fcdb4e 100644 --- a/lib/instances/restore_egress_test.go +++ b/lib/instances/restore_egress_test.go @@ -1,6 +1,9 @@ package instances import ( + "encoding/binary" + "encoding/json" + "os" "testing" "github.com/kernel/hypeman/lib/network" @@ -30,7 +33,25 @@ func TestNetworkConfigFromAllocation_PreservesDNS(t *testing.T) { assert.Equal(t, alloc.TAPDevice, cfg.TAPDevice) } -func TestGuestNetworkReconfigureCommand_AppliesAllocatedMAC(t *testing.T) { +func TestGuestNetworkReconfigureConfig_AppliesAllocatedMAC(t *testing.T) { + t.Parallel() + + alloc := &network.Allocation{ + IP: "10.102.146.62", + MAC: "02:00:00:85:17:c8", + Gateway: "10.102.0.1", + Netmask: "255.255.0.0", + } + + cfg, err := guestNetworkReconfigureConfig(alloc) + require.NoError(t, err) + assert.Equal(t, "10.102.146.62", cfg.ip) + assert.Equal(t, "02:00:00:85:17:c8", cfg.mac) + assert.Equal(t, "10.102.0.1", cfg.gateway) + assert.Equal(t, 16, cfg.prefix) +} + +func TestGuestNetworkReconfigureCommand_FallbackPreservesShellBehavior(t *testing.T) { t.Parallel() alloc := &network.Allocation{ @@ -42,18 +63,15 @@ func TestGuestNetworkReconfigureCommand_AppliesAllocatedMAC(t *testing.T) { cmd, err := guestNetworkReconfigureCommand(alloc) require.NoError(t, err) - assert.Contains(t, cmd, "ip link set dev eth0 down") assert.Contains(t, cmd, "ip link set dev eth0 address 02:00:00:85:17:c8") assert.Contains(t, cmd, "ip addr add 10.102.146.62/16 dev eth0") - assert.Contains(t, cmd, "ip route replace default via 10.102.0.1 dev eth0") assert.Contains(t, cmd, "(ip neigh flush dev eth0 || true)") - assert.NotContains(t, cmd, "cat /sys/class/net/eth0/address") } -func TestGuestNetworkReconfigureCommand_RequiresAllocatedMAC(t *testing.T) { +func TestGuestNetworkReconfigureConfig_RequiresAllocatedMAC(t *testing.T) { t.Parallel() - _, err := guestNetworkReconfigureCommand(&network.Allocation{ + _, err := guestNetworkReconfigureConfig(&network.Allocation{ IP: "10.102.146.62", Gateway: "10.102.0.1", Netmask: "255.255.0.0", @@ -62,6 +80,40 @@ func TestGuestNetworkReconfigureCommand_RequiresAllocatedMAC(t *testing.T) { assert.Contains(t, err.Error(), "missing network allocation MAC") } +func TestPatchGuestResumeNetworkMailbox(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + token := "test-token" + mem := make([]byte, 4096) + copy(mem[512:], guestResumeNetworkMailboxMagic) + copy(mem[512+len(guestResumeNetworkMailboxMagic):], token) + require.NoError(t, os.WriteFile(dir+"/"+firecrackerSnapshotMemoryFile, mem, 0644)) + + payload := &guestResumeNetworkPayload{ + InterfaceName: "eth0", + MAC: "02:00:00:85:17:c8", + IPv4: "10.102.146.62", + Prefix: 16, + Gateway: "10.102.0.1", + AckPort: 43210, + } + require.NoError(t, patchGuestResumeNetworkMailbox(dir, token, payload)) + + patched, err := os.ReadFile(dir + "/" + firecrackerSnapshotMemoryFile) + require.NoError(t, err) + + offset := 512 + require.Equal(t, uint32(1), binary.LittleEndian.Uint32(patched[offset+guestResumeNetworkMailboxSeqOffset:])) + payloadLen := binary.LittleEndian.Uint32(patched[offset+guestResumeNetworkMailboxLengthOffset:]) + require.NotZero(t, payloadLen) + + var decoded guestResumeNetworkPayload + err = json.Unmarshal(patched[offset+guestResumeNetworkMailboxPayloadOffset:offset+guestResumeNetworkMailboxPayloadOffset+int(payloadLen)], &decoded) + require.NoError(t, err) + assert.Equal(t, *payload, decoded) +} + func TestRequiresRestoreConfigDiskRefresh(t *testing.T) { t.Parallel() diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 05d8084e..523917b4 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -127,7 +127,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps } if restoreSource { - _, restoreErr := m.restoreInstance(ctx, id) + _, restoreErr := m.restoreInstance(ctx, id, restoreInstanceOptions{}) if restoreErr != nil { if copyErr != nil { return nil, fmt.Errorf("snapshot copy failed: %v; additionally failed to restore source: %w", copyErr, restoreErr) @@ -337,7 +337,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str } return m.getInstance(ctx, id) case StateRunning: - inst, err := m.restoreInstance(ctx, id) + inst, err := m.restoreInstance(ctx, id, restoreInstanceOptions{}) if err != nil { return nil, err } @@ -486,7 +486,9 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS } cu.Release() - inst, err := m.applyForkTargetState(ctx, forkID, targetState) + inst, err := m.applyForkTargetState(ctx, forkID, targetState, restoreInstanceOptions{ + WaitForGuestNetwork: req.WaitForNetwork, + }) if err != nil { if cleanupErr := m.cleanupForkInstanceOnError(ctx, forkID); cleanupErr != nil { return nil, fmt.Errorf("apply snapshot fork target state: %w; additionally failed to cleanup forked instance %s: %v", err, forkID, cleanupErr) diff --git a/lib/instances/types.go b/lib/instances/types.go index 8c031fc9..e38f5499 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -268,9 +268,10 @@ type UpdateInstanceRequest struct { // ForkInstanceRequest is the domain request for forking an instance. type ForkInstanceRequest struct { - Name string // Required: name for the new forked instance - FromRunning bool // Optional: allow forking from Running by auto standby/fork/restore - TargetState State // Optional: desired final state of forked instance (Stopped, Standby, Running). Empty means inherit source state. + Name string // Required: name for the new forked instance + FromRunning bool // Optional: allow forking from Running by auto standby/fork/restore + TargetState State // Optional: desired final state of forked instance (Stopped, Standby, Running). Empty means inherit source state. + WaitForNetwork *bool // Optional: wait for guest networking before returning a Running fork. Nil defaults to true. } // SnapshotKind determines how snapshot data is captured and restored. @@ -314,6 +315,7 @@ type ForkSnapshotRequest struct { Name string // Required: name for the new instance TargetState State // Optional TargetHypervisor hypervisor.Type // Optional, allowed only for Stopped snapshots + WaitForNetwork *bool // Optional: wait for guest networking before returning a Running fork. Nil defaults to true. } // SnapshotPolicy defines default snapshot behavior for an instance. diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 1c053f62..add4df50 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -732,6 +732,10 @@ type ForkInstanceRequest struct { // TargetState Target state for the forked instance after fork completes TargetState *ForkTargetState `json:"target_state,omitempty"` + + // WaitForNetwork When the fork result is Running, wait for guest networking to be applied before returning. + // Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. + WaitForNetwork *bool `json:"wait_for_network,omitempty"` } // ForkSnapshotRequest defines model for ForkSnapshotRequest. @@ -745,6 +749,10 @@ type ForkSnapshotRequest struct { // TargetState Target state when restoring or forking from a snapshot TargetState *SnapshotTargetState `json:"target_state,omitempty"` + + // WaitForNetwork When the fork result is Running, wait for guest networking to be applied before returning. + // Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. + WaitForNetwork *bool `json:"wait_for_network,omitempty"` } // ForkSnapshotRequestTargetHypervisor Optional hypervisor override. Allowed only when forking from a Stopped snapshot. @@ -15953,213 +15961,215 @@ var swaggerSpec = []string{ "oWvlHKxmPrBPTXxg59CSG37TplMvIgsSFZnAHF0mp1cQlPGJWbTSrChb4IiGY8qS1MsSjaR8lQrQRU2j", "CE94qsw1o00NzzuBeGuwPaZaXLezc19xcbk2clWfxFkG/Fqv0CE40qfWVQOnOEb2a5ccUVD6sutAc2lq", "n0v0znxhPET5z0laxtPpQk/Wk8SQIFJxkKTWYWibaatd+vUWcJa66BHTXy47Hyl0pjc10Qb3a2GLGQHk", - "B7VWY9Gc8h7eP4fXWwfC6w/XOlJa0J2Rq8cgOmQK9DTb9iTDycNQfFUsW+ZryF+CU1jQkPQR7C4IqnGZ", - "iZWddq54kpAw8//0R8xGkmc/SXODoj80dFBzQgXigs5oueOyg+0hg+JuwoqOm27NjsUP6xoqPITwjeZN", - "j6fKoDxcumQtUsycsovQ6XbOM0wMK4nKpHmX4YrUKJIHedaG+Prsw01D2xLBp9SHdASxEPaptcxc0Ndv", - "u4Pz3vD/MQGcmt9ARaPMxE/EPKxAWNj32508r88+nDWNKQOVQMXR1eaURbysgtVyFLGXSvZW0lowjv31", - "wZJ1kuveL3y67FTgmEzS6ZSIcexxrr3Sz5F5wYQ2UYZOfy7rs1pvbms1n5UWB8zmKQ4sJkA76nsccpVp", - "dAvU/ORfrnfEHMNNmYR6qYR9xyYT9tGbDMYDvT77IFEepeTx1JWXtzFS/2y+lDTAkWnRJAZTVnSwAXO2", - "1pDP8g+tK9KjJ/vRX9xGQBuLWZLCNjx/1zt5+3ErDsmiWxoTRBbNeUT0uDcL0mLh8gnztIKSkFg0eToM", - "Y8i2G6hAq2wHtyZSYb96qKO4wtFYRtwXrPFeP0TwEG18fGXyvfQIuigpLaX+vUCFEn/ve3eMlkhN3Z5D", - "h1WXaWmDe23HMg6nca8Uplfq1LdVTJB9XcepQy/xy/JC88v1cD+mkeZ+j1weQMWpbRVJZNIFEKQLuPtn", - "ZD41XmvrGpEkwQIrEi2NZpEdfRGdkmAZRGaPk/pVIrkmwQ0SEV7q17+aPOJUkLGaCyLnPCpfQO9067Bv", - "EoIgF8TCb5g5FRzviqMYi0s4GJ0ijVJmKFBOOthZB2k5Vyq5waR+ef/+zFjXioiFiSgrBvnK2tXqMYnw", - "Ek2IuiKEualgiTB6zTOck2pWjmyAQBBqnBBBeZmGnR1Pv+cmMBPNBA4IMl85FEa7JBJOzbaktL14oLyC", - "gEjZsL7DVetrP52mUbs19g1ruBa0NLjJAr8/OnOp/BkqoSPzdp3KZ0T0zJZz8ISrl3ZbrgaVcF0xzki9", - "M8EnBFAmbOJNMUXIBZYAiob+vJSWUxAOspgPY/uBXWBI1TX7/FMrZa+63X0X6TFmoQ/d0UQ8m9zHWRrr", - "BdFDFincHdHQRJ2YdEyjlhcDtwXBIWVEykriWJCKqNPt9KZ2VgdbWxEPcDTnUh3s7gyfb62O31sZuGnj", - "VMYhXWXfuWgWE+/g8qsMyiJMuswSWzhJWnjADB3XnA8gnuqBYoDoqM+2gobnIocHg1oO3DUOlAPGAZdY", - "6coTF7dtolmyNB9oMIMV3nvxorg/B9476BzV27H/Vo339cxMMJnmEeNuqNDRMPlnr0bFhQ/9hAtlM1wm", - "xEWbZeehi+Wylyqlzp4Pnhdn2Qo8GYRNZZvbfeeZqnm7nPVGCuS2+9c2AHExZZ3DbenVWGmaLmt4SkvE", - "GkutJihPCLsRPfd2d7ZvRs+2EzlxSXkVueTLuT86PTY6UcCZwpQRgWKisEWKLwgZ8CVpKaMN/BCTGPIU", - "pv+xWrQ0xC8Uk+gbb8CPanBrD3L73QAT9M7Eb4YoxoxOtUC2bxZ7lnO8vbd/YMDMQjLd3dvv9/s3TS1+", - "mecSt1qKLZM+Wcgy7sv53dbhATKI28zlS+fs8P0vWpClUphDa0tOKDso/Dv7Z/4A/jD/nFDmzTxuhX9H", - "pzXcu3I8mDb4ze8HBahyp/e0giP2O4MhLBTgDrwwMQrPtG5jOO6ueDC3RozLYUtVASmumE3TAjWOfl59", - "jey8SvCO7TNlikY5oF79AvlWkIhyJWpUDTEqISzDiYoi81fA2ULvCh9oVOkkcs/uFHyxUvX6e13jWr/f", - "nOK1hm39XrZM/rUFy7OQNp6T6JtL/dsEKJV7fzv7zz/+X3n27B/DP377+PG/Fq//8/gN/a+P0dnbO2Wv", - "r0Yz+qaQRPeGQgRROSUoorasdIpV4PFGaUOngcL2ibGtVTDvoyPwmh+MWA/9RhURODpAo04lv2rUQRsE", - "bAL4Sit2uimbJrqpPz4zd2f64y9O4ftabSO0+aDCLkiWRS7TSchjTNnmiI2YbQu5iUjQgPVfIQpwosBx", - "QRnSlt4STQRUY7F3G3nnXfQFJ8nXzRGD6wFyrYSeQYKFyuDXXA/AFHZUJuDSvk5Ch+djrhdGLDuXMjgf", - "c8HVz9RcCGyopqv4ibLaUrE2wvOBD/gIQub1QkZUKqNsZ5yt2SiL5UfPB5t1y2WNNp3x0Ar2g51QL9Pk", - "mLLFXjIMDF0bwT12zrg1gQhaNpk9gsBWUhz+e45cQzktsiU2HnKTQCHNBauKZCF1YrPjBYmH1W05IXPD", - "CJ9FLVKuX5rcmve/nSNFROyyHTcCTc4pDfT8IHaSSplqVqQYHR6dvtzst6gzBbTNxr9iHd9nM6xmxtob", - "x6aL1NywwzHpopNjyG2yOzRX4CAm+RUXKDICJt/XB+iDJBUbEbLXICTSrGS0zK8tzQkw6my6FpOqpDhA", - "7zK9EWdDKdW2Kt+E5vsSmrVRKyZgutZ6t1Y1Rji7yIo2CI/GKsuw0ydusyho76iwFIc9XzGrb7y3izfJ", - "jUZzYe3vG+zu/tWdnZupO65wQTLH0sfd8+JVCLy0oloQrfh3RfO9f6nftQV7su6gBpE++oqf+0qt7PWG", - "w/fD3Zvb/DfFKSsDehSwZDKosvYYYw+B1VW3f6+pGjdGhCL92MZ/Oivv4ymaY8n+ouBhxdYb7jxrhTqv", - "e20bS1mMouRTM6RMSjl0kCwG0OCkXNIoMqG1ks4YjtALtHF+8vrXk99+20Q99PbtaXUpVn3hW58WkGVO", - "VLw++wDXaViOXThScwYOzrPYyDWVStbBU1pF9d0FIs182g5W3E3StJHji6/GWfulhIXmRcPZvEeANBeK", - "WSPjY0Cffcscl+8Pdm0lUNpd0c6s8fJAYGeNwt0HFFaW8+bn+4Ute5DhrC3SVzzrXQLirXHCuh3qSb46", - "lFoEkxCdnOVQ5bmT0TVfmZOteDkcDPrDQRuXa4yDFX2fHh6173ywbVSLAzw5CMIDMr2Dy9cytlHGcXSF", - "lxKNnLk06hj7rGCYFbatNalaXU/X4dhuh75WVWj8ctoodu7eX9ryNk3KTYvsmirYS5xGJoGzWBKnXuVT", - "JgbZzGACZ7rsiMEAuxZhJavMiYNApLk/w5VcM5pvmli+HzFBZMKZNIUY++hXspQopnCHkHUPkUMSZUHc", - "4YhtCBfwn0X2JziVJNQ/QDRt10Vt6qFRBdjH+oMRk/MUKsht9tERZzKNibCuHjSh4IfeRDI1xh2MF6gB", - "9UwlDYkYMf2aBzvtS6aoH+wPBoNBVtmuc7Cj/z3wcdMd8fPc5+1UDvt5rmv4eXgd3F47HL27gpetKodz", - "Xi6E09qiu0O1yFbB5k6ns2Hm9qvxTe7KCAp4GoXaTJjog8F4cUhonU2SqLzGEJwlH9gl0+xcmroNclMc", - "/ZESsUQfT09LF2yCTG0JlRYThw3VsA48udEybK8xrNeO5paQdo8BY1c91AvK1L2D1hU9/C7dzXBoC09/", - "blx5Q5QpM0uj+WTFnCo+2pAsxmnq09n1I5fk/uHDyXGJOTDeHz4fPH/Rez4Z7vd2w8Gwh4c7+73tPTyY", - "7gTPdhqKmLVPUbh91oHXQvNVpXKxh2MXA+ktPNwQgVo5Im1U3RVlIb9qVbI7692GUK3rvh4g2XoI3rBq", - "qHQMLTVIidNCPWMTVlipWdTgedp/Pxiu8Ty1K8PcIH/fi5QFBn0KJHHm0y0WYC4uVr0I683FKQzIhS+v", - "o1ax8/ZEGxzsvTjYuyvRXAjuujFW2ekRF7cp4MBBGFZifF2eScF/UahgDPqGcbPakOBOt5NFLcPfcNBW", - "IuKyx61C8Zs2bNcvRlbJ74aMtJOS2gy3sAbJKDzQWkCWzDRJFcoSHbV6cRTxNEQF348BdoGLkZOCCq2b", - "gXsK6xoyCF0mpFar2gAJCYDElGlBDBdCuhGbvnaAXsO78AjHxrqwg8AsrNyF4HBp7oL1/nJdG11/9ZDP", - "rZoP32idH0FhcT1tTQbrIlzdhNF8DtAbDt9kRgfjVV+jeR20/frrVb/khs0rc/nH0JlV4w7Qq0x1y5Q/", - "q+xtSGL/HFuBlaf9b5aSL+2KdzS35CtXyCvsdgxFO92OIxTkH9YzET/kXF/bf0VW9AVJEBzBXs4zvVJF", - "I4tyCTOhUFrbhgDrxW3SLywgPAnHxjBpCnky6UPWeMk+curLx1O0AXhWf0PWkar/tZmFR5XOuu0Xuy/2", - "n22/2G+FWpEPcL3aeQTJbfXBrdVBgyQdu6q1DVM/OvtgTPDAGLfgmbdzLyQJJ4Jr0aNnnpfBzTt/0X9R", - "BOsIeWpKgtshWWSfr4XC9ytrFjfE+PxBowWdTtkfn4PL7X8IGg+v9+X2xOvQzCvse70/J8Xb3pqrlEx6", - "poiHH08BGErIRsiRd0TCDNA5UQj4p4dwAKZDlpNmWc4Bk1iKexlrd2dn5/mzve1WfGVHV9g4Y/AFeQ5l", - "O4LCFoM30ca783O0VWA406ZL1AWAUGbNSv8+Q7YU2KCskPaHgx0flzQc3DnX2LYXcSPJP1rTzE7KEh1S", - "6zKzrbbLvdTe2Rk82917vtduG1sv5Vhcr5YwLvDckMfi2BZXfgO0yfeHZwjSuqY4KPtNhts7u3v7z57f", - "aFTqRqMCDGaDnXqDgT1/tr+3u7M9bIed44sCsKhQpQ1bll2eTedhCs9qeEhRF73dptPCp04ZBntHggjT", - "+DBwEbyV08dgpI6FeS1fhDYHg3WM1w6uFt+2chxVSk4b1YALlLIMmbu//grwdjd6zWLanAfrxXjdso8w", - "0+SyIA+mFMctaJcIsqA8lffQEFcm1WkacS5u9G2ThfKOyDRS5tqNSvTx9C8gRDRzIalIUjaaLPutgMK4", - "5eRutIFLPOHn6iZitVqNNku/asLdhm3aXZUHXdr+jYgzoRZVKVsffneEoyAF8HmcraeeFWBHQCZnkkRL", - "E6gaRZwzFMwxmxEoI2hKXbAZwmjOo7DvDR7UT8ZT77U9v0IRN1iZl4QkFpfdDEJ/pnUWuiBoo5BJigwr", - "Vcoz7cVGqljk7TI37sX+QkBY+pIfsgxGTU+seAHG0XxS8jFGfCbBClQQgtuvogcnWJjIWsxMnYFFbIzH", - "MvTOtj7tPUOsSG/fEWqOTj61Fq3VMSA/0FASB4JLiUhEZ4Bp//G0kna2IociSz5bH1JXHmwL1jUXaZ6z", - "C8402bocie9A9ASn3+VIBB6GHJQVwWrOGxljlgJSe4GRyXVChWGPdgFpcy7VOIMTueFgpRoDCncqSI45", - "lCVLZg4g9473XHSi7TbkspGft/q6xlX+ppoG2CxTvRT1U6ub8aCPjeuAKisxXHJQmCoCyE0gf3LYZiqh", - "VVpAm0EbjKuSWCpAD2+2Cc7w26i6n5p5aqtM/bY7OG+LxrMafOcMq/kJm3JPyvYNriGd69nGCyZExBRw", - "6FFIGCWhMx6z+0jr24I8v0gSFKbEUs4opAJbgmOzvSHtmjmnGGWziqyvdtjGH2zGsBqkG/q1L7YJs5H+", - "7LD3IgVamcA4iXCeJ9YqypDKsf/+qt6wILM0wgJVEadWDFku44iyyzaty2U84RENkP6gesk85VHEr8b6", - "kfwJ5rLZanb6g3Ge5lC5NDaDs0kuZkEq/eZT+EnPcrOSYgeuly3z/RYk+reJWvLG6r6iEbGgTB8YvS4w", - "ehnFdnd70JR92dBoKe+yDuh1U8ltWda34x3W1mFWNNNzS2mibit3pWVH5NqbPgjrXpVrWnfFoA0XCOVQ", - "gst0LaD1tvKEtIssr4b8udFsSRKUe999vvdsvyVc8p18nSaF/b49m4t4hUezYaVO27jNnu89f/FiZ3fv", - "xfaNHFQuOrRhfZoiRIvrU6mNW3Ga7UEk1eBGgzLxof4hNcSIlgdUqnN76wF9XbF1m4IL8r3ZdMkZFVfS", - "3bOUPaDtfIwrtKXDkspVqAK/QaZTAkbl2NCtlw+mkiHYagwBTnBA1dLjMMFXpmRf9koF7q2NN608WA9J", - "bdsWGkhLLplO8iSKDdc5+qtxrVd44Xlr1HWZTprc+G+rvRonfu4DKl4RtbihyQtD1t0F2XyusCzFmum/", - "A4iezKv8V2NmzRuroaeqgB9wCWiLCxQiKXyQhZXzz35UXP7KchbcviUluUrxVUdo8xa8kQ3tOZE9JnSw", - "PhOmIh/sAXi7r8aTYj2ElQUnSsUT8lP35v22SPapg4VmJ9jN+yukPdzkwyowFvCjHYMled52t8QSDdxU", - "CNP1mCM8Ir0szsHG8CKZGv+q3vMWa9GThxFc8um0DPi01wwQCMh8ELLtesFKkThRXUSuwUwnYQ1dztXR", - "3pOjDuICjTrDeNSpOAG9SRAxvh7bDsq5yoNViH1ZJaDqIKWbwSTiwaWB+leCEtlHAxQTzCRKGWz+io9y", - "OFjta+t2ksLaZPh4xNwQ18QWjGlC5nhBAZbVeqhmpTgWck2VhHgbaOcAhRzs23KdIztD/ZpJUTjIJw2H", - "DmZL27BuUL/HmQsIyt8Fc2kK1ZXYZyJ41ybfaYn99u1p19z/QOSGGVgpPMRN1IxAC8isiwrGaP67P/xq", - "EpExjLuKWRnX6VhMJQM/tSCSKGlB7HJ2qDABCnjKVBXMMm4XwlmOeK8fSSmDWAl7ewaQDbZ3Wx0xJIGt", - "QFt3L5UY/RbMXQm7tJT2xV3u+FgYNgV45vye93fWvV4dgERYkGKJMtNOMSzO+FzHUnGLaZ/t6jG5DggJ", - "q3A8/lfahhraL72hhr9hm/OeVQ+zb0O4WH12/YeLPYexNlG7GBLJOOtBtq1bUpsZa1BDbO51mdFK+HuF", - "DNSxD/zI90KbvClyvZrWb8i1ApDAMI00eZtY14oqexito/itky6aNjQX64ttPkDtBBOud6vqCTbS71EK", - "KNifH6RoQm05zoly755bvmmulVmCNy55BF2ApHulfEVpeKeL7ImOhvFmhed25343iAXoapkvwnBMxokg", - "U3q9glvMC8YSLmcj5zunVE9Voo0YX6PdZyiYYyErY2d0NlfRsnx/uesBA7hTKRFBFGHqBpVn89V0H9aD", - "BexyFlv3acPnhdR9f0VcEo5XwdgdZa+569gEL8Ft0+hjfbazOxjsbA9uhWN3X4V6C+00pSAUvrP3JKWo", - "nmILWcJXvZrTlaCQN5aRSSpBcHwAgcoJDgiKyBRQXrIqemsPi1rXqwdvNSibR57xv1sou27uCqMMFJ11", - "ZSEA3TQ6LgCqDB1QfF4f9goomEzMBDVMGE+Owk5vsP9+uHOwt38wHD4E8F1GpKbo2Gefh1fPom083Y2e", - "L5/9MZw/m23HO17D6wFqQpdzPqslou0cEiKqZbqq5e0kiSgjPZlFlK9P61ghC0yQxtr9fzPHvpnBSmXh", - "vDzJos6AVU6cEmc9Ek6GHf3K24nq8E+OVw/7ViHa1YH4Gaw6FOCndoMBINbhXVE/U9by3PlQeLH1ybMy", - "bWDd2ePL8oSt7V3lBor7+LkkGEs7bNWJXT/VPN7RGRdUzePVx0P2WoYhCHFmn6UKy7gMfXQyY1CUr/hz", - "FlZQNJP0x51uJ/q8W94z9vf2CB0WMy9jQLvURTWgxbU71HxcTQV4JTcthIn809a4HvNPw97wBQS/RZ93", - "fxr0XvTR3wtBeF1DrSL5hu7t0q+DNjQsVrpwCOnDFzeKUHP0XMVBv1JfnYb8ILZoepbH84po7qxwGUml", - "Bc4f19a4AiPQqHHeVbWzp9m4qCWFJMJLXx3rgitWVuzDIpOhCZlRJtt4ZncGmWt2Lx51+ujQYlCCtZoX", - "vCw1D6UOC3xC45iEVCuVxrhvjvjcbultqxoPNwMmdl951LO+Xz97sT6HdF2A+rpjsn+HhKU7mbvtTNxV", - "6c3gOXM2KZRAgRe7iE4RZpU6PLbqrM00hMwRQJg5cIAqOctaGSBzxc95QrpoxhXKcwxbetRS1uz5y8ZP", - "rsGjuiKp2DDE9r1kjGfoJXSV+Do5RongYRrkCTYRDDpPiRZpBcxxhVa/PoTpIR0akLk25QKtd2g0eTDa", - "eSCb1rvifdQM27zUw8H6pX4QL0i3kybhehlmXmonwW6ENbomZcPjkymTvaIJFibzqYVEf1ekYN3INd7i", - "QKtEaeKuUDRP1TnJc6EClwg+2L1jEhF9TNUbQTwK86hSKnMpul6kDvefz5suMeHOqT6QXwlJtK0CABHQ", - "X4zZ0juwapFVtDFwlbOkudLqGaxyS63y4J6t1cQal6p9vdqKV9sklBfLA2egm/dbrNZ+ubaU+EP44b6l", - "kvbWXi5U4NUc+F8GV+r6NxkzAM/Iosp5veu7gPexxXtrGTdBxlWzZopu5sPefxu3Mhr3D7Z++tv/3fv0", - "V697uWI3SyJ6IZlCKNElWfYA7x5pG71fBkwDrF6tTNva/orgGJxGwSUxTqoYXxfHuzfIhMbyDY5rU4AY", - "rJiy7N9rJ/S3f2uOYCqQ8QPIybUse2co64eoE6S4O442YiJmrpKxC7zf7I8YxPJfkqVEhWIEVqVxjPoX", - "mX2iVXRwWuIIXRg1sE/Y4gJNKNR0kSOmrVocBCTR1oQFZaemGB8H6SMIjort2KIILlHOXjmaiAGCPp7W", - "0Pbefnj/89sPb47Hb89evjk8Gf/68r8giOOqZ3oIe5r3dvf2bQm+IiWHniW+A/DvnVD8fOxmsMA8/AVJ", - "KVDX0KMwUwkppS6goPAy2iBxopau2JDLbdm8GTbZYdagN5ztnlHYBy/uo+jMh5VVZhY86mmNugFE1+vA", - "NLTwhmNDUybMvdPk2J55yoCfW2/ijM6wx5ftrUB6H8Vh3IDWgsbV1r+xtIM/OP64ijBspIEhVQURt2KX", - "StVrjp2PtSI1zms9liMyUmbTS2ghYKucSxIztWWLOPlSWkN98q5OKMp3mUMs6sFH6/NkVqryhZkVRtK8", - "NqdOY63o1CsIdKZJczUnghQWAj7IkVtvSDKb7NEiUdqUWkmIyAMhXaYI1D4XFLJHMmeDI0GWEFT3wK5G", - "5j3F11kP4L3HsnbHBfPIMfKHr38edTb76J0rVUqnrgkYRsWe8AOhlrloFU0cV9UXo8hV9Xmb970bz8qq", - "FdKvaW9VmDPvo8SaPn78O6bqFRdggTSnJT84nipYNyERgMtSRUttBTVKYxKOs4LNTfvf1Wg2OclZOey8", - "jJOztjAwsRZy60vtuMTZfAx1SmtykCAVVC3PobCriRAmWBBxmJoN7+rD2p/zjqEq0tev4KecerIQXhNG", - "BA3Q4dkJ7McYM1DS0cfTQiUTU9SmhqEG6uXboxNr4ToYPrBYqALWc8F8h2cnnW5nQYSx8jqD/nZ/AJs5", - "IQwntHPQ2ekP+4OOqegLU9yCIorwp00wzCylk9DqQT+bV/RXAsdEESE7B797EvUgkA1eBn0XzwoWS4Kp", - "sCZLEkH6oGEVqr8FYF13lB6Y89gW5G3toJNqaZMpSPLWLusnUCdh18AUtwcDCzOq7MELqSAm/nzrHzYY", - "Me+3lT4H5PGgzNYsCqdTWpJ/7XZ2B8MbjWfVMGDH+rr9wHCq5lzQzwSGuXdDItyq0xNmMryQwQqzgTbF", - "fQYsVNxhv3/S6yXTOMZi6ciV0yrhskkZJhJhxMiVrQj6Dz7pI3v9AGVj5JynkZYmyKSvOUeDwqI/+4yw", - "COZ0QUbMntNxGimaYAFuhBjp89kYTOWtYbo2q58BC/zMw2WFullzW7q5nnM65wSupiVIMgYc4nFTud/c", - "3UwZ02ISS2JraWR1L+vRPFpcjmXAfflE7wnDTPVkQgI6pQGCl/XutR5tb4OtoPm0wINlIQIQs5yHZnvT", - "n5MKVT/86dzH2TNkyVtWJxhcAwVRGuY6l0uTwmKCo8iL3TSL+ARHY0OfS+JRUV/DG5YoxQIpTrlhPCSm", - "2EWyVHPOzN/pJGUqNX9PBL+SRGgVyBYxs7QmoSlbZlj3CrA+YygkZkqk6j63zBC3vlyS5df+iB2GsSt/", - "K80nOJJcn5q26KDNCzBb2vCuvyxLQ1zJUSoVjy1LZRUY82HyVCWpsnfqkihbeQ1epxIlqZyTcMQUR18E", - "mVGpxPLr1pe8x69guxAcaj4pvGKmtPWFhl+bRi3HWM9+DK96rD8CBBh19Oky6ui/ZwJr2yWVc3CiSHCc", - "zIpLupHh6Wi9cLNK4QAzlPDEYBEBU82xZrlSGxCPjqMIKdhK7lutbcJKNszHphfHk8bcYpMMWtlGlKHT", - "nwubabD73L+fJAkE8Tk4/vP87RsER5VeA/Na7rAyl9pMn6IoTEGTh977I/YSB3Nk9CbAmx11aDjqZNZF", - "uAljTaWN0O71QMX9SQ/tJ9NNl4Y/9fu6KaM9H6Dfv5hWDvReSuKx4peEjTpfu6jwYEbVPJ1kzz75CdqU", - "onleEgRow8j+TVeDGKCi8mPQnBuYhYhbWRstEUa5BCr6USaUYbGygLKH9JaC2pTHM1kkxpcR+G5HnYOR", - "896OOt1Rh7AF/GZdvKPOVz8FrBLdDG5qakg7XTtjov3BYHM9doKlr0eFLr2ot9/Xmva1fW+Kh1W66oqH", - "mZxDZtYraKqBG3XrETSfn3Ho6kv+UPHWqHjWc1FQ3uD74jlg2DcixsCtaGDano2cBrbSOjFsAQUTwOJw", - "SCfG4KBOg8uZt2h+VM35ulmx27TLAhhi5Phv9xH4D/rNShGafl88Vr84ApxxB1z/xNgRFssxYtdvEb8m", - "6nvguMFjiVILiv4t+fep8M9rYvW+nGgVabZFFu6+yY/nBMkm0rZiXta26jmMqXdOmEIv4de+/a+zeCBb", - "+iLis4sDZEgY8RmKKLP3gIXbIn0oWlrCRybfJPvOpp84MM0Nc37+83/+FwZF2eyf//O/Wps2f8F23zKJ", - "kwC+fzEnWKgJweriAP1KSNLDEV0QNxmAxyYLIpZoZwBqZiLgUbGkktVN5IiN2DuiUsEK96UG11LaBsH0", - "YDAfylIibb6OfpFOLeiWcTB7THi3lw0pH3VHdz2JzjCDwgT0qeh4ALJEbfk1a391/N4zM+eS/6zqK695", - "TNfLF0WuleHenhngDQUMkNi37+CBnTTaOD9/udlHYGMYrgBgNdCY82as8tz/IZPWyyQjUcoCBahsZJNJ", - "Ylvt/z2277RzANsW/0weYIu1eQMXsHF5QO66W4EftkILd7Cfbs417PPPHrsszWYH7e3nW+zCxTG1MoTv", - "b50d79Vpbp4USPYtTGC04aLhwY3IBTo7OnE1bTe/GdM/yqmhZ2pr9WVHB+IMsNcezSw74mwa0UChnhsL", - "FGaKSWaqlRnkqYiDd3bUCLt5VSGMi+fbVgmRr/Gky8D58iPv4U+PSqc3OUZymOWc136cJOtY55jKgOtv", - "C9zSC3AChHTqS7ZPi1y0ziFlguuzI2elumTF88mx25CP55qyXaesejY8glA8rgjEbygIK8XjC8DkT4mb", - "P2Sr6BApVniuvi/WHDyeFvTYXiwfmz8lN1ZYIZuWgiaou/EAfU2UCeXuPOBC2x48Ez8nwu1qV0cCZp1N", - "y3xqCquaCcGF9Grb98S80s70Ne39mSxfIM9NNBZL8h8qSgtjN6fVKgP3xJYsfzj7Fnq4kXl7f/e8lsE8", - "RIZgk4nzWJu6u1guWbD5p7rqfZTTzBD7SR5mZ2kUuRuPBREKvT06MTureAZsfYGwpPW6vdttK4+DD+9+", - "6xEWcIhDy2Ko/EqUfXLPGr5ZMDOVH2zSxiY0adHUnWdNGs4d1t+ECyIT4din/N+3X0V0IrBY/vv2Kxwl", - "lJF/3zmMsCJSbT4YswweSzQ/tsb9hJlPK9y0TDQQTQzqva/TULO3Wiqp7v0/lZ5qJn0jTTWj6w9ltY2y", - "WiTXSn3VLsWDaqymj290JZMxm4/a8MjFJ/7JNNXH9fJZjnRA0VSWrz1skT0uwM8LjyhDqSRPMICSZhxX", - "PDZauqvzDbny+HCse3LcBUJCXQRAbbIJIo/kvHbjeHTl1vb7+J7rw3hCZylPZTH3JMYqmBNpk5UiUhbA", - "T03tzo/nRsX7O+bSwWMeHY+uV//g+wfS+KsLaoS3uYFap/O7t9rq/PZ9rfObFGqbu2ahpboOdnCzIajQ", - "JVG3ZeNSrnk92NE3Lp8tgj5oQyU3FxBYEAcj9n+0/fG7Ijj+9JNLkkkHg+19+J2wxaefXJ4MO3WsQhhU", - "PAKU2MM3x3DtN4PscwCSzVPyquMwlSeA9Rx0zr+cgZTffLa3kBwX/rCQWllIBXKttpDsWjysiVSG33p0", - "G8nxm4/gFsTkh5X0GFaSTKdTGlDCVF7ztBYkZksmP8HcMmbvhwrBHaWDtrWVlG3KNQpoXhbg0QN7TnIc", - "xMc2jlwFgqcZI88TC+ltzZH8MGy2R743fhg8rnB+fDvkKbOYUfjrpEu0TukrpQ0Yk3EKZSFRjhACUZ9I", - "2PqPrsU+yitYyzRJuFDS4FSCAmyQ7OdaAfZhWpZhKn24lIDFSInsjhhUKtCPTS7/1iVZGhRKylkGOFmt", - "x+rLvSqjgH7TbXT/OpYf4rSVjvXI29iCVn87HeubiY5H0bROSrUANrKNAQblhGQ7mWfJffQzZbPNJxWB", - "aoRVNrcCnpFH1dqCSn8W13dLZtVkmw7aArSvLT37L3ji1ifp09odFm2BgCikeMa4VDQoVt4twoP+OKFb", - "n9CrKevl5qktke436F9xcdn2iPPUFXsCJ11xht+hL0EPD9DAvr1LAYxtcxpopnn0U7BWLO5bpmDQ6rkY", - "RGkIVentgehUyang8dj+aPBq9a6waKDgoghsq99a2OjeH8Fh9IYrROMkIlqLJyHqGW7Sq2lVfwc3T2Wh", - "tOLNhKHeNsWEGANGJ11pIisi4XLNLdgG3LPXl8srNSM+Ww+CkXXuEB88KBgjZuDwicPOv0CZkIXiXSQi", - "gUJXcxrMAREDCnpBRVcAq8BJcpFBYG0eoNewU4tIYND5hiRCG0IBZ5JHxABdLOL44qCO2Prx9BQ+MmAY", - "Bpv14iAruZ4dEFK/VUS4yGoevbG4HRuakwSPIrOiF9pqLMxv02Jf5BBlI+bDwWDkyjZIp+iiAIlx0YCJ", - "4QTqb3z2zbStbjOwpJmL4kgA4QxvEhZ2mi5iaORHwxgOvPVgWiJzmGE8MDBHbTC/8VkGalliZZwkbdnX", - "DhO4eBHHK3gYbRSKs0oV8lT9TaqQCAEfW+5uYm60gQPzD4UvNaPawkJZeVtgP+91o0GZ85JKC9VCFR3z", - "r0Ucd7odO54COt0NtPc1CCfVBuvXYnplCjAmP/TumwCUlIV9AaGkcnLY8v/NKvc788Kf3j9rCRX+Gbws", - "5fusfBSU5QWgBFRwd1W4nhTSASxkTRczhZF8e8TNsicLxUPbXW/Vyo5+B0bruluvrIZkVuDysa+/6iN4", - "ykkwsjabKRfV9Ph192LfPSPd35LUptqGQ37w5s3dc60YM0lX1BKFUqgS/HxQXxNwnYM557LA9hMyxwvK", - "hUVgt17XjDPBZWGsRxs9d6FZ9cL6by+sen5gfU0IFx/ZPvrwuY2583/hHuVfvCpY25nE7zqVGlAgJcJo", - "IiiZogSnkmhtKY0JMhVGLJA3wcHcVQvvj9j7OUG2PmbBgZCVU6YSXQzjiy6apApFWMzA2jEPTSSdIAGP", - "Y8JCU/N2xOYEL6g21QSKsCIsWPYkgRrIC5IXMNGmu72hNKW2syqrXeSK84KD4aJQevcCJYIAExlzmZXq", - "3I6YSNl/GORK3eyFG+gFIlLhSUTlPKsVEeCQsMALC3n+fYux+3finhNVr077Te4sbyVLv+UlZtGXmdUH", - "/y7uN59YoBYXrrJmCzG/QumVzaZhOfLxPK/I+y+4pc1c3Ry/0c1MRuJVu/j7uJIpleT/cS2j7JYMU9Md", - "KZet/9PeteR1pFNWum6xPtnbXrhklRAyMt9I5m19cX+e3MJH9p1Iwm6jYd+EuZ1P+nsQuZaqt5K538g5", - "aH1JBa/YNxTBdlDfTn3ioiDlvgsxbDZcJo2LMkcJDDYVZz+EcVUY2/CA2wpj53GtXYAXxDNlvSTCTXI5", - "r1rvF8DWIfAvGv1amV1BEH5zwZffCDyasDvJxJsReAleRhz/2e9lAi6ESei05YifDqBYwRdYuGDaAI9b", - "N5MQXZdN8vH0dLNJSgi1UkYI9YQlRLmsaRB7qjW+XRAhaOhKRx6dHtvoVSqRSFkfvY0p1HO8JCSBQjGU", - "pxJBZm5fz8+lttaL4JVyWLsdwpRYJpwytXYU+asPM5ivtyqd98hy0kIq/ukvj8EL//SEFMgOra7YCay2", - "IhVWjcF4LjiNMlPvUmtbeMJT3bqWLK7Q7gzOtimNiFxKRWITmTdNI9hEALprazLZ70xGaRdRJZHeD13I", - "wEuIiKmUlDM5Yrb8e0KE7lt/DsV/8yAjr/Ne4UxqnhnR930EsOnBmJgtrJqoBtACUAe0c9DZwkmyBeWi", - "/UFSdnh3GNIriEhDchlPeEQDFFF2KdFGRC+N0YEWEkX6j82VIW1j+O6+K07dfmdpSp+wKfcW5TA8mzHz", - "nyMJqSzW3CXikxNrr0lxszj5AwvtF2tyrVwTBEc9RWOSJb+jVNGIfjaiTjdCpaKByavJUy+hCLPNvhyx", - "U6KEfgcLggIeRSRQzrmylQgebI3SwWAnSCiglOwQGBwIvObHMfR4dPYB3jOForsjpv8BDb8/PDM3sVNs", - "fQSFgTKirri4RCdbb9cE+Z4Dmf6Fo+TMBFfmQHoX/Mf13c0zmxv3kGzYojxZZQDx5E8fxmk1uB/egqfp", - "LQBoiWw2GzOBA1CK5TxVIb9ifs/AgkdprP9h/jhZB1CicDD/CK9+N9quGc7abtwEn8SmtHMKiSka9E0u", - "KAzBnmp8qSacmwIoMaXIPe8pcKj+jNx9/075Ih2/w6tJS1FXkOu72VuPffLZMTjcrSI9nso2N5zmZqL4", - "au/TFabN3qefIx5cSpQyRaMSqIG22wAHVP+Y4zbaiz9QEyA70pUSR+Q6oQIQbCrwCIjoGUuEkSIipgxH", - "WzBn0wggUDovFl5wCknKQUQhTYyGBCU8igBl52pOGNKzAUeVa6BwTyttBYjiO8UrRsXRhAQ8Jg6Vc9Nn", - "uv0dU/WKizLE5vciF98X6K/no6eq57kGVbS5xzuhjJ7iawhrDlN7TexGtPGa5z8aV1AXwdqMOjsDOep0", - "0aizHY86egWOMLhQsUJ7KKYsVUT20bHxb0Ea6v4ASRJwFkoHDuo8eDsD2ZSUatiyIcNxH757TLXHchWQ", - "8p3txCce9HtIfw8JNmijuOHsngy7sOlCxFMFAdxuX9m3QqLAPbL56DewhT3yw7ZvI8n/brdvSUbBKmtx", - "WVh6I9kz+Mi1XjeXVDHnMkedRAFOcEDVsotwFPEg9x6kMrsd6GVDmQiCL7UN1R+xdxlwpU2EQEdnH7rO", - "aYZCKi9NC9Yv1kdvF0TIdJINDoE0MB48WAwSjpjiKMBRkEaab8l0SgLIYYhoTJVs8KtlQ3nIMoh5J56F", - "dw8z2Jqn5Uzy8wSsXs4WssJxW2aptwQJIkzjolOpShxQfeFKF9y+E90o18fwNLLXW4HgUiLbVI9EdEYn", - "kb2skX30XqscOCYjlkSYMSJQKk3ckR56LxFEytQkxugGoM6s4aguyoFOEsGVdRNHnAtpPLuawz+eIqlI", - "soLN3pmWT2HODwQTbBq3PX0jg6EyhuZjyb6C9IIYTjEE13ykj+lvEOxjBvSt4YSfysZ/L+hsRoTeFdgI", - "WXM1ara1I6fZ9KVMj0aM/PPsrXYY+VmrhWjuQqTzSqCKsXtxDAr0TW5gPZ1f0kYsE/voZtkXv+qPWvZd", - "jvL3D8I+uuMs/yylx84LwdVtkfVzDn9qIPeFkZe2ailBYT0cQeuMhIfMEGiNO/DN4AaeMsoALqUdNMEJ", - "fH+MMHjc7LjHhtl+2rxVQgkoFdZpSJVaD9/5XXDgw+B2fuPs0Fvgdn5X+UqAu/jt8ka/q0ylkh/QFQ/5", - "0yNzPlSCkoHnBBiLpgQlI/VsIMFKQ+mjfaedmWRb/DNp8Pbu+Qb6uyP7D6u/hclQIJbfZWdyox1uC4kT", - "tXSXi3xauQCU9DMkY/iAH7IYgofDW7jF9fr9sYfj08bL9R/1tB7t/j4vOnxy/PSLaBX3XOlg2dKnTg+L", - "YE4XpNnpXt7BlkSJIL2EJ3C5EhqCWXq4s0xh0Z99RrZ5i1Vl/4WogzgmIQqpIIGKlogyxUEimD7+IpHg", - "2hKA51wsfc704s59JXh8aGez5jy0e8o6w/I733jZC7HCvYWTNitcaHe4aXd321rgIcrQ65/RBrlWwiDu", - "oqm2fBCdZiQl1wEhoQSe3CwOeDho8GzSz2Q8m7QZ5Qrs5LcWmxoFqVQ8dmt/cow2oNjCjDC9FlrVn4Im", - "mwi+oKEpRJoTdcEjQ9VhA0Fv6nfVSkVWKcMZF2Zw30SHaXMgzT7TpCwWTOhC56AzoQzD4NaiFJf3lEmo", - "0v1hCmkN+d5xnNP5cYRZy2/DGTuaE7WR44ioODfQeJs/jrmnfMwVA1PdmVY67dqVimwXq9oyhPQhAHOz", - "OObHdVt//H7CK6l8kpGV1nW+yAzSJrf598WCg8c7Hx7bXf7xCYfjvybO+C64yqEB3aKPYX7jAY5QSBYk", - "4glUkTTvdrqdVESdg85cqeRgayvS7825VAfPB88Hna+fvv7/AQAA//8QadjqX44BAA==", + "B7VWY9Gc8h7eP4fXv3Y7V5gqSPwvxeGsjq5gGQ31WqWRKq21bhEIbaJrCy4/xeEywTq2LEqMICoVdnmL", + "oSjg2UTnRFmHo+L2VYSnylp2H0+NkSbTGDxlNCL1XqeUUVg9LJcsmAvOeCqjZStvYttEAU3YtY6mFnzJ", + "yNVjMCVkUvT0tu5JhpOH4chVsX6ZLyZ/CbQUQUPSRyB9IOjIZW5WJNG54klCwsw/1h8xG2mf/STNDRMw", + "KdBBzQkViAs6o+WOyw7IhwwavMlWddz0Y7s+0HYtErZu4cBDCP9pPjTMxKYmRtEk7Rcz7yyTdrqd8wxT", + "xVK9zDrvMlyaGsfkQcK1Ib4++3DT0MhE8Cn1IWVBLI19ai17FzT42+7gvDf8f0wAsN6PoOJTZuJvYh5W", + "IFDs++00l9dnH86axpSBkqDi6GpzyiKmVsGyOYrYS0l7q20tYCcetGKSdZLbbi98ttBU4JhM0umUiHHs", + "cc6+0s+RecGExlGGTn8u20Pa7mrrdTkrLQ64XaY4sJgS7ajvcehWptEtUPOTf7neEaPGNWWi6qUS9h2b", + "jNpHbzIYGPT67INEeZSbx9NbXt7GTI+z+VLSAEemRZNYTlnRQQvM2drCOss/tK5sj53lRw9yGwFtLGZJ", + "Ctvw/F3v5O3HrTgki25pTBCZNucR0ePeLEiLhctHzdNSSkJi0eQpM4wh226gAq2yHdyaSIX96qGO4gpH", + "YxlxX7DPe/0QwUO08fGVyRfUI+iipLSU+vcCFUr8ve/dMVoiNXV7Dh1WXe6lDe71PZRxXI17rjC9Uqe+", + "rWKSNOo6YB26i1+WF5pfroeLMo0093vk8kgquoA1RJBJN0GQbuLiF5D51JzQ1rUmSYIFViRaGs0rO/oi", + "OiXBMojMHif1q2hyTYIbJLK81K9/NXnoqSBjNRdEznlUDmDY6dZhAyUE0S6IhW8xcypc3CiOYiwu4WB0", + "hhhKmaFAOWllZx0k6lyp5AaT+uX9+zPjnVFELExEYjFIXNau5o9JhJdoQtQVIcxNBUuE0Wue4eRUs7pk", + "A4SGUOOECMrLNOzsePo9N4G9aCZwQJD5yil8dkkknJptSWl78UDBBQGRsmF9h6vW1346TaN2a+wb1nAt", + "6G1wkwV+f3TmoCAyVEtH5u06lc+I6Jkt5+AtVy/ttlwNSuK6YpyRemeCTwiglNjErWKKmQtMAhQW/Xkp", + "rasgHGQxn8r2A7vAkKpr9vmnVspedbv7AjFizEIfOqiJmDe5s7M01gsCFkYKd480NFFLxpYwankx8F8Q", + "HFJGpKwkHgapiDrdTm9qZ3WwtRXxAEdzLtXB7s7w+dbq+M+Vgb/WnhmHdJX966weEy/j8vMMSidMuswS", + "WzhJWnhQDR3XnA8gnuqBhoAIqs+2gobnIs8Hg1oO5TUOlANWApdq6cocF7dtolmyNB9oMIOl3nvxorg/", + "B94YhhwV3rH/Vo339cxMMCJYoeCOqdDRMPlnr0bFhQ89hwtlM6QmxEUrZuehiwW0l3Klzp4Pnhdn2Qp8", + "G4RNZZvbfeeZqnm7nDVJCuS2+9c2AHFVZZ3DbenVWHuaLmt4SkvEGkutJihPCLsRPfd2d7ZvRs+2Ezlx", + "SZ0VueTDbDg6PTY6UcCZwpQRgWKisK00UBAy4GvTUkYb+CEmMeS5TP9jtWhpiH8pgjA0RlAc1eD6HiR6", + "ogFm6p2J/w1RjBmdaoFs3yz2LOd4e2//wIDhhWS6u7ff7/dvmpr+Ms9Fb7UUWyb9tpCl3pfzu63DA2Sg", + "t5nLl87Z4ftftCBLpTCH1pacUHZQ+Hf2z/wB/GH+OaHMm7neCj+RTmu4ieV4Qm3wm98PClD3Tu9pBWft", + "d5ZDWDHAZXhhhhSead3GcNxd8YRujTiYw96qAtJgMRurBeog/bw6DMF5leAd22fKFI1yQMZ6AMKtIDXl", + "StSxGuJYQliGMxZF5q+As4XeFT7QsdJJ5J7dKXhnper197rGtX6/OcVrDdv6vWyZ/GsLtmghkTwn0TeX", + "+rcJcCv3/nb2n3/8v/Ls2T+Gf/z28eN/LV7/5/Eb+l8fo7O3d0I/WI2G9U0hre4NxQqiukpQVm1Z6RSr", + "wOON0oZOA4XtE2Nbq2DeR0fgNT8YsR76jSoicHSARp1Kft6ogzYI2ATwlVbsdFM2zXhTf3xm7hb1x1+c", + "wve12kZo84mFXZAMhUCmk5DHmLLNERsx2xZyE5GgAeu/QhTgRIHjgjKkLb0lmgio5mPvNvLOu+gLTpKv", + "myMG1wPkWgk9gwQLlcH3uR6AKeyoTMCufZ2EDg/KXC+MWHYuZXBQ5gKwn6m5EBhTTXfyE2W1pWJthOcD", + "H3AWpFzohYyoVEbZzjhbs1GWC4KeDzbrlssabTrjoRXsBzuhXubLMWWLvWQYGLo2gnvsnHFrAlm0bDJ7", + "BIGtpDj89xy5hnJaZEtsPOTm4lOaC2gVyULqzWbHW2QAVrflhMwNI3wWtUjZf2lys97/do4UEbHLlt0I", + "NDmnNNDzg9hbKmWqWZFidHh0+nKz36JOGdA2G/+KdXyfzbCaWW1vHJsuUnPDDseki06OITfO7tBcgYOY", + "9ldcoMgImHxfH6APklRsRMh+hJBas5LRMr+2NCfAqLPpWkyqkuIAvcv0RpwNpVQbrXwTmu9LaNZGPZmA", + "+1rr3VrVIeHsIivaILweqyxDU5+4zaKgvaPCUhz2fMWsvvHeLt4kNxrNhbW/b7DE+1d3dm6m7rjCF8kc", + "Sx93z4tXIfDSimpTtOLfFc33/qV+1xZ8yrqDGlb66Ct+7ivVs9cbDt8Pd29u898U564MCFPAIsqg7tpj", + "1D0E1lvd/r2matwYUYz0Yxs/7Ky8j6dojiX7i4KHFVtvuPOsVdUC3WvbWNxiFC6fmiFlUsqhy2QxpAZn", + "55JGkQnNlnTGcIReoI3zk9e/nvz22ybqobdvT6tLseoL3/q0gLxzouL12Qe4TsNy7MK1mjO4cJ4FSa6p", + "VLIOvtMqKvQuEHvm03aw9G6Spo0cn341Tt8vJSw9L5rS5j0C7LlQ3hoZHwM671vmSH1/sH0rgfbuipZn", + "jZcHAstrFO4+oLmynDc/3y/s3YMMZ22Rx+JZ7xJYb40z1+1QT/LeodQimITo5CyHus+djK75ypxsxdTh", + "YNAfDtq4XGMcrOj79PCofeeDbaNaHODJQRAekOkdXL6WsY0yjqMrvJRo5MylUcfYZwXDrLBtrUnV6nq6", + "Dud3O/S+qkLjl9NGsXP3/tKWR2pSblpkZ1XBguI0MgnAxZJK9SqxMjHIeAZTOtNlRwwG2LUIPVllVxwE", + "Is39Ga5kn9F808Ty/YgJIhPOpCnk2Ue/kqVEMYU7hKx7iBySKAtyD0dsQ7io5CwzJMGpJKH+AaJpuy5q", + "Uw+NKsDO1h+MmJynUIFws4+OOJNpTIR19aAJBT/0JpKpMe5gvEANqIcraUjEiOnXPNh7XzJF/WB/MBgM", + "ssqInYMd/e+Bj5vuiL/oPm+nctjPc13Dz8Pr4Brb4TDeFfxuVTml83IhpdYW3R2qjbYKxnc6XRaGD1+N", + "b3JXRlDA0yjUZsJEHwzGi0NC62ySROU1quAs+cAumWbn0tRtkJvi6I+UiCX6eHpaumATZGpL8LSYOGyo", + "hnXgyY2WYXuNYb12NLeERHwMGMTqoV5Qpu4d9LDo4XfpkoZDW3j6c+PKG6JMmVkazScr5lTx0YZkMU5T", + "n86uHzmQhA8fTo5LzIHx/vD54PmL3vPJcL+3Gw6GPTzc2e9t7+HBdCd4ttNQBK99isLtsw68FpqvqpmL", + "PRy7GEhv4eqGCNTKEWmj6q4oC/lVq5LvWe82hGpd9/UAydZD8IZVQ6VsaKlBSpwW6mGbsMJKzasGz9P+", + "+8FwjeepXRnvBvn7XqQsMOhlIIkzn26xgHdxsepFfG8uTmFALnx5HbWKnbcn2uBg78XB3l2J5kJw142x", + "yk6PuLhNAQcOArMS4+vyTAr+i0IFbNA3jJvVhgR3up0sahn+hoO2EhGXPW4Vit+0Ybt+MbJKfjdkpJ2U", + "1Ga4hTVIWOGB1gKyZKZJqlCWCKrVi6OIpyEq+H4MMBBcjJwUVOgDl55nXUPF9DytagOkKABaU6YFMVwI", + "6UZs+toBeg3vwiMcG+vCDgKzsHIXgsOluQvW+8t1bXT91UM+t2o+fKN1fgSF6fW0NRmsi3B1E0bzOUBv", + "OHyTGR2MV32N5nXQ9uuvV/2SGzavzOWvQ2dWjTtArzLVLVP+rLK3IYn9c2wFVg4bsVlKTrUr3tHckq9c", + "Ia+w2zEU7XQ7jlCQf1jPRPyQc31t/xVZ0RckQXAEeznP9EoVjSxKKsyEQml2GwKsF7dJv7AFBUg4NoZJ", + "U8iTSR+yxkv2kVNfPp6iDcBD+xuyjlT9r80sPKp01m2/2H2x/2z7xX4r1JN8gOvVziNIbqsPbq0OGiTp", + "2FU9bpj60dkHY4IHxrgFz7ydeyGJOhFcix4987yMct75i/6LIthLyFNTUt4OySJDQc5Km5rXDTE+f9Bo", + "QadT9sfn4HL7H4LGw+t9uT3xOjSzjvzen5PibW/NVUomPVMExo/HAQwlZCNkzTsiYQaQtgz800M4ANMh", + "y0mzLOeAbVwas4+xdnd2dp4/29tuxVd2dIWNMwZfkOdQtiMobDF4E228Oz9HWwWGM226RF0AmGXWrPTv", + "M2RLyQ3KCml/ONjxcUnDwZ1zjW17ETeS/KM1zeykLNEhtS4z22q73EvtnZ3Bs92953vttrH1Uo7F9WoJ", + "4wLPDXksDnJx5TdAm3x/eIYgrWuKg7LfZLi9s7u3/+z5jUalbjQqwPA22Ls3GNjzZ/t7uzvbw3bYS74o", + "AIsqVtqwZdnl2XQepvCshocUddHbbTotfOqUYbB3JIgwjQ8DF8FbOX0MtsFYmNfyRWhzMFjHeO3gavFt", + "K8dRpWS5UQ24QCnLkN37668Ab3ej1yymzXmwXozXLfsIM00uC4JhSrncgnaJIAvKU3kPDXFlUp2mEefi", + "Rt82WSjvDL4GXLtRiT6e/gWEiGYuJBVJykaTZb8VUCG3nNyNNnCJJ/xc3USsVqvRZulXTbjbsE27q/Kg", + "S9u/EZEn1KIqZevD745wFKRQvABn66lnBdgRkMmZJNHSBKpGEecMBXPMZgTKUJpSKWyGMJrzKOx7gwf1", + "k/HUe23Pr1DEDR7LJSGJxfU3g9CfaZ2FLgjaKGSSIsNKlfJee7GRKha5vcyNe7G/kBSWvuSHLINR0xMr", + "XoABNZ+UfIwRn0mwAhWE4Par6NMJFiayFjNTp2IRG+OxDE20rU97zxAr0tt3hJqjk0+tRWt1DMgPNJTE", + "geBSIhLRGdRE+HhaSTtbkUORJZ+tD6krD7YF65qLNM/ZBWeabF3OxncgeoLT73IkAg9DDsqKYDXnjYwx", + "SwHpv8DI5DqhwrBHu4C0OZdqnMGJ3HCwUo0BxT0VJMdkypIlMweQe8d7LjrRdhty2cjPW31d4yp/U00D", + "bJapXor6qdXNeNDHxnVAlZUYLjkoTBUB5CaQPznsN5XQKi2gzaANxlVJLBWgqzfbBGf4bVTdT808tVXK", + "ftsdnLdF41kNvnOG1fyETbknZfsG15DO9WzjBRMiYgp1DFBIGCWhMx6z+0jr24I8v0gSFKbEUs4opAJb", + "gmOzvSHtmjmnGGWziqyvdtjGH2zGsBrkHfq1L7YJs5H+7LD3IgVamcA4iXCeJ9YqypDKsf/+qt6wILM0", + "wgJVEadWDFku44iyyzaty2U84RENkP6gesk85VHEr8b6kfwJ5rLZanb6g3Ge5lC5NDaDs0kuZkEq/eZT", + "+EnPcrOSYgeuly3z/RYk+reJWvLG6r6iEbGgTB8YvS4wehkFeXd70JR92dBoKe+yDuh1U8ltWda34x3W", + "1mFWdNVzS2mibit3pWVH5NqbPgjrXpVrWnfFoA0XCOVQpst0LaA9t/KEtIssr4b8udFsSRKUe999vvds", + "vyXc9p18nSaF/b49m4t4hUezYaVO27jNnu89f/FiZ3fvxfaNHFQuOrRhfZoiRIvrU6mtXHGa7UEk1eBG", + "gzLxof4hNcSIlgdUqpN86wF9XbF1m4IL8r3ZdMkZFVfS3bOUPaDtfIwrtKXDksqV13ZGG2Q6JWBUjg3d", + "evlgKhmCrcYQ4AQHVC09DhN8ZUo+Zq9U4N7aeNPKg/WQ1LZtoYG05JLpJE+i2HCdo78a13qFF563Ru2X", + "6aTJjf+22qtx4uc+oOIVUYsbmrywaN1dkM3nCstSrJn+O4DoSRcnXo+ZNW+shp6qAn7AJaAtTlGIpPBB", + "FlbOP/tRcfkry1lw+5aU5CrFVx2hzVvwRja050T2mNDB+kyYinywB+DtvhpPivU0VhYsKRXfyE/dm/fb", + "ItmnDhaanWA376+Q9nCTD6vAWMCPdgyW5Hnb3RJLNHBTIUzXY47wiPSyOAcbw4tkavyres9brEVPHkZw", + "yafTMuDTXjNAICDzQci26wUrReJEdRG5BjOdhDV0OVeHfU+OOogLNOoM41Gn4gT0JkHE+HpsOyjnKg9W", + "IfZllaSqg5RuBpOIB5emVIQSlMg+GqCYYCZRymDzV3yUw8FqX1u3kxTWJsPHI+aGuCa2YEwTMscLCrCs", + "1kM1K8WxkGuqJMTbQDsHKORg35brZNkZ6tdMisJBPmk4dDBb2oZ1g/o9zlxAUP4umEtTqM7FPhPBuzb5", + "Tkvst29Pu+b+ByI3zMBK4SFuomYEWkBmXVQwRvPf/eFXk4iMYdxVzMq4TsdiKhn4qQWRREkLYpezQ4UJ", + "UMBTpqpglnG7EM5yxHv9SEoZxErY2zOAbLC92+qaIQlsBeO6e6nE6Ldg7krYpaW0L+5yx8fCsCnAM+f3", + "vL+z7vXqACTCghRL3Jl2imFxxuc6lopbzP9sV4/JdUBIWIXj8b/SNtTQfukNNfwN25z3rPqcfRvCxeqz", + "6z9c7DmMtYnaxZBIxlkPsm3dktrMWIMaYnOvy4xWwt8rZKCOfeBHvhfa5E2R69W0fkOuFYAEhmmkydvE", + "ulZU2cNoHcVvnXTRtKG5WF+s9QFqS5hwvVtVl7CRfo9SYML+fP9FJXzLcU6Ue/fc8k1zrdUSvHHJI+gC", + "JN0r5StKwztdZE90NIw3Kzy3O/e7QSxAV8t8EYZjMk4EmdLrFdxiXjCWcDkbOd85pXq8Em3E+BrtPkPB", + "HAtZGTujs7mKluX7y10PGMCdSq0IoghTN6hcnK+m+7AeLGCXs9i6Txs+L6Tu+ysqk3C8CsbuKHvNXccm", + "eAlum0Yf67Od3cFgZ3twKxy7+yr0XGinKQWh8J29JylF9RRbyBK+6tXArgSFvLGMTFIJguMDCFROcEBQ", + "RKaA8pJVYVx7WNS6Xj14q0HZPPKM/91C2XVzVxhloOisKwsB6KbRcQFQZeiA4vP6sFdAwWRiJqhhwnhy", + "FHZ6g/33w52Dvf2D4fAhgO8yIjVFxz77PLx6Fm3j6W70fPnsj+H82Ww73vEaXg9QU7yc81ktMW7nkBBR", + "LfNWLY8oSUQZ6cksonx9WscKWWCCNNbu/5s59s0MVioL5+VJFnUGrHLilDjrkXAy7OhX3k5Uh39yvHrY", + "twrRrg7Ez2DVoQA/tRsMALEO74r6mbKW586HwoutT56VaQPrzh5flidsbe8qN1Dcx88lwVjaYatO7Pqp", + "5vGOzrigah6vPh6y1zIMQYgz+yxVWMZl6KOTGYOijsWfs7CCopmkP+50O9Hn3fKesb+3R+iwmHkZA9ql", + "LqoBLa7doWboairAK7lpIUzkn7bG9Zh/GvaGLyD4Lfq8+9Og96KP/l4IwusaahXJN3Rvl34dtKFhsdKF", + "Q0gfvrhRhJqj5yoO+pX66jTkB7FF07M8nldEc2eFy0gqLXD+uLbGFRiBRo3zrqqdPc3GRS0pJBFe+uqg", + "F1yxsmIfFpkMTciMMtnGM7szyFyze/Go00eHFoMSrNW8YGqpeSgFWeATGsckpFqpNMZ9c8TndktvW9V4", + "uBkwsfvKo571/frZi/U5pOsC1Ncdk/07JCzdydxtZ+KuSm8Gz5mzSaEECrzYRXSKMKvU4bFVi22mIWSO", + "AMLMgQNUyVnWygCZK37OE9JFM65QnmPY0qOWsmbPXzZ+cg0e1RVJxYYhtu8lYzxDL6GrxNfJMUoED9Mg", + "T7CJYNB5SrRIK2COK7T69SFMD+nQgMw1qO+51qHR5MFo54FsWu+K91EzbPNSDwfrl/pBvCDdTpqE62WY", + "eamdBLsR1uialA2PT6ZM9oomWJjMpxYS/V2RgnUj13iLA60SpYm7QtE8Veckz4UKXCL4YPeOSUT0MVVv", + "BPEozKNKqcyl6HqROtx/Pm+6xIQ7p/pAfiUk0bYKAERAfzFmS+/AqkVW0cbAVc6S5kqrZ7DKLbXKg3u2", + "VhNrXKr29WorXm2TUF4sn5yBbt5vsVr75dpS9A/hh/uWStpbe7lQgVdz4H8ZXKnr32TMADwjiyrn9a7v", + "At7HFu+tZdwEGVfNmim6mQ97/23cymjcP9j66W//d+/TX73u5YrdLInohWQKoUSXZNkDvHukbfR+GTAN", + "sHq1Mj2zrEJwDE6j4JIYJ1WMr4vj3RtkQmP5Bse1KUAMVkxZ9u+1E/rbvzVHMBXI+AHk5FqWvTOU9UPU", + "CVLcHUcbMREzV8nYBd5v9kcMYvkvyVKiQjECq9I4Rv2LzD7RKjo4LXGELowa2CdscYEmFGq6yBHTVi0O", + "ApJoa8KCslNTjI+D9BEER8V2bFEElyhnrxxNxABBH09raHtvP7z/+e2HN8fjt2cv3xyejH99+V8QxHHV", + "Mz2EPc17u3v7tgRfkZJDzxLfAfj3Tih+PnYzWGAe/oKkFKhr6FGYqYSUUhdQUHgZbZA4UUtXbMjltmze", + "DJvsMGvQG852zyjsgxf3UXTmw8oqMwse9bRG3QCi63VgGlp4w7GhKRPm3mlybM88ZcDPrTdxRmfY48v2", + "ViC9j+IwbkBrQeNq699Y2sEfHH9cRRg20sCQqoKIW7FLpeo1x87HWpEa57UeyxEZKbPpJbQQsFXOJYmZ", + "2rJFnHwpraE+eVcnFOW7zCEW9eCj9XkyK1X5wswKI2lem1OnsVZ06hUEOtOkuZoTQQoLAR/kyK03JJlN", + "9miRKG1KrSRE5IGQLlMEap8LCtkjmbPBkSBLCKp7YFcj857i66wH8N5jWbvjgnnkGPnD1z+POpt99M6V", + "KqVT1wQMo2JP+IFQy1y0iiaOq+qLUeSq+rzN+96NZ2XVCunXtLcqzJn3UWJNHz/+HVP1iguwQJrTkh8c", + "TxWsm5AIwGWpoqW2ghqlMQnHWcHmpv3vajSbnOSsHHZexslZWxiYWAu59aV2XOJsPoY6pTU5SJAKqpbn", + "UNjVRAgTLIg4TM2Gd/Vh7c95x1AV6etX8FNOPVkIrwkjggbo8OwE9mOMGSjp6ONpoZKJKWpTw1AD9fLt", + "0Ym1cB0MH1gsVAHruWC+w7OTTrezIMJYeZ1Bf7s/gM2cEIYT2jno7PSH/UHHVPSFKW5BEUX40yYYZpbS", + "SWj1oJ/NK/orgWOiiJCdg989iXoQyAYvg76LZwWLJcFUWJMliSB90LAK1d8CsK47Sg/MeWwL8rZ20Em1", + "tMkUJHlrl/UTqJOwa2CK24OBhRlV9uCFVBATf771DxuMmPfbSp8D8nhQZmsWhdMpLcm/dju7g+GNxrNq", + "GLBjfd1+YDhVcy7oZwLD3LshEW7V6QkzGV7IYIXZQJviPgMWKu6w3z/p9ZJpHGOxdOTKaZVw2aQME4kw", + "YuTKVgT9B5/0kb1+gLIxcs7TSEsTZNLXnKNBYdGffUZYBHO6ICNmz+k4jRRNsAA3Qoz0+WwMpvLWMF2b", + "1c+ABX7m4bJC3ay5Ld1czzmdcwJX0xIkGQMO8bip3G/ubqaMaTGJJbG1NLK6l/VoHi0uxzLgvnyi94Rh", + "pnoyIQGd0gDBy3r3Wo+2t8FW0Hxa4MGyEAGIWc5Ds73pz0mFqh/+dO7j7Bmy5C2rEwyugYIoDXOdy6VJ", + "YTHBUeTFbppFfIKjsaHPJfGoqK/hDUuUYoEUp9wwHhJT7CJZqjln5u90kjKVmr8ngl9JIrQKZIuYWVqT", + "0JQtM6x7BVifMRQSMyVSdZ9bZohbXy7J8mt/xA7D2JW/leYTHEmuT01bdNDmBZgtbXjXX5alIa7kKJWK", + "x5alsgqM+TB5qpJU2Tt1SZStvAavU4mSVM5JOGKKoy+CzKhUYvl160ve41ewXQgONZ8UXjFT2vpCw69N", + "o5ZjrGc/hlc91h8BAow6+nQZdfTfM4G17ZLKOThRJDhOZsUl3cjwdLReuFmlcIAZSnhisIiAqeZYs1yp", + "DYhHx1GEFGwl963WNmElG+Zj04vjSWNusUkGrWwjytDpz4XNNNh97t9PkgSC+Bwc/3n+9g2Co0qvgXkt", + "d1iZS22mT1EUpqDJQ+/9EXuJgzkyehPgzY46NBx1Musi3ISxptJGaPd6oOL+pIf2k+mmS8Of+n3dlNGe", + "D9DvX0wrB3ovJfFY8UvCRp2vXVR4MKNqnk6yZ5/8BG1K0TwvCQK0YWT/pqtBDFBR+TFozg3MQsStrI2W", + "CKNcAhX9KBPKsFhZQNlDektBbcrjmSwS48sIfLejzsHIeW9Hne6oQ9gCfrMu3lHnq58CVoluBjc1NaSd", + "rp0x0f5gsLkeO8HS16NCl17U2+9rTfvavjfFwypddcXDTM4hM+sVNNXAjbr1CJrPzzh09SV/qHhrVDzr", + "uSgob/B98Rww7BsRY+BWNDBtz0ZOA1tpnRi2gIIJYHE4pBNjcFCnweXMWzQ/quZ83azYbdplAQwxcvy3", + "+wj8B/1mpQhNvy8eq18cAc64A65/YuwIi+UYseu3iF8T9T1w3OCxRKkFRf+W/PtU+Oc1sXpfTrSKNNsi", + "C3ff5MdzgmQTaVsxL2tb9RzG1DsnTKGX8Gvf/tdZPJAtfRHx2cUBMiSM+AxFlNl7wMJtkT4ULS3hI5Nv", + "kn1n008cmOaGOT//+T//C4OibPbP//lfrU2bv2C7b5nESQDfv5gTLNSEYHVxgH4lJOnhiC6ImwzAY5MF", + "EUu0MwA1MxHwqFhSyeomcsRG7B1RqWCF+1KDayltg2B6MJgPZSmRNl9Hv0inFnTLOJg9Jrzby4aUj7qj", + "u55EZ5hBYQL6VHQ8AFmitvyatb86fu+ZmXPJf1b1ldc8puvliyLXynBvzwzwhgIGSOzbd/DAThptnJ+/", + "3OwjsDEMVwCwGmjMeTNWee7/kEnrZZKRKGWBAlQ2sskksa32/x7bd9o5gG2LfyYPsMXavIEL2Lg8IHfd", + "rcAPW6GFO9hPN+ca9vlnj12WZrOD9vbzLXbh4phaGcL3t86O9+o0N08KJPsWJjDacNHw4EbkAp0dnbia", + "tpvfjOkf5dTQM7W1+rKjA3EG2GuPZpYdcTaNaKBQz40FCjPFJDPVygzyVMTBOztqhN28qhDGxfNtq4TI", + "13jSZeB8+ZH38KdHpdObHCM5zHLOaz9OknWsc0xlwPW3BW7pBTgBQjr1JdunRS5a55AywfXZkbNSXbLi", + "+eTYbcjHc03ZrlNWPRseQSgeVwTiNxSEleLxBWDyp8TNH7JVdIgUKzxX3xdrDh5PC3psL5aPzZ+SGyus", + "kE1LQRPU3XiAvibKhHJ3HnChbQ+eiZ8T4Xa1qyMBs86mZT41hVXNhOBCerXte2JeaWf6mvb+TJYvkOcm", + "Gosl+Q8VpYWxm9NqlYF7YkuWP5x9Cz3cyLy9v3tey2AeIkOwycR5rE3dXSyXLNj8U131PsppZoj9JA+z", + "szSK3I3HggiF3h6dmJ1VPAO2vkBY0nrd3u22lcfBh3e/9QgLOMShZTFUfiXKPrlnDd8smJnKDzZpYxOa", + "tGjqzrMmDecO62/CBZGJcOxT/u/bryI6EVgs/337FY4Sysi/7xxGWBGpNh+MWQaPJZofW+N+wsynFW5a", + "JhqIJgb13tdpqNlbLZVU9/6fSk81k76RpprR9Yey2kZZLZJrpb5ql+JBNVbTxze6ksmYzUdteOTiE/9k", + "murjevksRzqgaCrL1x62yB4X4OeFR5ShVJInGEBJM44rHhst3dX5hlx5fDjWPTnuAiGhLgKgNtkEkUdy", + "XrtxPLpya/t9fM/1YTyhs5Snsph7EmMVzIm0yUoRKQvgp6Z258dzo+L9HXPp4DGPjkfXq3/w/QNp/NUF", + "NcLb3ECt0/ndW211fvu+1vlNCrXNXbPQUl0HO7jZEFTokqjbsnEp17we7Ogbl88WQR+0oZKbCwgsiIMR", + "+z/a/vhdERx/+sklyaSDwfY+/E7Y4tNPLk+GnTpWIQwqHgFK7OGbY7j2m0H2OQDJ5il51XGYyhPAeg46", + "51/OQMpvPttbSI4Lf1hIrSykArlWW0h2LR7WRCrDbz26jeT4zUdwC2Lyw0p6DCtJptMpDShhKq95WgsS", + "syWTn2BuGbP3Q4XgjtJB29pKyjblGgU0Lwvw6IE9JzkO4mMbR64CwdOMkeeJhfS25kh+GDbbI98bPwwe", + "Vzg/vh3ylFnMKPx10iVap/SV0gaMyTiFspAoRwiBqE8kbP1H12If5RWsZZokXChpcCpBATZI9nOtAPsw", + "LcswlT5cSsBipER2RwwqFejHJpd/65IsDQol5SwDnKzWY/XlXpVRQL/pNrp/HcsPcdpKx3rkbWxBq7+d", + "jvXNRMejaFonpVoAG9nGAINyQrKdzLPkPvqZstnmk4pANcIqm1sBz8ijam1BpT+L67sls2qyTQdtAdrX", + "lp79Fzxx65P0ae0Oi7ZAQBRSPGNcKhoUK+8W4UF/nNCtT+jVlPVy89SWSPcb9K+4uGx7xHnqij2Bk644", + "w+/Ql6CHB2hg396lAMa2OQ000zz6KVgrFvctUzBo9VwMojSEqvT2QHSq5FTweGx/NHi1eldYNFBwUQS2", + "1W8tbHTvj+AwesMVonESEa3FkxD1DDfp1bSqv4Obp7JQWvFmwlBvm2JCjAGjk640kRWRcLnmFmwD7tnr", + "y+WVmhGfrQfByDp3iA8eFIwRM3D4xGHnX6BMyELxLhKRQKGrOQ3mgIgBBb2goiuAVeAkucggsDYP0GvY", + "qUUkMOh8QxKhDaGAM8kjYoAuFnF8cVBHbP14egofGTAMg816cZCVXM8OCKnfKiJcZDWP3ljcjg3NSYJH", + "kVnRC201Fua3abEvcoiyEfPhYDByZRukU3RRgMS4aMDEcAL1Nz77ZtpWtxlY0sxFcSSAcIY3CQs7TRcx", + "NPKjYQwH3nowLZE5zDAeGJijNpjf+CwDtSyxMk6StuxrhwlcvIjjFTyMNgrFWaUKear+JlVIhICPLXc3", + "MTfawIH5h8KXmlFtYaGsvC2wn/e60aDMeUmlhWqhio751yKOO92OHU8Bne4G2vsahJNqg/VrMb0yBRiT", + "H3r3TQBKysK+gFBSOTls+f9mlfudeeFP75+1hAr/DF6W8n1WPgrK8gJQAiq4uypcTwrpABaypouZwki+", + "PeJm2ZOF4qHtrrdqZUe/A6N13a1XVkMyK3D52Ndf9RE85SQYWZvNlItqevy6e7HvnpHub0lqU23DIT94", + "8+buuVaMmaQraolCKVQJfj6orwm4zsGcc1lg+wmZ4wXlwiKwW69rxpngsjDWo42eu9CsemH9txdWPT+w", + "viaEi49sH3343Mbc+b9wj/IvXhWs7Uzid51KDSiQEmE0EZRMUYJTSbS2lMYEmQojFsib4GDuqoX3R+z9", + "nCBbH7PgQMjKKVOJLobxRRdNUoUiLGZg7ZiHJpJOkIDHMWGhqXk7YnOCF1SbagJFWBEWLHuSQA3kBckL", + "mGjT3d5QmlLbWZXVLnLFecHBcFEovXuBEkGAiYy5zEp1bkdMpOw/DHKlbvbCDfQCEanwJKJyntWKCHBI", + "WOCFhTz/vsXY/Ttxz4mqV6f9JneWt5Kl3/ISs+jLzOqDfxf3m08sUIsLV1mzhZhfofTKZtOwHPl4nlfk", + "/Rfc0maubo7f6GYmI/GqXfx9XMmUSvL/uJZRdkuGqemOlMvW/2nvWvI60ikrXbdYn+xtL1yySggZmW8k", + "87a+uD9PbuEj+04kYbfRsG/C3M4n/T2IXEvVW8ncb+QctL6kglfsG4pgO6hvpz5xUZBy34UYNhsuk8ZF", + "maMEBpuKsx/CuCqMbXjAbYWx87jWLsAL4pmyXhLhJrmcV633C2DrEPgXjX6tzK4gCL+54MtvBB5N2J1k", + "4s0IvAQvI47/7PcyARfCJHTacsRPB1Cs4AssXDBtgMetm0mIrssm+Xh6utkkJYRaKSOEesISolzWNIg9", + "1RrfLogQNHSlI49Oj230KpVIpKyP3sYU6jleEpJAoRjKU4kgM7ev5+dSW+tF8Eo5rN0OYUosE06ZWjuK", + "/NWHGczXW5XOe2Q5aSEV//SXx+CFf3pCCmSHVlfsBFZbkQqrxmA8F5xGmal3qbUtPOGpbl1LFldodwZn", + "25RGRC6lIrGJzJumEWwiAN21NZnsdyajtIuokkjvhy5k4CVExFRKypkcMVv+PSFC960/h+K/eZCR13mv", + "cCY1z4zo+z4C2PRgTMwWVk1UA2gBqAPaOehs4STZgnLR/iApO7w7DOkVRKQhuYwnPKIBiii7lGgjopfG", + "6EALiSL9x+bKkLYxfHffFaduv7M0pU/YlHuLchiezZj5z5GEVBZr7hLxyYm116S4WZz8gYX2izW5Vq4J", + "gqOeojHJkt9RqmhEPxtRpxuhUtHA5NXkqZdQhNlmX47YKVFCv4MFQQGPIhIo51zZSgQPtkbpYLATJBRQ", + "SnYIDA4EXvPjGHo8OvsA75lC0d0R0/+Aht8fnpmb2Cm2PoLCQBlRV1xcopOtt2uCfM+BTP/CUXJmgitz", + "IL0L/uP67uaZzY17SDZsUZ6sMoB48qcP47Qa3A9vwdP0FgC0RDabjZnAASjFcp6qkF8xv2dgwaM01v8w", + "f5ysAyhROJh/hFe/G23XDGdtN26CT2JT2jmFxBQN+iYXFIZgTzW+VBPOTQGUmFLknvcUOFR/Ru6+f6d8", + "kY7f4dWkpagryPXd7K3HPvnsGBzuVpEeT2WbG05zM1F8tffpCtNm79PPEQ8uJUqZolEJ1EDbbYADqn/M", + "cRvtxR+oCZAd6UqJI3KdUAEINhV4BET0jCXCSBERU4ajLZizaQQQKJ0XCy84hSTlIKKQJkZDghIeRYCy", + "czUnDOnZgKPKNVC4p5W2AkTxneIVo+JoQgIeE4fKuekz3f6OqXrFRRli83uRi+8L9Nfz0VPV81yDKtrc", + "451QRk/xNYQ1h6m9JnYj2njN8x+NK6iLYG1GnZ2BHHW6aNTZjkcdvQJHGFyoWKE9FFOWKiL76Nj4tyAN", + "dX+AJAk4C6UDB3UevJ2BbEpKNWzZkOG4D989ptpjuQpI+c524hMP+j2kv4cEG7RR3HB2T4Zd2HQh4qmC", + "AG63r+xbIVHgHtl89BvYwh75Ydu3keR/t9u3JKNglbW4LCy9kewZfORar5tLqphzmaNOogAnOKBq2UU4", + "iniQew9Smd0O9LKhTATBl9qG6o/Yuwy40iZCoKOzD13nNEMhlZemBesX66O3CyJkOskGh0AaGA8eLAYJ", + "R0xxFOAoSCPNt2Q6JQHkMEQ0pko2+NWyoTxkGcS8E8/Cu4cZbM3Tcib5eQJWL2cLWeG4LbPUW4IEEaZx", + "0alUJQ6ovnClC27fiW6U62N4GtnrrUBwKZFtqkciOqOTyF7WyD56r1UOHJMRSyLMGBEolSbuSA+9lwgi", + "ZWoSY3QDUGfWcFQX5UAnieDKuokjzoU0nl3N4R9PkVQkWcFm70zLpzDnB4IJNo3bnr6RwVAZQ/OxZF9B", + "ekEMpxiCaz7Sx/Q3CPYxA/rWcMJPZeO/F3Q2I0LvCmyErLkaNdvakdNs+lKmRyNG/nn2VjuM/KzVQjR3", + "IdJ5JVDF2L04BgX6Jjewns4vaSOWiX10s+yLX/VHLfsuR/n7B2Ef3XGWf5bSY+eF4Oq2yPo5hz81kPvC", + "yEtbtZSgsB6OoHVGwkNmCLTGHfhmcANPGWUAl9IOmuAEvj9GGDxudtxjw2w/bd4qoQSUCus0pEqth+/8", + "LjjwYXA7v3F26C1wO7+rfCXAXfx2eaPfVaZSyQ/oiof86ZE5HypBycBzAoxFU4KSkXo2kGClofTRvtPO", + "TLIt/pk0eHv3fAP93ZH9h9XfwmQoEMvvsjO50Q63hcSJWrrLRT6tXABK+hmSMXzAD1kMwcPhLdziev3+", + "2MPxaePl+o96Wo92f58XHT45fvpFtIp7rnSwbOlTp4dFMKcL0ux0L+9gS6JEkF7CE7hcCQ3BLD3cWaaw", + "6M8+I9u8xaqy/0LUQRyTEIVUkEBFS0SZ4iARTB9/kUhwbQnAcy6WPmd6cee+Ejw+tLNZcx7aPWWdYfmd", + "b7zshVjh3sJJmxUutDvctLu7bS3wEGXo9c9og1wrYRB30VRbPohOM5KS64CQUAJPbhYHPBw0eDbpZzKe", + "TdqMcgV28luLTY2CVCoeu7U/OUYbUGxhRpheC63qT0GTTQRf0NAUIs2JuuCRoeqwgaA39btqpSKrlOGM", + "CzO4b6LDtDmQZp9pUhYLJnShc9CZUIZhcGtRist7yiRU6f4whbSGfO84zun8OMKs5bfhjB3NidrIcURU", + "nBtovM0fx9xTPuaKganuTCuddu1KRbaLVW0ZQvoQgLlZHPPjuq0/fj/hlVQ+ychK6zpfZAZpk9v8+2LB", + "weOdD4/tLv/4hMPxXxNnfBdc5dCAbtHHML/xAEcoJAsS8QSqSJp3O91OKqLOQWeuVHKwtRXp9+ZcqoPn", + "g+eDztdPX///AAAA//9DeBEen5ABAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/system/guest_agent/main.go b/lib/system/guest_agent/main.go index 46ee3d7a..a968e89a 100644 --- a/lib/system/guest_agent/main.go +++ b/lib/system/guest_agent/main.go @@ -53,8 +53,10 @@ func main() { } // Create gRPC server + guestSvc := &guestServer{} + startResumeNetworkWatcher(guestSvc) grpcServer := grpc.NewServer() - pb.RegisterGuestServiceServer(grpcServer, &guestServer{}) + pb.RegisterGuestServiceServer(grpcServer, guestSvc) // Serve gRPC over vsock if err := grpcServer.Serve(l); err != nil { diff --git a/lib/system/guest_agent/network.go b/lib/system/guest_agent/network.go new file mode 100644 index 00000000..66b71bad --- /dev/null +++ b/lib/system/guest_agent/network.go @@ -0,0 +1,91 @@ +package main + +import ( + "context" + "fmt" + "net" + + pb "github.com/kernel/hypeman/lib/guest" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +func (s *guestServer) ReconfigureNetwork(_ context.Context, req *pb.ReconfigureNetworkRequest) (*pb.ReconfigureNetworkResponse, error) { + iface := req.InterfaceName + if iface == "" { + iface = "eth0" + } + + link, err := netlink.LinkByName(iface) + if err != nil { + return nil, fmt.Errorf("find interface %s: %w", iface, err) + } + + mac, err := net.ParseMAC(req.Mac) + if err != nil { + return nil, fmt.Errorf("parse mac %q: %w", req.Mac, err) + } + ip := net.ParseIP(req.Ipv4).To4() + if ip == nil { + return nil, fmt.Errorf("parse ipv4 %q", req.Ipv4) + } + if req.Prefix > 32 { + return nil, fmt.Errorf("invalid ipv4 prefix %d", req.Prefix) + } + gateway := net.ParseIP(req.Gateway).To4() + if gateway == nil { + return nil, fmt.Errorf("parse gateway %q", req.Gateway) + } + + if err := netlink.LinkSetDown(link); err != nil { + return nil, fmt.Errorf("set %s down: %w", iface, err) + } + if err := netlink.LinkSetHardwareAddr(link, mac); err != nil { + return nil, fmt.Errorf("set %s mac: %w", iface, err) + } + if err := flushIPv4Addrs(link); err != nil { + return nil, err + } + addr := &netlink.Addr{IPNet: &net.IPNet{IP: ip, Mask: net.CIDRMask(int(req.Prefix), 32)}} + if err := netlink.AddrAdd(link, addr); err != nil { + return nil, fmt.Errorf("add ipv4 address: %w", err) + } + if err := netlink.LinkSetUp(link); err != nil { + return nil, fmt.Errorf("set %s up: %w", iface, err) + } + if err := netlink.RouteReplace(&netlink.Route{ + LinkIndex: link.Attrs().Index, + Gw: gateway, + }); err != nil { + return nil, fmt.Errorf("replace default route: %w", err) + } + _ = flushNeighbors(link) + + return &pb.ReconfigureNetworkResponse{}, nil +} + +func flushIPv4Addrs(link netlink.Link) error { + addrs, err := netlink.AddrList(link, unix.AF_INET) + if err != nil { + return fmt.Errorf("list ipv4 addresses: %w", err) + } + for _, addr := range addrs { + if err := netlink.AddrDel(link, &addr); err != nil { + return fmt.Errorf("delete ipv4 address %s: %w", addr.String(), err) + } + } + return nil +} + +func flushNeighbors(link netlink.Link) error { + neighbors, err := netlink.NeighList(link.Attrs().Index, unix.AF_INET) + if err != nil { + return fmt.Errorf("list neighbors: %w", err) + } + for _, neighbor := range neighbors { + if err := netlink.NeighDel(&neighbor); err != nil { + return fmt.Errorf("delete neighbor %s: %w", neighbor.String(), err) + } + } + return nil +} diff --git a/lib/system/guest_agent/resume_network.go b/lib/system/guest_agent/resume_network.go new file mode 100644 index 00000000..060cd687 --- /dev/null +++ b/lib/system/guest_agent/resume_network.go @@ -0,0 +1,201 @@ +//go:build linux + +package main + +import ( + "bufio" + "context" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "log" + "net" + "os" + "strconv" + "strings" + "sync/atomic" + "time" + "unsafe" + + pb "github.com/kernel/hypeman/lib/guest" + "golang.org/x/sys/unix" +) + +const resumeNetworkMailboxEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX" +const resumeNetworkMailboxTokenEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX_TOKEN" +const vmgenIDKmsgSignal = "crng reseeded due to virtual machine fork" +const resumeNetworkMailboxSize = 4096 +const resumeNetworkMailboxSeqOffset = 64 +const resumeNetworkMailboxLengthOffset = 68 +const resumeNetworkMailboxPayloadOffset = 72 + +var resumeNetworkMailboxMagic = []byte("HYPEMAN_RESUME_NETWORK_MAILBOX_V1\x00") + +type resumeNetworkPayload struct { + InterfaceName string `json:"interface_name"` + MAC string `json:"mac"` + IPv4 string `json:"ipv4"` + Prefix uint32 `json:"prefix"` + Gateway string `json:"gateway"` + AckPort uint32 `json:"ack_port,omitempty"` +} + +type vmGenIDResumeWaiter struct { + file *os.File + reader *bufio.Reader +} + +func startResumeNetworkWatcher(s *guestServer) { + if strings.TrimSpace(os.Getenv(resumeNetworkMailboxEnv)) != "1" { + return + } + + mailbox := newResumeNetworkMailbox() + if mailbox == nil { + return + } + + go resumeNetworkMailboxLoop(s, mailbox) +} + +func newResumeNetworkMailbox() []byte { + token := strings.TrimSpace(os.Getenv(resumeNetworkMailboxTokenEnv)) + if token == "" { + log.Printf("[guest-agent] resume network mailbox disabled: missing %s", resumeNetworkMailboxTokenEnv) + return nil + } + if len(token) > resumeNetworkMailboxSeqOffset-len(resumeNetworkMailboxMagic) { + log.Printf("[guest-agent] resume network mailbox disabled: %s is too long", resumeNetworkMailboxTokenEnv) + return nil + } + + buf := make([]byte, resumeNetworkMailboxSize) + copy(buf, resumeNetworkMailboxMagic) + copy(buf[len(resumeNetworkMailboxMagic):resumeNetworkMailboxSeqOffset], token) + if err := unix.Mlock(buf); err != nil { + log.Printf("[guest-agent] resume network mailbox mlock failed: %v", err) + } + log.Printf("[guest-agent] resume network mailbox armed token=%s", token) + return buf +} + +func resumeNetworkMailboxLoop(s *guestServer, mailbox []byte) { + for { + waiter, err := newVMGenIDResumeWaiter() + if err != nil { + log.Printf("[guest-agent] resume network VMGenID prepare failed: %v", err) + time.Sleep(100 * time.Millisecond) + continue + } + + start := time.Now() + if err := waiter.Wait(); err != nil { + waiter.Close() + log.Printf("[guest-agent] resume network VMGenID wait failed: %v", err) + time.Sleep(100 * time.Millisecond) + continue + } + waiter.Close() + + if err := waitAndApplyResumeNetworkMailbox(s, mailbox); err != nil { + log.Printf("[guest-agent] resume network mailbox apply failed: %v", err) + time.Sleep(25 * time.Millisecond) + continue + } + log.Printf("[guest-agent] resume network mailbox applied in %s", time.Since(start)) + } +} + +func waitAndApplyResumeNetworkMailbox(s *guestServer, buf []byte) error { + for { + seq := atomicLoadUint32(buf[resumeNetworkMailboxSeqOffset:]) + if seq == 0 { + time.Sleep(100 * time.Microsecond) + continue + } + + payloadLen := binary.LittleEndian.Uint32(buf[resumeNetworkMailboxLengthOffset:]) + if payloadLen == 0 || int(payloadLen) > len(buf)-resumeNetworkMailboxPayloadOffset { + return fmt.Errorf("invalid mailbox payload length %d", payloadLen) + } + + var payload resumeNetworkPayload + if err := json.Unmarshal(buf[resumeNetworkMailboxPayloadOffset:resumeNetworkMailboxPayloadOffset+int(payloadLen)], &payload); err != nil { + return fmt.Errorf("decode mailbox payload: %w", err) + } + + _, err := s.ReconfigureNetwork(context.Background(), &pb.ReconfigureNetworkRequest{ + InterfaceName: payload.InterfaceName, + Mac: payload.MAC, + Ipv4: payload.IPv4, + Prefix: payload.Prefix, + Gateway: payload.Gateway, + }) + if err != nil { + return err + } + sendResumeNetworkAck(payload, "applied") + atomicStoreUint32(buf[resumeNetworkMailboxSeqOffset:], 0) + return nil + } +} + +func sendResumeNetworkAck(payload resumeNetworkPayload, stage string) { + if payload.AckPort == 0 || payload.Gateway == "" { + return + } + + addr := net.JoinHostPort(payload.Gateway, strconv.FormatUint(uint64(payload.AckPort), 10)) + conn, err := net.DialTimeout("udp4", addr, 100*time.Millisecond) + if err != nil { + log.Printf("[guest-agent] resume network ack dial failed: %v", err) + return + } + defer conn.Close() + + _, _ = fmt.Fprintf(conn, "stage=%s mac=%s ip=%s\n", stage, payload.MAC, payload.IPv4) +} + +func atomicLoadUint32(buf []byte) uint32 { + return atomic.LoadUint32((*uint32)(unsafe.Pointer(&buf[0]))) +} + +func atomicStoreUint32(buf []byte, value uint32) { + atomic.StoreUint32((*uint32)(unsafe.Pointer(&buf[0])), value) +} + +func newVMGenIDResumeWaiter() (*vmGenIDResumeWaiter, error) { + f, err := os.Open("/dev/kmsg") + if err != nil { + return nil, fmt.Errorf("open /dev/kmsg: %w", err) + } + + if _, err := f.Seek(0, io.SeekEnd); err != nil { + log.Printf("[guest-agent] warning: failed to seek /dev/kmsg to end: %v", err) + } + + return &vmGenIDResumeWaiter{ + file: f, + reader: bufio.NewReader(f), + }, nil +} + +func (w *vmGenIDResumeWaiter) Close() { + if w == nil || w.file == nil { + return + } + _ = w.file.Close() +} + +func (w *vmGenIDResumeWaiter) Wait() error { + for { + line, err := w.reader.ReadString('\n') + if err != nil { + return fmt.Errorf("read /dev/kmsg: %w", err) + } + if strings.Contains(line, vmgenIDKmsgSignal) { + return nil + } + } +} diff --git a/lib/system/guest_agent/resume_network_other.go b/lib/system/guest_agent/resume_network_other.go new file mode 100644 index 00000000..dfa74e07 --- /dev/null +++ b/lib/system/guest_agent/resume_network_other.go @@ -0,0 +1,7 @@ +//go:build !linux + +package main + +type resumeNetworkController struct{} + +func startResumeNetworkWatcher(_ *guestServer) {} diff --git a/lib/system/versions.go b/lib/system/versions.go index aa07108d..7743add0 100644 --- a/lib/system/versions.go +++ b/lib/system/versions.go @@ -20,14 +20,18 @@ const ( // Kernel_202603301 is the current kernel version with expanded nftables/raw support for Docker bridge networking Kernel_202603301 KernelVersion = "ch-6.12.8-kernel-1.6-202603301" + + // Kernel_202605291 is the current kernel version with VMGenID support for snapshot resume detection + Kernel_202605291 KernelVersion = "ch-6.12.8-kernel-3.0-202605291" ) var ( // DefaultKernelVersion is the kernel version used for new instances - DefaultKernelVersion = Kernel_202603301 + DefaultKernelVersion = Kernel_202605291 // SupportedKernelVersions lists all supported kernel versions SupportedKernelVersions = []KernelVersion{ + Kernel_202605291, Kernel_202603301, Kernel_202603091, Kernel_202602101, @@ -37,6 +41,10 @@ var ( // KernelDownloadURLs maps kernel versions and architectures to download URLs var KernelDownloadURLs = map[KernelVersion]map[string]string{ + Kernel_202605291: { + "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-3.0-202605291/vmlinux-x86_64", + "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-3.0-202605291/Image-arm64", + }, Kernel_202603301: { "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.6-202603301/vmlinux-x86_64", "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.6-202603301/Image-arm64", @@ -58,6 +66,10 @@ var KernelDownloadURLs = map[KernelVersion]map[string]string{ // KernelHeaderURLs maps kernel versions and architectures to kernel header tarball URLs // These tarballs contain kernel headers needed for DKMS to build out-of-tree modules (e.g., NVIDIA vGPU drivers) var KernelHeaderURLs = map[KernelVersion]map[string]string{ + Kernel_202605291: { + "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-3.0-202605291/kernel-headers-x86_64.tar.gz", + "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-3.0-202605291/kernel-headers-aarch64.tar.gz", + }, Kernel_202603301: { "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.6-202603301/kernel-headers-x86_64.tar.gz", "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.6-202603301/kernel-headers-aarch64.tar.gz", diff --git a/openapi.yaml b/openapi.yaml index c6bce872..e6d03230 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -690,6 +690,13 @@ components: Optional final state for the forked instance. Default is the source instance state at fork time. For example, forking from Running defaults the fork result to Running. + wait_for_network: + type: boolean + description: | + When the fork result is Running, wait for guest networking to be applied before returning. + Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. + default: true + example: true ForkTargetState: type: string @@ -875,6 +882,13 @@ components: Optional hypervisor override. Allowed only when forking from a Stopped snapshot. Standby snapshots must fork with their original hypervisor. example: cloud-hypervisor + wait_for_network: + type: boolean + description: | + When the fork result is Running, wait for guest networking to be applied before returning. + Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. + default: true + example: true SnapshotScheduleRetention: type: object From 05bc363918c524f2b5d2bd2d682db9e7ef1269fe Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 1 Jun 2026 13:24:02 +0000 Subject: [PATCH 02/16] Isolate linux test temp files --- .github/workflows/test.yml | 16 ++++++++++++++-- Makefile | 2 ++ lib/instances/test_network_config_test.go | 11 +++++++++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eec2948b..333f3655 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,7 +36,7 @@ jobs: local sleep_seconds=30 local n=1 while [ "$n" -le "$attempts" ]; do - if sudo apt-get update; then + if timeout 120s sudo apt-get update; then return 0 fi if [ "$n" -eq "$attempts" ]; then @@ -53,7 +53,7 @@ jobs: ! command -v qemu-system-x86_64 &> /dev/null || \ ! qemu-system-x86_64 --version >/dev/null 2>&1; then apt_update_with_retry - sudo apt-get install -y erofs-utils e2fsprogs iptables qemu-system-x86 qemu-utils + timeout 300s sudo apt-get install -y erofs-utils e2fsprogs iptables qemu-system-x86 qemu-utils fi go mod download @@ -69,6 +69,13 @@ jobs: sudo env "PATH=$TEST_PATH" bash -lc "command -v '$bin'" done + - name: Create run-scoped temp directory + run: | + TEST_NETWORK_TMPDIR="/tmp/hm-net-${{ github.run_id }}-${{ github.run_attempt }}" + sudo rm -rf "$TEST_NETWORK_TMPDIR" + mkdir -p "$TEST_NETWORK_TMPDIR" + sudo chown -R "$(id -u):$(id -g)" "$TEST_NETWORK_TMPDIR" + # Avoids rate limits when running the tests # Tests includes pulling, then converting to disk images - name: Login to Docker Hub @@ -106,6 +113,7 @@ jobs: - name: Run tests env: + HYPEMAN_TEST_NETWORK_TMPDIR: /tmp/hm-net-${{ github.run_id }}-${{ github.run_attempt }} GO_TEST_TIMEOUT: 600s # Docker auth for tests running as root (sudo) DOCKER_CONFIG: /home/debianuser/.docker @@ -122,6 +130,10 @@ jobs: export HYPEMAN_TEST_PREWARM_DIR="$HOME/.cache/hypeman-ci/linux-amd64" make test TEST_TIMEOUT=20m + - name: Cleanup + if: always() + run: sudo rm -rf "/tmp/hm-net-${{ github.run_id }}-${{ github.run_attempt }}" + test-darwin: runs-on: [self-hosted, macos, arm64] concurrency: diff --git a/Makefile b/Makefile index 08c14fa8..770d44e8 100644 --- a/Makefile +++ b/Makefile @@ -295,12 +295,14 @@ test-linux: ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries if [ -n "$(TEST)" ]; then \ echo "Running specific test: $(TEST)"; \ sudo env "PATH=$$TEST_PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" "CI=$${CI:-}" \ + "HYPEMAN_TEST_NETWORK_TMPDIR=$${HYPEMAN_TEST_NETWORK_TMPDIR:-}" \ "HYPEMAN_TEST_PREWARM_DIR=$${HYPEMAN_TEST_PREWARM_DIR:-}" \ "HYPEMAN_TEST_PREWARM_STRICT=$${HYPEMAN_TEST_PREWARM_STRICT:-}" \ "HYPEMAN_TEST_REGISTRY=$${HYPEMAN_TEST_REGISTRY:-}" \ go test -tags containers_image_openpgp -run=$(TEST) $$VERBOSE_FLAG -timeout=$(TEST_TIMEOUT) ./...; \ else \ sudo env "PATH=$$TEST_PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" "CI=$${CI:-}" \ + "HYPEMAN_TEST_NETWORK_TMPDIR=$${HYPEMAN_TEST_NETWORK_TMPDIR:-}" \ "HYPEMAN_TEST_PREWARM_DIR=$${HYPEMAN_TEST_PREWARM_DIR:-}" \ "HYPEMAN_TEST_PREWARM_STRICT=$${HYPEMAN_TEST_PREWARM_STRICT:-}" \ "HYPEMAN_TEST_REGISTRY=$${HYPEMAN_TEST_REGISTRY:-}" \ diff --git a/lib/instances/test_network_config_test.go b/lib/instances/test_network_config_test.go index 0ed5f97c..e1a1b104 100644 --- a/lib/instances/test_network_config_test.go +++ b/lib/instances/test_network_config_test.go @@ -183,7 +183,7 @@ func allocateTestNetworkLease(testName string, seq uint32) (*testNetworkLease, e } func withTestSubnetLock(fn func() error) error { - lockPath := filepath.Join(os.TempDir(), "hypeman-test-network.lock") + lockPath := filepath.Join(testNetworkTempDir(), "hypeman-test-network.lock") lockFile, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0o600) if err != nil { return fmt.Errorf("open subnet lock file: %w", err) @@ -199,7 +199,14 @@ func withTestSubnetLock(fn func() error) error { } func testSubnetLeaseFilePath() string { - return filepath.Join(os.TempDir(), "hypeman-test-network-leases.json") + return filepath.Join(testNetworkTempDir(), "hypeman-test-network-leases.json") +} + +func testNetworkTempDir() string { + if dir := os.Getenv("HYPEMAN_TEST_NETWORK_TMPDIR"); dir != "" { + return dir + } + return os.TempDir() } func loadSubnetLeases() (map[string]subnetLease, error) { From a0e9b45e5c90fe1af50bdae483fb085150accd17 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:24:41 +0000 Subject: [PATCH 03/16] Timeout missing resume network mailbox payload --- lib/system/guest_agent/resume_network.go | 9 +++++++++ lib/system/guest_agent/resume_network_test.go | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 lib/system/guest_agent/resume_network_test.go diff --git a/lib/system/guest_agent/resume_network.go b/lib/system/guest_agent/resume_network.go index 060cd687..8c1108ec 100644 --- a/lib/system/guest_agent/resume_network.go +++ b/lib/system/guest_agent/resume_network.go @@ -29,6 +29,7 @@ const resumeNetworkMailboxSize = 4096 const resumeNetworkMailboxSeqOffset = 64 const resumeNetworkMailboxLengthOffset = 68 const resumeNetworkMailboxPayloadOffset = 72 +const resumeNetworkMailboxPayloadTimeout = 5 * time.Second var resumeNetworkMailboxMagic = []byte("HYPEMAN_RESUME_NETWORK_MAILBOX_V1\x00") @@ -108,9 +109,17 @@ func resumeNetworkMailboxLoop(s *guestServer, mailbox []byte) { } func waitAndApplyResumeNetworkMailbox(s *guestServer, buf []byte) error { + return waitAndApplyResumeNetworkMailboxWithTimeout(s, buf, resumeNetworkMailboxPayloadTimeout) +} + +func waitAndApplyResumeNetworkMailboxWithTimeout(s *guestServer, buf []byte, timeout time.Duration) error { + deadline := time.Now().Add(timeout) for { seq := atomicLoadUint32(buf[resumeNetworkMailboxSeqOffset:]) if seq == 0 { + if time.Now().After(deadline) { + return fmt.Errorf("resume network mailbox payload was not patched within %s", timeout) + } time.Sleep(100 * time.Microsecond) continue } diff --git a/lib/system/guest_agent/resume_network_test.go b/lib/system/guest_agent/resume_network_test.go new file mode 100644 index 00000000..f908cbd9 --- /dev/null +++ b/lib/system/guest_agent/resume_network_test.go @@ -0,0 +1,19 @@ +//go:build linux + +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestWaitAndApplyResumeNetworkMailboxTimesOutWhenPayloadMissing(t *testing.T) { + buf := make([]byte, resumeNetworkMailboxSize) + + err := waitAndApplyResumeNetworkMailboxWithTimeout(&guestServer{}, buf, 5*time.Millisecond) + + require.Error(t, err) + require.Contains(t, err.Error(), "payload was not patched") +} From df8309e9a0d3b05ddd28193b45a35d1c2c00ca94 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:53:43 +0000 Subject: [PATCH 04/16] Align resume network ack timeout --- lib/instances/guest_resume_network.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/instances/guest_resume_network.go b/lib/instances/guest_resume_network.go index 354657b0..fd5abc60 100644 --- a/lib/instances/guest_resume_network.go +++ b/lib/instances/guest_resume_network.go @@ -26,6 +26,7 @@ const firecrackerSnapshotMemoryFile = "memory" const guestResumeNetworkMailboxSeqOffset = 64 const guestResumeNetworkMailboxLengthOffset = 68 const guestResumeNetworkMailboxPayloadOffset = 72 +const guestResumeNetworkUDPAckTimeout = 5 * time.Second var guestResumeNetworkMailboxMagic = []byte("HYPEMAN_RESUME_NETWORK_MAILBOX_V1\x00") var guestResumeNetworkMailboxOffsets sync.Map @@ -149,7 +150,7 @@ func (m *manager) waitForGuestResumeNetworkUDPAck(ctx context.Context, waiter *g attribute.String("hypervisor", string(stored.HypervisorType)), attribute.String("operation", "guest_resume_network_udp_ack_wait"), ) - waitCtx, cancel := context.WithTimeout(waitCtx, 2*time.Second) + waitCtx, cancel := context.WithTimeout(waitCtx, guestResumeNetworkUDPAckTimeout) defer cancel() elapsed, ack, err := waiter.WaitApplied(waitCtx, cfg.mac, cfg.ip) From 4ad440a1894efb98d6c1e989f26228ff633a5b08 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:17:02 +0000 Subject: [PATCH 05/16] Arm resume network mailbox for Firecracker guests --- lib/instances/create.go | 2 ++ lib/instances/guest_resume_network.go | 20 ++++++++++- lib/instances/restore_egress_test.go | 48 +++++++++++++++++++++++++++ lib/instances/start.go | 1 + 4 files changed, 70 insertions(+), 1 deletion(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index df991674..e2823c5e 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -474,6 +474,8 @@ func (m *manager) createInstance( stored.Volumes = req.Volumes } + ensureGuestInitiatedResumeNetworkMailbox(stored) + // 16. Create config disk (needs Instance for buildVMConfig) inst := &Instance{StoredMetadata: *stored} var proxyGuestConfig *egressproxy.GuestConfig diff --git a/lib/instances/guest_resume_network.go b/lib/instances/guest_resume_network.go index fd5abc60..37cae375 100644 --- a/lib/instances/guest_resume_network.go +++ b/lib/instances/guest_resume_network.go @@ -15,6 +15,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" + "github.com/nrednav/cuid2" "go.opentelemetry.io/otel/attribute" "golang.org/x/sys/unix" ) @@ -26,6 +27,7 @@ const firecrackerSnapshotMemoryFile = "memory" const guestResumeNetworkMailboxSeqOffset = 64 const guestResumeNetworkMailboxLengthOffset = 68 const guestResumeNetworkMailboxPayloadOffset = 72 +const guestResumeNetworkMailboxTokenMaxLen = guestResumeNetworkMailboxSeqOffset - len("HYPEMAN_RESUME_NETWORK_MAILBOX_V1\x00") const guestResumeNetworkUDPAckTimeout = 5 * time.Second var guestResumeNetworkMailboxMagic = []byte("HYPEMAN_RESUME_NETWORK_MAILBOX_V1\x00") @@ -56,7 +58,7 @@ func guestInitiatedResumeNetworkMailbox(stored *StoredMetadata) bool { stored.HypervisorType == hypervisor.TypeFirecracker && strings.TrimSpace(stored.Env[guestResumeNetworkMailboxEnv]) == "1" && token != "" && - len(token) <= guestResumeNetworkMailboxSeqOffset-len(guestResumeNetworkMailboxMagic) + len(token) <= guestResumeNetworkMailboxTokenMaxLen } func guestInitiatedResumeNetworkMailboxToken(stored *StoredMetadata) string { @@ -66,6 +68,22 @@ func guestInitiatedResumeNetworkMailboxToken(stored *StoredMetadata) string { return strings.TrimSpace(stored.Env[guestResumeNetworkMailboxTokenEnv]) } +func ensureGuestInitiatedResumeNetworkMailbox(stored *StoredMetadata) { + if stored == nil || + stored.HypervisorType != hypervisor.TypeFirecracker || + !stored.NetworkEnabled || + stored.SkipGuestAgent { + return + } + if stored.Env == nil { + stored.Env = make(map[string]string) + } + stored.Env[guestResumeNetworkMailboxEnv] = "1" + if token := guestInitiatedResumeNetworkMailboxToken(stored); token == "" || len(token) > guestResumeNetworkMailboxTokenMaxLen { + stored.Env[guestResumeNetworkMailboxTokenEnv] = cuid2.Generate() + } +} + func newGuestResumeNetworkPayload(cfg *guestNetworkConfig) guestResumeNetworkPayload { return guestResumeNetworkPayload{ InterfaceName: "eth0", diff --git a/lib/instances/restore_egress_test.go b/lib/instances/restore_egress_test.go index b5fcdb4e..c881bfb7 100644 --- a/lib/instances/restore_egress_test.go +++ b/lib/instances/restore_egress_test.go @@ -6,6 +6,7 @@ import ( "os" "testing" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/network" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -114,6 +115,53 @@ func TestPatchGuestResumeNetworkMailbox(t *testing.T) { assert.Equal(t, *payload, decoded) } +func TestEnsureGuestInitiatedResumeNetworkMailbox(t *testing.T) { + t.Parallel() + + stored := &StoredMetadata{ + HypervisorType: hypervisor.TypeFirecracker, + NetworkEnabled: true, + } + ensureGuestInitiatedResumeNetworkMailbox(stored) + + require.Equal(t, "1", stored.Env[guestResumeNetworkMailboxEnv]) + token := stored.Env[guestResumeNetworkMailboxTokenEnv] + require.NotEmpty(t, token) + require.LessOrEqual(t, len(token), guestResumeNetworkMailboxTokenMaxLen) + assert.True(t, guestInitiatedResumeNetworkMailbox(stored)) +} + +func TestEnsureGuestInitiatedResumeNetworkMailboxPreservesToken(t *testing.T) { + t.Parallel() + + stored := &StoredMetadata{ + HypervisorType: hypervisor.TypeFirecracker, + NetworkEnabled: true, + Env: map[string]string{ + guestResumeNetworkMailboxTokenEnv: "existing-token", + }, + } + ensureGuestInitiatedResumeNetworkMailbox(stored) + + assert.Equal(t, "1", stored.Env[guestResumeNetworkMailboxEnv]) + assert.Equal(t, "existing-token", stored.Env[guestResumeNetworkMailboxTokenEnv]) +} + +func TestEnsureGuestInitiatedResumeNetworkMailboxRequiresEligibleGuest(t *testing.T) { + t.Parallel() + + cases := []StoredMetadata{ + {HypervisorType: hypervisor.TypeCloudHypervisor, NetworkEnabled: true}, + {HypervisorType: hypervisor.TypeFirecracker, NetworkEnabled: false}, + {HypervisorType: hypervisor.TypeFirecracker, NetworkEnabled: true, SkipGuestAgent: true}, + } + for _, tc := range cases { + stored := tc + ensureGuestInitiatedResumeNetworkMailbox(&stored) + assert.False(t, guestInitiatedResumeNetworkMailbox(&stored)) + } +} + func TestRequiresRestoreConfigDiskRefresh(t *testing.T) { t.Parallel() diff --git a/lib/instances/start.go b/lib/instances/start.go index 181aaa8d..dd550a96 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -141,6 +141,7 @@ func (m *manager) startInstance( }) } } + ensureGuestInitiatedResumeNetworkMailbox(stored) // 4b. Recreate vGPU mdev if this instance had a GPU profile // Note: GPU availability was already validated in step 2b From 8c9b79ceb7fabc2cfb4f8e4eb0053203cf146c8f Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:17:05 +0000 Subject: [PATCH 06/16] Fall back after resume network ack timeout --- lib/instances/restore.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 40b91c18..fb40a513 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -351,6 +351,10 @@ func (m *manager) restoreInstance( var reconfigureErr error if resumeNetworkMailboxPatched && waitForGuestNetwork { reconfigureErr = m.waitForGuestResumeNetworkUDPAck(reconfigureCtx, resumeNetworkAckWaiter, stored, resumeNetworkAckCfg) + if reconfigureErr != nil { + log.ErrorContext(ctx, "guest resume network UDP ack wait failed; falling back to host-initiated reconfigure", "instance_id", id, "error", reconfigureErr) + reconfigureErr = reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet) + } } else if resumeNetworkMailboxPatched { log.InfoContext(ctx, "guest resume network mailbox patched", "instance_id", id) } else { From 571d01e02158f803319f4da6bfae3f406182d860 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:36:39 +0000 Subject: [PATCH 07/16] Share resume network mailbox contract --- lib/instances/guest_resume_network.go | 70 ++++++------------- lib/instances/restore_egress_test.go | 27 +++---- lib/resumenetwork/mailbox.go | 64 +++++++++++++++++ lib/resumenetwork/mailbox_test.go | 47 +++++++++++++ lib/system/guest_agent/resume_network.go | 57 +++++---------- lib/system/guest_agent/resume_network_test.go | 3 +- 6 files changed, 163 insertions(+), 105 deletions(-) create mode 100644 lib/resumenetwork/mailbox.go create mode 100644 lib/resumenetwork/mailbox_test.go diff --git a/lib/instances/guest_resume_network.go b/lib/instances/guest_resume_network.go index 37cae375..2311b405 100644 --- a/lib/instances/guest_resume_network.go +++ b/lib/instances/guest_resume_network.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/binary" - "encoding/json" "fmt" stdnet "net" "os" @@ -15,33 +14,18 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/resumenetwork" "github.com/nrednav/cuid2" "go.opentelemetry.io/otel/attribute" "golang.org/x/sys/unix" ) -const guestResumeNetworkMailboxEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX" -const guestResumeNetworkMailboxTokenEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX_TOKEN" const firecrackerSnapshotMemoryFile = "memory" -const guestResumeNetworkMailboxSeqOffset = 64 -const guestResumeNetworkMailboxLengthOffset = 68 -const guestResumeNetworkMailboxPayloadOffset = 72 -const guestResumeNetworkMailboxTokenMaxLen = guestResumeNetworkMailboxSeqOffset - len("HYPEMAN_RESUME_NETWORK_MAILBOX_V1\x00") const guestResumeNetworkUDPAckTimeout = 5 * time.Second -var guestResumeNetworkMailboxMagic = []byte("HYPEMAN_RESUME_NETWORK_MAILBOX_V1\x00") var guestResumeNetworkMailboxOffsets sync.Map -type guestResumeNetworkPayload struct { - InterfaceName string `json:"interface_name"` - MAC string `json:"mac"` - IPv4 string `json:"ipv4"` - Prefix uint32 `json:"prefix"` - Gateway string `json:"gateway"` - AckPort uint32 `json:"ack_port,omitempty"` -} - type guestResumeNetworkUDPAck struct { received time.Time text string @@ -56,16 +40,15 @@ func guestInitiatedResumeNetworkMailbox(stored *StoredMetadata) bool { token := guestInitiatedResumeNetworkMailboxToken(stored) return stored != nil && stored.HypervisorType == hypervisor.TypeFirecracker && - strings.TrimSpace(stored.Env[guestResumeNetworkMailboxEnv]) == "1" && - token != "" && - len(token) <= guestResumeNetworkMailboxTokenMaxLen + strings.TrimSpace(stored.Env[resumenetwork.MailboxEnv]) == "1" && + resumenetwork.ValidToken(token) } func guestInitiatedResumeNetworkMailboxToken(stored *StoredMetadata) string { if stored == nil { return "" } - return strings.TrimSpace(stored.Env[guestResumeNetworkMailboxTokenEnv]) + return strings.TrimSpace(stored.Env[resumenetwork.MailboxTokenEnv]) } func ensureGuestInitiatedResumeNetworkMailbox(stored *StoredMetadata) { @@ -78,14 +61,14 @@ func ensureGuestInitiatedResumeNetworkMailbox(stored *StoredMetadata) { if stored.Env == nil { stored.Env = make(map[string]string) } - stored.Env[guestResumeNetworkMailboxEnv] = "1" - if token := guestInitiatedResumeNetworkMailboxToken(stored); token == "" || len(token) > guestResumeNetworkMailboxTokenMaxLen { - stored.Env[guestResumeNetworkMailboxTokenEnv] = cuid2.Generate() + stored.Env[resumenetwork.MailboxEnv] = "1" + if token := guestInitiatedResumeNetworkMailboxToken(stored); !resumenetwork.ValidToken(token) { + stored.Env[resumenetwork.MailboxTokenEnv] = cuid2.Generate() } } -func newGuestResumeNetworkPayload(cfg *guestNetworkConfig) guestResumeNetworkPayload { - return guestResumeNetworkPayload{ +func newGuestResumeNetworkPayload(cfg *guestNetworkConfig) resumenetwork.Payload { + return resumenetwork.Payload{ InterfaceName: "eth0", MAC: cfg.mac, IPv4: cfg.ip, @@ -180,23 +163,10 @@ func (m *manager) waitForGuestResumeNetworkUDPAck(ctx context.Context, waiter *g return nil } -func patchGuestResumeNetworkMailbox(snapshotDir, token string, payload *guestResumeNetworkPayload) error { - if token == "" { - return fmt.Errorf("resume network mailbox token is empty") - } - if len(token) > guestResumeNetworkMailboxSeqOffset-len(guestResumeNetworkMailboxMagic) { - return fmt.Errorf("resume network mailbox token is too long") - } - if payload == nil { - return fmt.Errorf("resume network mailbox payload is nil") - } - - payloadBytes, err := json.Marshal(payload) +func patchGuestResumeNetworkMailbox(snapshotDir, token string, payload *resumenetwork.Payload) error { + payloadBytes, err := resumenetwork.MarshalPayload(payload) if err != nil { - return fmt.Errorf("marshal resume network mailbox payload: %w", err) - } - if len(payloadBytes) > 4096-guestResumeNetworkMailboxPayloadOffset { - return fmt.Errorf("resume network mailbox payload too large: %d bytes", len(payloadBytes)) + return err } file, err := os.OpenFile(filepath.Join(snapshotDir, firecrackerSnapshotMemoryFile), os.O_RDWR, 0) @@ -213,28 +183,28 @@ func patchGuestResumeNetworkMailbox(snapshotDir, token string, payload *guestRes return fmt.Errorf("resume network mailbox memory file is empty") } - marker := make([]byte, 0, len(guestResumeNetworkMailboxMagic)+len(token)) - marker = append(marker, guestResumeNetworkMailboxMagic...) - marker = append(marker, []byte(token)...) - + marker, err := resumenetwork.Marker(token) + if err != nil { + return err + } idx, err := findGuestResumeNetworkMailbox(file, info.Size(), marker, token) if err != nil { return err } - if idx+int64(guestResumeNetworkMailboxPayloadOffset)+int64(len(payloadBytes)) > info.Size() { + if idx+int64(resumenetwork.MailboxPayloadOffset)+int64(len(payloadBytes)) > info.Size() { return fmt.Errorf("resume network mailbox marker is too close to end of memory file") } - if _, err := file.WriteAt(payloadBytes, idx+int64(guestResumeNetworkMailboxPayloadOffset)); err != nil { + if _, err := file.WriteAt(payloadBytes, idx+int64(resumenetwork.MailboxPayloadOffset)); err != nil { return fmt.Errorf("write resume network mailbox payload: %w", err) } var u32 [4]byte binary.LittleEndian.PutUint32(u32[:], uint32(len(payloadBytes))) - if _, err := file.WriteAt(u32[:], idx+int64(guestResumeNetworkMailboxLengthOffset)); err != nil { + if _, err := file.WriteAt(u32[:], idx+int64(resumenetwork.MailboxLengthOffset)); err != nil { return fmt.Errorf("write resume network mailbox payload length: %w", err) } binary.LittleEndian.PutUint32(u32[:], 1) - if _, err := file.WriteAt(u32[:], idx+int64(guestResumeNetworkMailboxSeqOffset)); err != nil { + if _, err := file.WriteAt(u32[:], idx+int64(resumenetwork.MailboxSeqOffset)); err != nil { return fmt.Errorf("write resume network mailbox sequence: %w", err) } return nil diff --git a/lib/instances/restore_egress_test.go b/lib/instances/restore_egress_test.go index c881bfb7..3696f745 100644 --- a/lib/instances/restore_egress_test.go +++ b/lib/instances/restore_egress_test.go @@ -8,6 +8,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/resumenetwork" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -87,11 +88,11 @@ func TestPatchGuestResumeNetworkMailbox(t *testing.T) { dir := t.TempDir() token := "test-token" mem := make([]byte, 4096) - copy(mem[512:], guestResumeNetworkMailboxMagic) - copy(mem[512+len(guestResumeNetworkMailboxMagic):], token) + copy(mem[512:], resumenetwork.MailboxMagic) + copy(mem[512+len(resumenetwork.MailboxMagic):], token) require.NoError(t, os.WriteFile(dir+"/"+firecrackerSnapshotMemoryFile, mem, 0644)) - payload := &guestResumeNetworkPayload{ + payload := &resumenetwork.Payload{ InterfaceName: "eth0", MAC: "02:00:00:85:17:c8", IPv4: "10.102.146.62", @@ -105,12 +106,12 @@ func TestPatchGuestResumeNetworkMailbox(t *testing.T) { require.NoError(t, err) offset := 512 - require.Equal(t, uint32(1), binary.LittleEndian.Uint32(patched[offset+guestResumeNetworkMailboxSeqOffset:])) - payloadLen := binary.LittleEndian.Uint32(patched[offset+guestResumeNetworkMailboxLengthOffset:]) + require.Equal(t, uint32(1), binary.LittleEndian.Uint32(patched[offset+resumenetwork.MailboxSeqOffset:])) + payloadLen := binary.LittleEndian.Uint32(patched[offset+resumenetwork.MailboxLengthOffset:]) require.NotZero(t, payloadLen) - var decoded guestResumeNetworkPayload - err = json.Unmarshal(patched[offset+guestResumeNetworkMailboxPayloadOffset:offset+guestResumeNetworkMailboxPayloadOffset+int(payloadLen)], &decoded) + var decoded resumenetwork.Payload + err = json.Unmarshal(patched[offset+resumenetwork.MailboxPayloadOffset:offset+resumenetwork.MailboxPayloadOffset+int(payloadLen)], &decoded) require.NoError(t, err) assert.Equal(t, *payload, decoded) } @@ -124,10 +125,10 @@ func TestEnsureGuestInitiatedResumeNetworkMailbox(t *testing.T) { } ensureGuestInitiatedResumeNetworkMailbox(stored) - require.Equal(t, "1", stored.Env[guestResumeNetworkMailboxEnv]) - token := stored.Env[guestResumeNetworkMailboxTokenEnv] + require.Equal(t, "1", stored.Env[resumenetwork.MailboxEnv]) + token := stored.Env[resumenetwork.MailboxTokenEnv] require.NotEmpty(t, token) - require.LessOrEqual(t, len(token), guestResumeNetworkMailboxTokenMaxLen) + require.LessOrEqual(t, len(token), resumenetwork.MailboxTokenMaxLen) assert.True(t, guestInitiatedResumeNetworkMailbox(stored)) } @@ -138,13 +139,13 @@ func TestEnsureGuestInitiatedResumeNetworkMailboxPreservesToken(t *testing.T) { HypervisorType: hypervisor.TypeFirecracker, NetworkEnabled: true, Env: map[string]string{ - guestResumeNetworkMailboxTokenEnv: "existing-token", + resumenetwork.MailboxTokenEnv: "existing-token", }, } ensureGuestInitiatedResumeNetworkMailbox(stored) - assert.Equal(t, "1", stored.Env[guestResumeNetworkMailboxEnv]) - assert.Equal(t, "existing-token", stored.Env[guestResumeNetworkMailboxTokenEnv]) + assert.Equal(t, "1", stored.Env[resumenetwork.MailboxEnv]) + assert.Equal(t, "existing-token", stored.Env[resumenetwork.MailboxTokenEnv]) } func TestEnsureGuestInitiatedResumeNetworkMailboxRequiresEligibleGuest(t *testing.T) { diff --git a/lib/resumenetwork/mailbox.go b/lib/resumenetwork/mailbox.go new file mode 100644 index 00000000..11342130 --- /dev/null +++ b/lib/resumenetwork/mailbox.go @@ -0,0 +1,64 @@ +package resumenetwork + +import ( + "encoding/json" + "fmt" +) + +const MailboxEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX" +const MailboxTokenEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX_TOKEN" + +const MailboxSize = 4096 +const MailboxMagic = "HYPEMAN_RESUME_NETWORK_MAILBOX_V1\x00" +const MailboxSeqOffset = 64 +const MailboxLengthOffset = 68 +const MailboxPayloadOffset = 72 +const MailboxTokenMaxLen = MailboxSeqOffset - len(MailboxMagic) + +type Payload struct { + InterfaceName string `json:"interface_name"` + MAC string `json:"mac"` + IPv4 string `json:"ipv4"` + Prefix uint32 `json:"prefix"` + Gateway string `json:"gateway"` + AckPort uint32 `json:"ack_port,omitempty"` +} + +func ValidToken(token string) bool { + return token != "" && len(token) <= MailboxTokenMaxLen +} + +func Marker(token string) ([]byte, error) { + if !ValidToken(token) { + return nil, fmt.Errorf("resume network mailbox token is invalid") + } + marker := make([]byte, 0, len(MailboxMagic)+len(token)) + marker = append(marker, MailboxMagic...) + marker = append(marker, token...) + return marker, nil +} + +func MarshalPayload(payload *Payload) ([]byte, error) { + if payload == nil { + return nil, fmt.Errorf("resume network mailbox payload is nil") + } + payloadBytes, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("marshal resume network mailbox payload: %w", err) + } + if len(payloadBytes) > MailboxSize-MailboxPayloadOffset { + return nil, fmt.Errorf("resume network mailbox payload too large: %d bytes", len(payloadBytes)) + } + return payloadBytes, nil +} + +func DecodePayloadFrame(buf []byte, payloadLen uint32) (Payload, error) { + if payloadLen == 0 || int(payloadLen) > len(buf)-MailboxPayloadOffset { + return Payload{}, fmt.Errorf("invalid mailbox payload length %d", payloadLen) + } + var payload Payload + if err := json.Unmarshal(buf[MailboxPayloadOffset:MailboxPayloadOffset+int(payloadLen)], &payload); err != nil { + return Payload{}, fmt.Errorf("decode mailbox payload: %w", err) + } + return payload, nil +} diff --git a/lib/resumenetwork/mailbox_test.go b/lib/resumenetwork/mailbox_test.go new file mode 100644 index 00000000..4f036181 --- /dev/null +++ b/lib/resumenetwork/mailbox_test.go @@ -0,0 +1,47 @@ +package resumenetwork + +import ( + "encoding/binary" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMarker(t *testing.T) { + t.Parallel() + + marker, err := Marker("token") + require.NoError(t, err) + assert.Equal(t, MailboxMagic+"token", string(marker)) + + _, err = Marker("") + require.Error(t, err) + + _, err = Marker(strings.Repeat("x", MailboxTokenMaxLen+1)) + require.Error(t, err) +} + +func TestPayloadRoundTrip(t *testing.T) { + t.Parallel() + + want := &Payload{ + InterfaceName: "eth0", + MAC: "02:00:00:85:17:c8", + IPv4: "10.102.146.62", + Prefix: 16, + Gateway: "10.102.0.1", + AckPort: 43210, + } + payloadBytes, err := MarshalPayload(want) + require.NoError(t, err) + + buf := make([]byte, MailboxSize) + copy(buf[MailboxPayloadOffset:], payloadBytes) + binary.LittleEndian.PutUint32(buf[MailboxLengthOffset:], uint32(len(payloadBytes))) + + got, err := DecodePayloadFrame(buf, binary.LittleEndian.Uint32(buf[MailboxLengthOffset:])) + require.NoError(t, err) + assert.Equal(t, *want, got) +} diff --git a/lib/system/guest_agent/resume_network.go b/lib/system/guest_agent/resume_network.go index 8c1108ec..cb4cd723 100644 --- a/lib/system/guest_agent/resume_network.go +++ b/lib/system/guest_agent/resume_network.go @@ -6,7 +6,6 @@ import ( "bufio" "context" "encoding/binary" - "encoding/json" "fmt" "io" "log" @@ -19,36 +18,20 @@ import ( "unsafe" pb "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/resumenetwork" "golang.org/x/sys/unix" ) -const resumeNetworkMailboxEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX" -const resumeNetworkMailboxTokenEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX_TOKEN" const vmgenIDKmsgSignal = "crng reseeded due to virtual machine fork" -const resumeNetworkMailboxSize = 4096 -const resumeNetworkMailboxSeqOffset = 64 -const resumeNetworkMailboxLengthOffset = 68 -const resumeNetworkMailboxPayloadOffset = 72 const resumeNetworkMailboxPayloadTimeout = 5 * time.Second -var resumeNetworkMailboxMagic = []byte("HYPEMAN_RESUME_NETWORK_MAILBOX_V1\x00") - -type resumeNetworkPayload struct { - InterfaceName string `json:"interface_name"` - MAC string `json:"mac"` - IPv4 string `json:"ipv4"` - Prefix uint32 `json:"prefix"` - Gateway string `json:"gateway"` - AckPort uint32 `json:"ack_port,omitempty"` -} - type vmGenIDResumeWaiter struct { file *os.File reader *bufio.Reader } func startResumeNetworkWatcher(s *guestServer) { - if strings.TrimSpace(os.Getenv(resumeNetworkMailboxEnv)) != "1" { + if strings.TrimSpace(os.Getenv(resumenetwork.MailboxEnv)) != "1" { return } @@ -61,19 +44,15 @@ func startResumeNetworkWatcher(s *guestServer) { } func newResumeNetworkMailbox() []byte { - token := strings.TrimSpace(os.Getenv(resumeNetworkMailboxTokenEnv)) - if token == "" { - log.Printf("[guest-agent] resume network mailbox disabled: missing %s", resumeNetworkMailboxTokenEnv) - return nil - } - if len(token) > resumeNetworkMailboxSeqOffset-len(resumeNetworkMailboxMagic) { - log.Printf("[guest-agent] resume network mailbox disabled: %s is too long", resumeNetworkMailboxTokenEnv) + token := strings.TrimSpace(os.Getenv(resumenetwork.MailboxTokenEnv)) + if !resumenetwork.ValidToken(token) { + log.Printf("[guest-agent] resume network mailbox disabled: invalid %s", resumenetwork.MailboxTokenEnv) return nil } - buf := make([]byte, resumeNetworkMailboxSize) - copy(buf, resumeNetworkMailboxMagic) - copy(buf[len(resumeNetworkMailboxMagic):resumeNetworkMailboxSeqOffset], token) + buf := make([]byte, resumenetwork.MailboxSize) + copy(buf, resumenetwork.MailboxMagic) + copy(buf[len(resumenetwork.MailboxMagic):resumenetwork.MailboxSeqOffset], token) if err := unix.Mlock(buf); err != nil { log.Printf("[guest-agent] resume network mailbox mlock failed: %v", err) } @@ -115,7 +94,7 @@ func waitAndApplyResumeNetworkMailbox(s *guestServer, buf []byte) error { func waitAndApplyResumeNetworkMailboxWithTimeout(s *guestServer, buf []byte, timeout time.Duration) error { deadline := time.Now().Add(timeout) for { - seq := atomicLoadUint32(buf[resumeNetworkMailboxSeqOffset:]) + seq := atomicLoadUint32(buf[resumenetwork.MailboxSeqOffset:]) if seq == 0 { if time.Now().After(deadline) { return fmt.Errorf("resume network mailbox payload was not patched within %s", timeout) @@ -124,17 +103,13 @@ func waitAndApplyResumeNetworkMailboxWithTimeout(s *guestServer, buf []byte, tim continue } - payloadLen := binary.LittleEndian.Uint32(buf[resumeNetworkMailboxLengthOffset:]) - if payloadLen == 0 || int(payloadLen) > len(buf)-resumeNetworkMailboxPayloadOffset { - return fmt.Errorf("invalid mailbox payload length %d", payloadLen) - } - - var payload resumeNetworkPayload - if err := json.Unmarshal(buf[resumeNetworkMailboxPayloadOffset:resumeNetworkMailboxPayloadOffset+int(payloadLen)], &payload); err != nil { - return fmt.Errorf("decode mailbox payload: %w", err) + payloadLen := binary.LittleEndian.Uint32(buf[resumenetwork.MailboxLengthOffset:]) + payload, err := resumenetwork.DecodePayloadFrame(buf, payloadLen) + if err != nil { + return err } - _, err := s.ReconfigureNetwork(context.Background(), &pb.ReconfigureNetworkRequest{ + _, err = s.ReconfigureNetwork(context.Background(), &pb.ReconfigureNetworkRequest{ InterfaceName: payload.InterfaceName, Mac: payload.MAC, Ipv4: payload.IPv4, @@ -145,12 +120,12 @@ func waitAndApplyResumeNetworkMailboxWithTimeout(s *guestServer, buf []byte, tim return err } sendResumeNetworkAck(payload, "applied") - atomicStoreUint32(buf[resumeNetworkMailboxSeqOffset:], 0) + atomicStoreUint32(buf[resumenetwork.MailboxSeqOffset:], 0) return nil } } -func sendResumeNetworkAck(payload resumeNetworkPayload, stage string) { +func sendResumeNetworkAck(payload resumenetwork.Payload, stage string) { if payload.AckPort == 0 || payload.Gateway == "" { return } diff --git a/lib/system/guest_agent/resume_network_test.go b/lib/system/guest_agent/resume_network_test.go index f908cbd9..17c77834 100644 --- a/lib/system/guest_agent/resume_network_test.go +++ b/lib/system/guest_agent/resume_network_test.go @@ -6,11 +6,12 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/resumenetwork" "github.com/stretchr/testify/require" ) func TestWaitAndApplyResumeNetworkMailboxTimesOutWhenPayloadMissing(t *testing.T) { - buf := make([]byte, resumeNetworkMailboxSize) + buf := make([]byte, resumenetwork.MailboxSize) err := waitAndApplyResumeNetworkMailboxWithTimeout(&guestServer{}, buf, 5*time.Millisecond) From 0838b80a7f9377552e680848675f31302fc67757 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:38:26 +0000 Subject: [PATCH 08/16] Always wait for fork network readiness --- cmd/api/api/instances.go | 7 +- cmd/api/api/instances_test.go | 6 +- cmd/api/api/snapshots.go | 3 +- cmd/api/api/snapshots_test.go | 8 +- lib/instances/fork.go | 10 +- lib/instances/fork_test.go | 2 +- lib/instances/manager.go | 6 +- lib/instances/restore.go | 32 +-- lib/instances/snapshot.go | 8 +- lib/instances/types.go | 8 +- lib/oapi/oapi.go | 424 +++++++++++++++++----------------- openapi.yaml | 14 -- 12 files changed, 236 insertions(+), 292 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 9d59515d..2c85284b 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -657,10 +657,9 @@ func (s *ApiService) ForkInstance(ctx context.Context, request oapi.ForkInstance } result, err := s.InstanceManager.ForkInstance(ctx, inst.Id, instances.ForkInstanceRequest{ - Name: request.Body.Name, - FromRunning: request.Body.FromRunning != nil && *request.Body.FromRunning, - TargetState: targetState, - WaitForNetwork: request.Body.WaitForNetwork, + Name: request.Body.Name, + FromRunning: request.Body.FromRunning != nil && *request.Body.FromRunning, + TargetState: targetState, }) if err != nil { switch { diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 896a0632..4e7863aa 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -1165,15 +1165,13 @@ func TestForkInstance_Success(t *testing.T) { result: forked, } svc.InstanceManager = mockMgr - waitForNetwork := false resp, err := svc.ForkInstance( mw.WithResolvedInstance(ctx(), source.Id, source), oapi.ForkInstanceRequestObject{ Id: source.Id, Body: &oapi.ForkInstanceRequest{ - Name: "forked-instance", - WaitForNetwork: &waitForNetwork, + Name: "forked-instance", }, }, ) @@ -1187,8 +1185,6 @@ func TestForkInstance_Success(t *testing.T) { assert.Equal(t, "forked-instance", mockMgr.lastReq.Name) assert.False(t, mockMgr.lastReq.FromRunning) assert.Equal(t, instances.State(""), mockMgr.lastReq.TargetState) - require.NotNil(t, mockMgr.lastReq.WaitForNetwork) - assert.False(t, *mockMgr.lastReq.WaitForNetwork) } func TestForkInstance_NotSupported(t *testing.T) { diff --git a/cmd/api/api/snapshots.go b/cmd/api/api/snapshots.go index ffe35474..00b6d06b 100644 --- a/cmd/api/api/snapshots.go +++ b/cmd/api/api/snapshots.go @@ -176,8 +176,7 @@ func (s *ApiService) ForkSnapshot(ctx context.Context, request oapi.ForkSnapshot } domainReq := instances.ForkSnapshotRequest{ - Name: request.Body.Name, - WaitForNetwork: request.Body.WaitForNetwork, + Name: request.Body.Name, } if request.Body.TargetState != nil { domainReq.TargetState = instances.State(*request.Body.TargetState) diff --git a/cmd/api/api/snapshots_test.go b/cmd/api/api/snapshots_test.go index f87e9148..1cfaf1d8 100644 --- a/cmd/api/api/snapshots_test.go +++ b/cmd/api/api/snapshots_test.go @@ -52,7 +52,7 @@ func TestSnapshotScheduleToOAPIPreservesZeroMaxCount(t *testing.T) { assert.Equal(t, "24h0m0s", *out.Retention.MaxAge) } -func TestForkSnapshotMapsWaitForNetwork(t *testing.T) { +func TestForkSnapshotSuccess(t *testing.T) { t.Parallel() svc := newTestService(t) @@ -71,13 +71,11 @@ func TestForkSnapshotMapsWaitForNetwork(t *testing.T) { result: forked, } svc.InstanceManager = mockMgr - waitForNetwork := false resp, err := svc.ForkSnapshot(ctx(), oapi.ForkSnapshotRequestObject{ SnapshotId: "snap-123", Body: &oapi.ForkSnapshotRequest{ - Name: "forked-instance", - WaitForNetwork: &waitForNetwork, + Name: "forked-instance", }, }) require.NoError(t, err) @@ -88,6 +86,4 @@ func TestForkSnapshotMapsWaitForNetwork(t *testing.T) { assert.Equal(t, "snap-123", mockMgr.lastID) require.NotNil(t, mockMgr.lastReq) assert.Equal(t, "forked-instance", mockMgr.lastReq.Name) - require.NotNil(t, mockMgr.lastReq.WaitForNetwork) - assert.False(t, *mockMgr.lastReq.WaitForNetwork) } diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 2c61fa25..4e4e8e99 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -81,9 +81,7 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR // the source data directory. Restore the fork while source remains standby and // under lock, then restore the source. if forkErr == nil && targetState == StateRunning { - restoredFork, err := m.applyForkTargetState(ctx, forked.Id, StateRunning, restoreInstanceOptions{ - WaitForGuestNetwork: req.WaitForNetwork, - }) + restoredFork, err := m.applyForkTargetState(ctx, forked.Id, StateRunning) if err != nil { forkErr = fmt.Errorf("restore forked instance before source restore: %w", err) if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { @@ -95,7 +93,7 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR } log.InfoContext(ctx, "restoring source instance after running fork", "source_instance_id", id) - restoredSource, restoreErr := m.restoreInstance(ctx, id, restoreInstanceOptions{}) + restoredSource, restoreErr := m.restoreInstance(ctx, id) if restoreErr != nil { if forkErr != nil { @@ -403,7 +401,7 @@ func resolveForkTargetState(requested State, sourceState State) (State, error) { return requested, nil } -func (m *manager) applyForkTargetState(ctx context.Context, forkID string, target State, restoreOpts restoreInstanceOptions) (*Instance, error) { +func (m *manager) applyForkTargetState(ctx context.Context, forkID string, target State) (*Instance, error) { lock := m.getInstanceLock(forkID) lock.Lock() defer lock.Unlock() @@ -445,7 +443,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe case StateStandby: switch target { case StateRunning: - return returnWithReadiness(m.restoreInstance(ctx, forkID, restoreOpts)) + return returnWithReadiness(m.restoreInstance(ctx, forkID)) case StateStopped: if err := os.RemoveAll(m.paths.InstanceSnapshotLatest(forkID)); err != nil { return nil, fmt.Errorf("remove fork snapshot: %w", err) diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index f158a645..606b7671 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -308,7 +308,7 @@ func TestApplyForkTargetStateStoppedRefreshesSnapshotForkCID(t *testing.T) { meta.StoredMetadata.Phases.Record(phasetracking.PhaseStandby, time.Now()) require.NoError(t, manager.saveMetadata(meta)) - inst, err := manager.applyForkTargetState(ctx, forkID, StateStopped, restoreInstanceOptions{}) + inst, err := manager.applyForkTargetState(ctx, forkID, StateStopped) require.NoError(t, err) require.Equal(t, StateStopped, inst.State) require.Equal(t, generateVsockCID(forkID), inst.VsockCID) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index c55ba93a..58a1877a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -389,9 +389,7 @@ func (m *manager) ForkInstance(ctx context.Context, id string, req ForkInstanceR return nil, err } - inst, err := m.applyForkTargetState(ctx, forked.Id, targetState, restoreInstanceOptions{ - WaitForGuestNetwork: req.WaitForNetwork, - }) + inst, err := m.applyForkTargetState(ctx, forked.Id, targetState) if err != nil { if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { return nil, fmt.Errorf("apply fork target state: %w; additionally failed to cleanup forked instance %s: %v", err, forked.Id, cleanupErr) @@ -451,7 +449,7 @@ func (m *manager) RestoreInstance(ctx context.Context, id string) (*Instance, er if current.State == StateRunning || current.State == StateInitializing { return current, nil } - inst, err := m.restoreInstance(ctx, id, restoreInstanceOptions{}) + inst, err := m.restoreInstance(ctx, id) if err == nil { m.notifyLifecycleEvent(ctx, LifecycleEventRestore, inst) } diff --git a/lib/instances/restore.go b/lib/instances/restore.go index fb40a513..b2ca8063 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -20,20 +20,11 @@ import ( "google.golang.org/grpc/status" ) -type restoreInstanceOptions struct { - WaitForGuestNetwork *bool -} - -func (o restoreInstanceOptions) waitForGuestNetwork() bool { - return o.WaitForGuestNetwork == nil || *o.WaitForGuestNetwork -} - // RestoreInstance restores an instance from standby. // Multi-hop orchestration: Standby → Paused → Running. func (m *manager) restoreInstance( ctx context.Context, id string, - opts restoreInstanceOptions, ) (_ *Instance, retErr error) { start := time.Now() log := logger.FromContext(ctx) @@ -54,7 +45,6 @@ func (m *manager) restoreInstance( inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata - waitForGuestNetwork := opts.waitForGuestNetwork() ctx = enrichInstancesTrace(ctx, attribute.String("hypervisor", string(stored.HypervisorType))) log.DebugContext(ctx, "loaded instance", "instance_id", id, "state", inst.State, "has_snapshot", inst.HasSnapshot) @@ -267,17 +257,15 @@ func (m *manager) restoreInstance( log.WarnContext(ctx, "failed to build guest resume network mailbox payload; falling back to host-initiated reconfigure", "instance_id", id, "error", cfgErr) } else { payload := newGuestResumeNetworkPayload(resumeNetworkCfg) - if waitForGuestNetwork { - var waitErr error - resumeNetworkAckWaiter, waitErr = startGuestResumeNetworkUDPWaiter() - if waitErr != nil { - log.ErrorContext(ctx, "failed to start guest resume network UDP ack waiter", "instance_id", id, "error", waitErr) - releaseNetwork() - return nil, fmt.Errorf("start guest resume network UDP ack waiter: %w", waitErr) - } - resumeNetworkAckCfg = resumeNetworkCfg - payload.AckPort = resumeNetworkAckWaiter.Port() + var waitErr error + resumeNetworkAckWaiter, waitErr = startGuestResumeNetworkUDPWaiter() + if waitErr != nil { + log.ErrorContext(ctx, "failed to start guest resume network UDP ack waiter", "instance_id", id, "error", waitErr) + releaseNetwork() + return nil, fmt.Errorf("start guest resume network UDP ack waiter: %w", waitErr) } + resumeNetworkAckCfg = resumeNetworkCfg + payload.AckPort = resumeNetworkAckWaiter.Port() if patchErr := patchGuestResumeNetworkMailbox(snapshotDir, guestInitiatedResumeNetworkMailboxToken(stored), &payload); patchErr != nil { if resumeNetworkAckWaiter != nil { resumeNetworkAckWaiter.Close() @@ -349,14 +337,12 @@ func (m *manager) restoreInstance( attribute.String("operation", "reconfigure_guest_network"), ) var reconfigureErr error - if resumeNetworkMailboxPatched && waitForGuestNetwork { + if resumeNetworkMailboxPatched { reconfigureErr = m.waitForGuestResumeNetworkUDPAck(reconfigureCtx, resumeNetworkAckWaiter, stored, resumeNetworkAckCfg) if reconfigureErr != nil { log.ErrorContext(ctx, "guest resume network UDP ack wait failed; falling back to host-initiated reconfigure", "instance_id", id, "error", reconfigureErr) reconfigureErr = reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet) } - } else if resumeNetworkMailboxPatched { - log.InfoContext(ctx, "guest resume network mailbox patched", "instance_id", id) } else { reconfigureErr = reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet) } diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 523917b4..05d8084e 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -127,7 +127,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps } if restoreSource { - _, restoreErr := m.restoreInstance(ctx, id, restoreInstanceOptions{}) + _, restoreErr := m.restoreInstance(ctx, id) if restoreErr != nil { if copyErr != nil { return nil, fmt.Errorf("snapshot copy failed: %v; additionally failed to restore source: %w", copyErr, restoreErr) @@ -337,7 +337,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str } return m.getInstance(ctx, id) case StateRunning: - inst, err := m.restoreInstance(ctx, id, restoreInstanceOptions{}) + inst, err := m.restoreInstance(ctx, id) if err != nil { return nil, err } @@ -486,9 +486,7 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS } cu.Release() - inst, err := m.applyForkTargetState(ctx, forkID, targetState, restoreInstanceOptions{ - WaitForGuestNetwork: req.WaitForNetwork, - }) + inst, err := m.applyForkTargetState(ctx, forkID, targetState) if err != nil { if cleanupErr := m.cleanupForkInstanceOnError(ctx, forkID); cleanupErr != nil { return nil, fmt.Errorf("apply snapshot fork target state: %w; additionally failed to cleanup forked instance %s: %v", err, forkID, cleanupErr) diff --git a/lib/instances/types.go b/lib/instances/types.go index e38f5499..8c031fc9 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -268,10 +268,9 @@ type UpdateInstanceRequest struct { // ForkInstanceRequest is the domain request for forking an instance. type ForkInstanceRequest struct { - Name string // Required: name for the new forked instance - FromRunning bool // Optional: allow forking from Running by auto standby/fork/restore - TargetState State // Optional: desired final state of forked instance (Stopped, Standby, Running). Empty means inherit source state. - WaitForNetwork *bool // Optional: wait for guest networking before returning a Running fork. Nil defaults to true. + Name string // Required: name for the new forked instance + FromRunning bool // Optional: allow forking from Running by auto standby/fork/restore + TargetState State // Optional: desired final state of forked instance (Stopped, Standby, Running). Empty means inherit source state. } // SnapshotKind determines how snapshot data is captured and restored. @@ -315,7 +314,6 @@ type ForkSnapshotRequest struct { Name string // Required: name for the new instance TargetState State // Optional TargetHypervisor hypervisor.Type // Optional, allowed only for Stopped snapshots - WaitForNetwork *bool // Optional: wait for guest networking before returning a Running fork. Nil defaults to true. } // SnapshotPolicy defines default snapshot behavior for an instance. diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index add4df50..1c053f62 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -732,10 +732,6 @@ type ForkInstanceRequest struct { // TargetState Target state for the forked instance after fork completes TargetState *ForkTargetState `json:"target_state,omitempty"` - - // WaitForNetwork When the fork result is Running, wait for guest networking to be applied before returning. - // Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. - WaitForNetwork *bool `json:"wait_for_network,omitempty"` } // ForkSnapshotRequest defines model for ForkSnapshotRequest. @@ -749,10 +745,6 @@ type ForkSnapshotRequest struct { // TargetState Target state when restoring or forking from a snapshot TargetState *SnapshotTargetState `json:"target_state,omitempty"` - - // WaitForNetwork When the fork result is Running, wait for guest networking to be applied before returning. - // Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. - WaitForNetwork *bool `json:"wait_for_network,omitempty"` } // ForkSnapshotRequestTargetHypervisor Optional hypervisor override. Allowed only when forking from a Stopped snapshot. @@ -15961,215 +15953,213 @@ var swaggerSpec = []string{ "oWvlHKxmPrBPTXxg59CSG37TplMvIgsSFZnAHF0mp1cQlPGJWbTSrChb4IiGY8qS1MsSjaR8lQrQRU2j", "CE94qsw1o00NzzuBeGuwPaZaXLezc19xcbk2clWfxFkG/Fqv0CE40qfWVQOnOEb2a5ccUVD6sutAc2lq", "n0v0znxhPET5z0laxtPpQk/Wk8SQIFJxkKTWYWibaatd+vUWcJa66BHTXy47Hyl0pjc10Qb3a2GLGQHk", - "B7VWY9Gc8h7eP4fXv3Y7V5gqSPwvxeGsjq5gGQ31WqWRKq21bhEIbaJrCy4/xeEywTq2LEqMICoVdnmL", - "oSjg2UTnRFmHo+L2VYSnylp2H0+NkSbTGDxlNCL1XqeUUVg9LJcsmAvOeCqjZStvYttEAU3YtY6mFnzJ", - "yNVjMCVkUvT0tu5JhpOH4chVsX6ZLyZ/CbQUQUPSRyB9IOjIZW5WJNG54klCwsw/1h8xG2mf/STNDRMw", - "KdBBzQkViAs6o+WOyw7IhwwavMlWddz0Y7s+0HYtErZu4cBDCP9pPjTMxKYmRtEk7Rcz7yyTdrqd8wxT", - "xVK9zDrvMlyaGsfkQcK1Ib4++3DT0MhE8Cn1IWVBLI19ai17FzT42+7gvDf8f0wAsN6PoOJTZuJvYh5W", - "IFDs++00l9dnH86axpSBkqDi6GpzyiKmVsGyOYrYS0l7q20tYCcetGKSdZLbbi98ttBU4JhM0umUiHHs", - "cc6+0s+RecGExlGGTn8u20Pa7mrrdTkrLQ64XaY4sJgS7ajvcehWptEtUPOTf7neEaPGNWWi6qUS9h2b", - "jNpHbzIYGPT67INEeZSbx9NbXt7GTI+z+VLSAEemRZNYTlnRQQvM2drCOss/tK5sj53lRw9yGwFtLGZJ", - "Ctvw/F3v5O3HrTgki25pTBCZNucR0ePeLEiLhctHzdNSSkJi0eQpM4wh226gAq2yHdyaSIX96qGO4gpH", - "YxlxX7DPe/0QwUO08fGVyRfUI+iipLSU+vcCFUr8ve/dMVoiNXV7Dh1WXe6lDe71PZRxXI17rjC9Uqe+", - "rWKSNOo6YB26i1+WF5pfroeLMo0093vk8kgquoA1RJBJN0GQbuLiF5D51JzQ1rUmSYIFViRaGs0rO/oi", - "OiXBMojMHif1q2hyTYIbJLK81K9/NXnoqSBjNRdEznlUDmDY6dZhAyUE0S6IhW8xcypc3CiOYiwu4WB0", - "hhhKmaFAOWllZx0k6lyp5AaT+uX9+zPjnVFELExEYjFIXNau5o9JhJdoQtQVIcxNBUuE0Wue4eRUs7pk", - "A4SGUOOECMrLNOzsePo9N4G9aCZwQJD5yil8dkkknJptSWl78UDBBQGRsmF9h6vW1346TaN2a+wb1nAt", - "6G1wkwV+f3TmoCAyVEtH5u06lc+I6Jkt5+AtVy/ttlwNSuK6YpyRemeCTwiglNjErWKKmQtMAhQW/Xkp", - "rasgHGQxn8r2A7vAkKpr9vmnVspedbv7AjFizEIfOqiJmDe5s7M01gsCFkYKd480NFFLxpYwankx8F8Q", - "HFJGpKwkHgapiDrdTm9qZ3WwtRXxAEdzLtXB7s7w+dbq+M+Vgb/WnhmHdJX966weEy/j8vMMSidMuswS", - "WzhJWnhQDR3XnA8gnuqBhoAIqs+2gobnIs8Hg1oO5TUOlANWApdq6cocF7dtolmyNB9oMIOl3nvxorg/", - "B94YhhwV3rH/Vo339cxMMCJYoeCOqdDRMPlnr0bFhQ89hwtlM6QmxEUrZuehiwW0l3Klzp4Pnhdn2Qp8", - "G4RNZZvbfeeZqnm7nDVJCuS2+9c2AHFVZZ3DbenVWHuaLmt4SkvEGkutJihPCLsRPfd2d7ZvRs+2Ezlx", - "SZ0VueTDbDg6PTY6UcCZwpQRgWKisK00UBAy4GvTUkYb+CEmMeS5TP9jtWhpiH8pgjA0RlAc1eD6HiR6", - "ogFm6p2J/w1RjBmdaoFs3yz2LOd4e2//wIDhhWS6u7ff7/dvmpr+Ms9Fb7UUWyb9tpCl3pfzu63DA2Sg", - "t5nLl87Z4ftftCBLpTCH1pacUHZQ+Hf2z/wB/GH+OaHMm7neCj+RTmu4ieV4Qm3wm98PClD3Tu9pBWft", - "d5ZDWDHAZXhhhhSead3GcNxd8YRujTiYw96qAtJgMRurBeog/bw6DMF5leAd22fKFI1yQMZ6AMKtIDXl", - "StSxGuJYQliGMxZF5q+As4XeFT7QsdJJ5J7dKXhnper197rGtX6/OcVrDdv6vWyZ/GsLtmghkTwn0TeX", - "+rcJcCv3/nb2n3/8v/Ls2T+Gf/z28eN/LV7/5/Eb+l8fo7O3d0I/WI2G9U0hre4NxQqiukpQVm1Z6RSr", - "wOON0oZOA4XtE2Nbq2DeR0fgNT8YsR76jSoicHSARp1Kft6ogzYI2ATwlVbsdFM2zXhTf3xm7hb1x1+c", - "wve12kZo84mFXZAMhUCmk5DHmLLNERsx2xZyE5GgAeu/QhTgRIHjgjKkLb0lmgio5mPvNvLOu+gLTpKv", - "myMG1wPkWgk9gwQLlcH3uR6AKeyoTMCufZ2EDg/KXC+MWHYuZXBQ5gKwn6m5EBhTTXfyE2W1pWJthOcD", - "H3AWpFzohYyoVEbZzjhbs1GWC4KeDzbrlssabTrjoRXsBzuhXubLMWWLvWQYGLo2gnvsnHFrAlm0bDJ7", - "BIGtpDj89xy5hnJaZEtsPOTm4lOaC2gVyULqzWbHW2QAVrflhMwNI3wWtUjZf2lys97/do4UEbHLlt0I", - "NDmnNNDzg9hbKmWqWZFidHh0+nKz36JOGdA2G/+KdXyfzbCaWW1vHJsuUnPDDseki06OITfO7tBcgYOY", - "9ldcoMgImHxfH6APklRsRMh+hJBas5LRMr+2NCfAqLPpWkyqkuIAvcv0RpwNpVQbrXwTmu9LaNZGPZmA", - "+1rr3VrVIeHsIivaILweqyxDU5+4zaKgvaPCUhz2fMWsvvHeLt4kNxrNhbW/b7DE+1d3dm6m7rjCF8kc", - "Sx93z4tXIfDSimpTtOLfFc33/qV+1xZ8yrqDGlb66Ct+7ivVs9cbDt8Pd29u898U564MCFPAIsqg7tpj", - "1D0E1lvd/r2matwYUYz0Yxs/7Ky8j6dojiX7i4KHFVtvuPOsVdUC3WvbWNxiFC6fmiFlUsqhy2QxpAZn", - "55JGkQnNlnTGcIReoI3zk9e/nvz22ybqobdvT6tLseoL3/q0gLxzouL12Qe4TsNy7MK1mjO4cJ4FSa6p", - "VLIOvtMqKvQuEHvm03aw9G6Spo0cn341Tt8vJSw9L5rS5j0C7LlQ3hoZHwM671vmSH1/sH0rgfbuipZn", - "jZcHAstrFO4+oLmynDc/3y/s3YMMZ22Rx+JZ7xJYb40z1+1QT/LeodQimITo5CyHus+djK75ypxsxdTh", - "YNAfDtq4XGMcrOj79PCofeeDbaNaHODJQRAekOkdXL6WsY0yjqMrvJRo5MylUcfYZwXDrLBtrUnV6nq6", - "Dud3O/S+qkLjl9NGsXP3/tKWR2pSblpkZ1XBguI0MgnAxZJK9SqxMjHIeAZTOtNlRwwG2LUIPVllVxwE", - "Is39Ga5kn9F808Ty/YgJIhPOpCnk2Ue/kqVEMYU7hKx7iBySKAtyD0dsQ7io5CwzJMGpJKH+AaJpuy5q", - "Uw+NKsDO1h+MmJynUIFws4+OOJNpTIR19aAJBT/0JpKpMe5gvEANqIcraUjEiOnXPNh7XzJF/WB/MBgM", - "ssqInYMd/e+Bj5vuiL/oPm+nctjPc13Dz8Pr4Brb4TDeFfxuVTml83IhpdYW3R2qjbYKxnc6XRaGD1+N", - "b3JXRlDA0yjUZsJEHwzGi0NC62ySROU1quAs+cAumWbn0tRtkJvi6I+UiCX6eHpaumATZGpL8LSYOGyo", - "hnXgyY2WYXuNYb12NLeERHwMGMTqoV5Qpu4d9LDo4XfpkoZDW3j6c+PKG6JMmVkazScr5lTx0YZkMU5T", - "n86uHzmQhA8fTo5LzIHx/vD54PmL3vPJcL+3Gw6GPTzc2e9t7+HBdCd4ttNQBK99isLtsw68FpqvqpmL", - "PRy7GEhv4eqGCNTKEWmj6q4oC/lVq5LvWe82hGpd9/UAydZD8IZVQ6VsaKlBSpwW6mGbsMJKzasGz9P+", - "+8FwjeepXRnvBvn7XqQsMOhlIIkzn26xgHdxsepFfG8uTmFALnx5HbWKnbcn2uBg78XB3l2J5kJw142x", - "yk6PuLhNAQcOArMS4+vyTAr+i0IFbNA3jJvVhgR3up0sahn+hoO2EhGXPW4Vit+0Ybt+MbJKfjdkpJ2U", - "1Ga4hTVIWOGB1gKyZKZJqlCWCKrVi6OIpyEq+H4MMBBcjJwUVOgDl55nXUPF9DytagOkKABaU6YFMVwI", - "6UZs+toBeg3vwiMcG+vCDgKzsHIXgsOluQvW+8t1bXT91UM+t2o+fKN1fgSF6fW0NRmsi3B1E0bzOUBv", - "OHyTGR2MV32N5nXQ9uuvV/2SGzavzOWvQ2dWjTtArzLVLVP+rLK3IYn9c2wFVg4bsVlKTrUr3tHckq9c", - "Ia+w2zEU7XQ7jlCQf1jPRPyQc31t/xVZ0RckQXAEeznP9EoVjSxKKsyEQml2GwKsF7dJv7AFBUg4NoZJ", - "U8iTSR+yxkv2kVNfPp6iDcBD+xuyjlT9r80sPKp01m2/2H2x/2z7xX4r1JN8gOvVziNIbqsPbq0OGiTp", - "2FU9bpj60dkHY4IHxrgFz7ydeyGJOhFcix4987yMct75i/6LIthLyFNTUt4OySJDQc5Km5rXDTE+f9Bo", - "QadT9sfn4HL7H4LGw+t9uT3xOjSzjvzen5PibW/NVUomPVMExo/HAQwlZCNkzTsiYQaQtgz800M4ANMh", - "y0mzLOeAbVwas4+xdnd2dp4/29tuxVd2dIWNMwZfkOdQtiMobDF4E228Oz9HWwWGM226RF0AmGXWrPTv", - "M2RLyQ3KCml/ONjxcUnDwZ1zjW17ETeS/KM1zeykLNEhtS4z22q73EvtnZ3Bs92953vttrH1Uo7F9WoJ", - "4wLPDXksDnJx5TdAm3x/eIYgrWuKg7LfZLi9s7u3/+z5jUalbjQqwPA22Ls3GNjzZ/t7uzvbw3bYS74o", - "AIsqVtqwZdnl2XQepvCshocUddHbbTotfOqUYbB3JIgwjQ8DF8FbOX0MtsFYmNfyRWhzMFjHeO3gavFt", - "K8dRpWS5UQ24QCnLkN37668Ab3ej1yymzXmwXozXLfsIM00uC4JhSrncgnaJIAvKU3kPDXFlUp2mEefi", - "Rt82WSjvDL4GXLtRiT6e/gWEiGYuJBVJykaTZb8VUCG3nNyNNnCJJ/xc3USsVqvRZulXTbjbsE27q/Kg", - "S9u/EZEn1KIqZevD745wFKRQvABn66lnBdgRkMmZJNHSBKpGEecMBXPMZgTKUJpSKWyGMJrzKOx7gwf1", - "k/HUe23Pr1DEDR7LJSGJxfU3g9CfaZ2FLgjaKGSSIsNKlfJee7GRKha5vcyNe7G/kBSWvuSHLINR0xMr", - "XoABNZ+UfIwRn0mwAhWE4Par6NMJFiayFjNTp2IRG+OxDE20rU97zxAr0tt3hJqjk0+tRWt1DMgPNJTE", - "geBSIhLRGdRE+HhaSTtbkUORJZ+tD6krD7YF65qLNM/ZBWeabF3OxncgeoLT73IkAg9DDsqKYDXnjYwx", - "SwHpv8DI5DqhwrBHu4C0OZdqnMGJ3HCwUo0BxT0VJMdkypIlMweQe8d7LjrRdhty2cjPW31d4yp/U00D", - "bJapXor6qdXNeNDHxnVAlZUYLjkoTBUB5CaQPznsN5XQKi2gzaANxlVJLBWgqzfbBGf4bVTdT808tVXK", - "ftsdnLdF41kNvnOG1fyETbknZfsG15DO9WzjBRMiYgp1DFBIGCWhMx6z+0jr24I8v0gSFKbEUs4opAJb", - "gmOzvSHtmjmnGGWziqyvdtjGH2zGsBrkHfq1L7YJs5H+7LD3IgVamcA4iXCeJ9YqypDKsf/+qt6wILM0", - "wgJVEadWDFku44iyyzaty2U84RENkP6gesk85VHEr8b6kfwJ5rLZanb6g3Ge5lC5NDaDs0kuZkEq/eZT", - "+EnPcrOSYgeuly3z/RYk+reJWvLG6r6iEbGgTB8YvS4wehkFeXd70JR92dBoKe+yDuh1U8ltWda34x3W", - "1mFWdNVzS2mibit3pWVH5NqbPgjrXpVrWnfFoA0XCOVQpst0LaA9t/KEtIssr4b8udFsSRKUe999vvds", - "vyXc9p18nSaF/b49m4t4hUezYaVO27jNnu89f/FiZ3fvxfaNHFQuOrRhfZoiRIvrU6mtXHGa7UEk1eBG", - "gzLxof4hNcSIlgdUqpN86wF9XbF1m4IL8r3ZdMkZFVfS3bOUPaDtfIwrtKXDksqV13ZGG2Q6JWBUjg3d", - "evlgKhmCrcYQ4AQHVC09DhN8ZUo+Zq9U4N7aeNPKg/WQ1LZtoYG05JLpJE+i2HCdo78a13qFF563Ru2X", - "6aTJjf+22qtx4uc+oOIVUYsbmrywaN1dkM3nCstSrJn+O4DoSRcnXo+ZNW+shp6qAn7AJaAtTlGIpPBB", - "FlbOP/tRcfkry1lw+5aU5CrFVx2hzVvwRja050T2mNDB+kyYinywB+DtvhpPivU0VhYsKRXfyE/dm/fb", - "ItmnDhaanWA376+Q9nCTD6vAWMCPdgyW5Hnb3RJLNHBTIUzXY47wiPSyOAcbw4tkavyres9brEVPHkZw", - "yafTMuDTXjNAICDzQci26wUrReJEdRG5BjOdhDV0OVeHfU+OOogLNOoM41Gn4gT0JkHE+HpsOyjnKg9W", - "IfZllaSqg5RuBpOIB5emVIQSlMg+GqCYYCZRymDzV3yUw8FqX1u3kxTWJsPHI+aGuCa2YEwTMscLCrCs", - "1kM1K8WxkGuqJMTbQDsHKORg35brZNkZ6tdMisJBPmk4dDBb2oZ1g/o9zlxAUP4umEtTqM7FPhPBuzb5", - "Tkvst29Pu+b+ByI3zMBK4SFuomYEWkBmXVQwRvPf/eFXk4iMYdxVzMq4TsdiKhn4qQWRREkLYpezQ4UJ", - "UMBTpqpglnG7EM5yxHv9SEoZxErY2zOAbLC92+qaIQlsBeO6e6nE6Ldg7krYpaW0L+5yx8fCsCnAM+f3", - "vL+z7vXqACTCghRL3Jl2imFxxuc6lopbzP9sV4/JdUBIWIXj8b/SNtTQfukNNfwN25z3rPqcfRvCxeqz", - "6z9c7DmMtYnaxZBIxlkPsm3dktrMWIMaYnOvy4xWwt8rZKCOfeBHvhfa5E2R69W0fkOuFYAEhmmkydvE", - "ulZU2cNoHcVvnXTRtKG5WF+s9QFqS5hwvVtVl7CRfo9SYML+fP9FJXzLcU6Ue/fc8k1zrdUSvHHJI+gC", - "JN0r5StKwztdZE90NIw3Kzy3O/e7QSxAV8t8EYZjMk4EmdLrFdxiXjCWcDkbOd85pXq8Em3E+BrtPkPB", - "HAtZGTujs7mKluX7y10PGMCdSq0IoghTN6hcnK+m+7AeLGCXs9i6Txs+L6Tu+ysqk3C8CsbuKHvNXccm", - "eAlum0Yf67Od3cFgZ3twKxy7+yr0XGinKQWh8J29JylF9RRbyBK+6tXArgSFvLGMTFIJguMDCFROcEBQ", - "RKaA8pJVYVx7WNS6Xj14q0HZPPKM/91C2XVzVxhloOisKwsB6KbRcQFQZeiA4vP6sFdAwWRiJqhhwnhy", - "FHZ6g/33w52Dvf2D4fAhgO8yIjVFxz77PLx6Fm3j6W70fPnsj+H82Ww73vEaXg9QU7yc81ktMW7nkBBR", - "LfNWLY8oSUQZ6cksonx9WscKWWCCNNbu/5s59s0MVioL5+VJFnUGrHLilDjrkXAy7OhX3k5Uh39yvHrY", - "twrRrg7Ez2DVoQA/tRsMALEO74r6mbKW586HwoutT56VaQPrzh5flidsbe8qN1Dcx88lwVjaYatO7Pqp", - "5vGOzrigah6vPh6y1zIMQYgz+yxVWMZl6KOTGYOijsWfs7CCopmkP+50O9Hn3fKesb+3R+iwmHkZA9ql", - "LqoBLa7doWboairAK7lpIUzkn7bG9Zh/GvaGLyD4Lfq8+9Og96KP/l4IwusaahXJN3Rvl34dtKFhsdKF", - "Q0gfvrhRhJqj5yoO+pX66jTkB7FF07M8nldEc2eFy0gqLXD+uLbGFRiBRo3zrqqdPc3GRS0pJBFe+uqg", - "F1yxsmIfFpkMTciMMtnGM7szyFyze/Go00eHFoMSrNW8YGqpeSgFWeATGsckpFqpNMZ9c8TndktvW9V4", - "uBkwsfvKo571/frZi/U5pOsC1Ncdk/07JCzdydxtZ+KuSm8Gz5mzSaEECrzYRXSKMKvU4bFVi22mIWSO", - "AMLMgQNUyVnWygCZK37OE9JFM65QnmPY0qOWsmbPXzZ+cg0e1RVJxYYhtu8lYzxDL6GrxNfJMUoED9Mg", - "T7CJYNB5SrRIK2COK7T69SFMD+nQgMw1qO+51qHR5MFo54FsWu+K91EzbPNSDwfrl/pBvCDdTpqE62WY", - "eamdBLsR1uialA2PT6ZM9oomWJjMpxYS/V2RgnUj13iLA60SpYm7QtE8Veckz4UKXCL4YPeOSUT0MVVv", - "BPEozKNKqcyl6HqROtx/Pm+6xIQ7p/pAfiUk0bYKAERAfzFmS+/AqkVW0cbAVc6S5kqrZ7DKLbXKg3u2", - "VhNrXKr29WorXm2TUF4sn5yBbt5vsVr75dpS9A/hh/uWStpbe7lQgVdz4H8ZXKnr32TMADwjiyrn9a7v", - "At7HFu+tZdwEGVfNmim6mQ97/23cymjcP9j66W//d+/TX73u5YrdLInohWQKoUSXZNkDvHukbfR+GTAN", - "sHq1Mj2zrEJwDE6j4JIYJ1WMr4vj3RtkQmP5Bse1KUAMVkxZ9u+1E/rbvzVHMBXI+AHk5FqWvTOU9UPU", - "CVLcHUcbMREzV8nYBd5v9kcMYvkvyVKiQjECq9I4Rv2LzD7RKjo4LXGELowa2CdscYEmFGq6yBHTVi0O", - "ApJoa8KCslNTjI+D9BEER8V2bFEElyhnrxxNxABBH09raHtvP7z/+e2HN8fjt2cv3xyejH99+V8QxHHV", - "Mz2EPc17u3v7tgRfkZJDzxLfAfj3Tih+PnYzWGAe/oKkFKhr6FGYqYSUUhdQUHgZbZA4UUtXbMjltmze", - "DJvsMGvQG852zyjsgxf3UXTmw8oqMwse9bRG3QCi63VgGlp4w7GhKRPm3mlybM88ZcDPrTdxRmfY48v2", - "ViC9j+IwbkBrQeNq699Y2sEfHH9cRRg20sCQqoKIW7FLpeo1x87HWpEa57UeyxEZKbPpJbQQsFXOJYmZ", - "2rJFnHwpraE+eVcnFOW7zCEW9eCj9XkyK1X5wswKI2lem1OnsVZ06hUEOtOkuZoTQQoLAR/kyK03JJlN", - "9miRKG1KrSRE5IGQLlMEap8LCtkjmbPBkSBLCKp7YFcj857i66wH8N5jWbvjgnnkGPnD1z+POpt99M6V", - "KqVT1wQMo2JP+IFQy1y0iiaOq+qLUeSq+rzN+96NZ2XVCunXtLcqzJn3UWJNHz/+HVP1iguwQJrTkh8c", - "TxWsm5AIwGWpoqW2ghqlMQnHWcHmpv3vajSbnOSsHHZexslZWxiYWAu59aV2XOJsPoY6pTU5SJAKqpbn", - "UNjVRAgTLIg4TM2Gd/Vh7c95x1AV6etX8FNOPVkIrwkjggbo8OwE9mOMGSjp6ONpoZKJKWpTw1AD9fLt", - "0Ym1cB0MH1gsVAHruWC+w7OTTrezIMJYeZ1Bf7s/gM2cEIYT2jno7PSH/UHHVPSFKW5BEUX40yYYZpbS", - "SWj1oJ/NK/orgWOiiJCdg989iXoQyAYvg76LZwWLJcFUWJMliSB90LAK1d8CsK47Sg/MeWwL8rZ20Em1", - "tMkUJHlrl/UTqJOwa2CK24OBhRlV9uCFVBATf771DxuMmPfbSp8D8nhQZmsWhdMpLcm/dju7g+GNxrNq", - "GLBjfd1+YDhVcy7oZwLD3LshEW7V6QkzGV7IYIXZQJviPgMWKu6w3z/p9ZJpHGOxdOTKaZVw2aQME4kw", - "YuTKVgT9B5/0kb1+gLIxcs7TSEsTZNLXnKNBYdGffUZYBHO6ICNmz+k4jRRNsAA3Qoz0+WwMpvLWMF2b", - "1c+ABX7m4bJC3ay5Ld1czzmdcwJX0xIkGQMO8bip3G/ubqaMaTGJJbG1NLK6l/VoHi0uxzLgvnyi94Rh", - "pnoyIQGd0gDBy3r3Wo+2t8FW0Hxa4MGyEAGIWc5Ds73pz0mFqh/+dO7j7Bmy5C2rEwyugYIoDXOdy6VJ", - "YTHBUeTFbppFfIKjsaHPJfGoqK/hDUuUYoEUp9wwHhJT7CJZqjln5u90kjKVmr8ngl9JIrQKZIuYWVqT", - "0JQtM6x7BVifMRQSMyVSdZ9bZohbXy7J8mt/xA7D2JW/leYTHEmuT01bdNDmBZgtbXjXX5alIa7kKJWK", - "x5alsgqM+TB5qpJU2Tt1SZStvAavU4mSVM5JOGKKoy+CzKhUYvl160ve41ewXQgONZ8UXjFT2vpCw69N", - "o5ZjrGc/hlc91h8BAow6+nQZdfTfM4G17ZLKOThRJDhOZsUl3cjwdLReuFmlcIAZSnhisIiAqeZYs1yp", - "DYhHx1GEFGwl963WNmElG+Zj04vjSWNusUkGrWwjytDpz4XNNNh97t9PkgSC+Bwc/3n+9g2Co0qvgXkt", - "d1iZS22mT1EUpqDJQ+/9EXuJgzkyehPgzY46NBx1Musi3ISxptJGaPd6oOL+pIf2k+mmS8Of+n3dlNGe", - "D9DvX0wrB3ovJfFY8UvCRp2vXVR4MKNqnk6yZ5/8BG1K0TwvCQK0YWT/pqtBDFBR+TFozg3MQsStrI2W", - "CKNcAhX9KBPKsFhZQNlDektBbcrjmSwS48sIfLejzsHIeW9Hne6oQ9gCfrMu3lHnq58CVoluBjc1NaSd", - "rp0x0f5gsLkeO8HS16NCl17U2+9rTfvavjfFwypddcXDTM4hM+sVNNXAjbr1CJrPzzh09SV/qHhrVDzr", - "uSgob/B98Rww7BsRY+BWNDBtz0ZOA1tpnRi2gIIJYHE4pBNjcFCnweXMWzQ/quZ83azYbdplAQwxcvy3", - "+wj8B/1mpQhNvy8eq18cAc64A65/YuwIi+UYseu3iF8T9T1w3OCxRKkFRf+W/PtU+Oc1sXpfTrSKNNsi", - "C3ff5MdzgmQTaVsxL2tb9RzG1DsnTKGX8Gvf/tdZPJAtfRHx2cUBMiSM+AxFlNl7wMJtkT4ULS3hI5Nv", - "kn1n008cmOaGOT//+T//C4OibPbP//lfrU2bv2C7b5nESQDfv5gTLNSEYHVxgH4lJOnhiC6ImwzAY5MF", - "EUu0MwA1MxHwqFhSyeomcsRG7B1RqWCF+1KDayltg2B6MJgPZSmRNl9Hv0inFnTLOJg9Jrzby4aUj7qj", - "u55EZ5hBYQL6VHQ8AFmitvyatb86fu+ZmXPJf1b1ldc8puvliyLXynBvzwzwhgIGSOzbd/DAThptnJ+/", - "3OwjsDEMVwCwGmjMeTNWee7/kEnrZZKRKGWBAlQ2sskksa32/x7bd9o5gG2LfyYPsMXavIEL2Lg8IHfd", - "rcAPW6GFO9hPN+ca9vlnj12WZrOD9vbzLXbh4phaGcL3t86O9+o0N08KJPsWJjDacNHw4EbkAp0dnbia", - "tpvfjOkf5dTQM7W1+rKjA3EG2GuPZpYdcTaNaKBQz40FCjPFJDPVygzyVMTBOztqhN28qhDGxfNtq4TI", - "13jSZeB8+ZH38KdHpdObHCM5zHLOaz9OknWsc0xlwPW3BW7pBTgBQjr1JdunRS5a55AywfXZkbNSXbLi", - "+eTYbcjHc03ZrlNWPRseQSgeVwTiNxSEleLxBWDyp8TNH7JVdIgUKzxX3xdrDh5PC3psL5aPzZ+SGyus", - "kE1LQRPU3XiAvibKhHJ3HnChbQ+eiZ8T4Xa1qyMBs86mZT41hVXNhOBCerXte2JeaWf6mvb+TJYvkOcm", - "Gosl+Q8VpYWxm9NqlYF7YkuWP5x9Cz3cyLy9v3tey2AeIkOwycR5rE3dXSyXLNj8U131PsppZoj9JA+z", - "szSK3I3HggiF3h6dmJ1VPAO2vkBY0nrd3u22lcfBh3e/9QgLOMShZTFUfiXKPrlnDd8smJnKDzZpYxOa", - "tGjqzrMmDecO62/CBZGJcOxT/u/bryI6EVgs/337FY4Sysi/7xxGWBGpNh+MWQaPJZofW+N+wsynFW5a", - "JhqIJgb13tdpqNlbLZVU9/6fSk81k76RpprR9Yey2kZZLZJrpb5ql+JBNVbTxze6ksmYzUdteOTiE/9k", - "murjevksRzqgaCrL1x62yB4X4OeFR5ShVJInGEBJM44rHhst3dX5hlx5fDjWPTnuAiGhLgKgNtkEkUdy", - "XrtxPLpya/t9fM/1YTyhs5Snsph7EmMVzIm0yUoRKQvgp6Z258dzo+L9HXPp4DGPjkfXq3/w/QNp/NUF", - "NcLb3ECt0/ndW211fvu+1vlNCrXNXbPQUl0HO7jZEFTokqjbsnEp17we7Ogbl88WQR+0oZKbCwgsiIMR", - "+z/a/vhdERx/+sklyaSDwfY+/E7Y4tNPLk+GnTpWIQwqHgFK7OGbY7j2m0H2OQDJ5il51XGYyhPAeg46", - "51/OQMpvPttbSI4Lf1hIrSykArlWW0h2LR7WRCrDbz26jeT4zUdwC2Lyw0p6DCtJptMpDShhKq95WgsS", - "syWTn2BuGbP3Q4XgjtJB29pKyjblGgU0Lwvw6IE9JzkO4mMbR64CwdOMkeeJhfS25kh+GDbbI98bPwwe", - "Vzg/vh3ylFnMKPx10iVap/SV0gaMyTiFspAoRwiBqE8kbP1H12If5RWsZZokXChpcCpBATZI9nOtAPsw", - "LcswlT5cSsBipER2RwwqFejHJpd/65IsDQol5SwDnKzWY/XlXpVRQL/pNrp/HcsPcdpKx3rkbWxBq7+d", - "jvXNRMejaFonpVoAG9nGAINyQrKdzLPkPvqZstnmk4pANcIqm1sBz8ijam1BpT+L67sls2qyTQdtAdrX", - "lp79Fzxx65P0ae0Oi7ZAQBRSPGNcKhoUK+8W4UF/nNCtT+jVlPVy89SWSPcb9K+4uGx7xHnqij2Bk644", - "w+/Ql6CHB2hg396lAMa2OQ000zz6KVgrFvctUzBo9VwMojSEqvT2QHSq5FTweGx/NHi1eldYNFBwUQS2", - "1W8tbHTvj+AwesMVonESEa3FkxD1DDfp1bSqv4Obp7JQWvFmwlBvm2JCjAGjk640kRWRcLnmFmwD7tnr", - "y+WVmhGfrQfByDp3iA8eFIwRM3D4xGHnX6BMyELxLhKRQKGrOQ3mgIgBBb2goiuAVeAkucggsDYP0GvY", - "qUUkMOh8QxKhDaGAM8kjYoAuFnF8cVBHbP14egofGTAMg816cZCVXM8OCKnfKiJcZDWP3ljcjg3NSYJH", - "kVnRC201Fua3abEvcoiyEfPhYDByZRukU3RRgMS4aMDEcAL1Nz77ZtpWtxlY0sxFcSSAcIY3CQs7TRcx", - "NPKjYQwH3nowLZE5zDAeGJijNpjf+CwDtSyxMk6StuxrhwlcvIjjFTyMNgrFWaUKear+JlVIhICPLXc3", - "MTfawIH5h8KXmlFtYaGsvC2wn/e60aDMeUmlhWqhio751yKOO92OHU8Bne4G2vsahJNqg/VrMb0yBRiT", - "H3r3TQBKysK+gFBSOTls+f9mlfudeeFP75+1hAr/DF6W8n1WPgrK8gJQAiq4uypcTwrpABaypouZwki+", - "PeJm2ZOF4qHtrrdqZUe/A6N13a1XVkMyK3D52Ndf9RE85SQYWZvNlItqevy6e7HvnpHub0lqU23DIT94", - "8+buuVaMmaQraolCKVQJfj6orwm4zsGcc1lg+wmZ4wXlwiKwW69rxpngsjDWo42eu9CsemH9txdWPT+w", - "viaEi49sH3343Mbc+b9wj/IvXhWs7Uzid51KDSiQEmE0EZRMUYJTSbS2lMYEmQojFsib4GDuqoX3R+z9", - "nCBbH7PgQMjKKVOJLobxRRdNUoUiLGZg7ZiHJpJOkIDHMWGhqXk7YnOCF1SbagJFWBEWLHuSQA3kBckL", - "mGjT3d5QmlLbWZXVLnLFecHBcFEovXuBEkGAiYy5zEp1bkdMpOw/DHKlbvbCDfQCEanwJKJyntWKCHBI", - "WOCFhTz/vsXY/Ttxz4mqV6f9JneWt5Kl3/ISs+jLzOqDfxf3m08sUIsLV1mzhZhfofTKZtOwHPl4nlfk", - "/Rfc0maubo7f6GYmI/GqXfx9XMmUSvL/uJZRdkuGqemOlMvW/2nvWvI60ikrXbdYn+xtL1yySggZmW8k", - "87a+uD9PbuEj+04kYbfRsG/C3M4n/T2IXEvVW8ncb+QctL6kglfsG4pgO6hvpz5xUZBy34UYNhsuk8ZF", - "maMEBpuKsx/CuCqMbXjAbYWx87jWLsAL4pmyXhLhJrmcV633C2DrEPgXjX6tzK4gCL+54MtvBB5N2J1k", - "4s0IvAQvI47/7PcyARfCJHTacsRPB1Cs4AssXDBtgMetm0mIrssm+Xh6utkkJYRaKSOEesISolzWNIg9", - "1RrfLogQNHSlI49Oj230KpVIpKyP3sYU6jleEpJAoRjKU4kgM7ev5+dSW+tF8Eo5rN0OYUosE06ZWjuK", - "/NWHGczXW5XOe2Q5aSEV//SXx+CFf3pCCmSHVlfsBFZbkQqrxmA8F5xGmal3qbUtPOGpbl1LFldodwZn", - "25RGRC6lIrGJzJumEWwiAN21NZnsdyajtIuokkjvhy5k4CVExFRKypkcMVv+PSFC960/h+K/eZCR13mv", - "cCY1z4zo+z4C2PRgTMwWVk1UA2gBqAPaOehs4STZgnLR/iApO7w7DOkVRKQhuYwnPKIBiii7lGgjopfG", - "6EALiSL9x+bKkLYxfHffFaduv7M0pU/YlHuLchiezZj5z5GEVBZr7hLxyYm116S4WZz8gYX2izW5Vq4J", - "gqOeojHJkt9RqmhEPxtRpxuhUtHA5NXkqZdQhNlmX47YKVFCv4MFQQGPIhIo51zZSgQPtkbpYLATJBRQ", - "SnYIDA4EXvPjGHo8OvsA75lC0d0R0/+Aht8fnpmb2Cm2PoLCQBlRV1xcopOtt2uCfM+BTP/CUXJmgitz", - "IL0L/uP67uaZzY17SDZsUZ6sMoB48qcP47Qa3A9vwdP0FgC0RDabjZnAASjFcp6qkF8xv2dgwaM01v8w", - "f5ysAyhROJh/hFe/G23XDGdtN26CT2JT2jmFxBQN+iYXFIZgTzW+VBPOTQGUmFLknvcUOFR/Ru6+f6d8", - "kY7f4dWkpagryPXd7K3HPvnsGBzuVpEeT2WbG05zM1F8tffpCtNm79PPEQ8uJUqZolEJ1EDbbYADqn/M", - "cRvtxR+oCZAd6UqJI3KdUAEINhV4BET0jCXCSBERU4ajLZizaQQQKJ0XCy84hSTlIKKQJkZDghIeRYCy", - "czUnDOnZgKPKNVC4p5W2AkTxneIVo+JoQgIeE4fKuekz3f6OqXrFRRli83uRi+8L9Nfz0VPV81yDKtrc", - "451QRk/xNYQ1h6m9JnYj2njN8x+NK6iLYG1GnZ2BHHW6aNTZjkcdvQJHGFyoWKE9FFOWKiL76Nj4tyAN", - "dX+AJAk4C6UDB3UevJ2BbEpKNWzZkOG4D989ptpjuQpI+c524hMP+j2kv4cEG7RR3HB2T4Zd2HQh4qmC", - "AG63r+xbIVHgHtl89BvYwh75Ydu3keR/t9u3JKNglbW4LCy9kewZfORar5tLqphzmaNOogAnOKBq2UU4", - "iniQew9Smd0O9LKhTATBl9qG6o/Yuwy40iZCoKOzD13nNEMhlZemBesX66O3CyJkOskGh0AaGA8eLAYJ", - "R0xxFOAoSCPNt2Q6JQHkMEQ0pko2+NWyoTxkGcS8E8/Cu4cZbM3Tcib5eQJWL2cLWeG4LbPUW4IEEaZx", - "0alUJQ6ovnClC27fiW6U62N4GtnrrUBwKZFtqkciOqOTyF7WyD56r1UOHJMRSyLMGBEolSbuSA+9lwgi", - "ZWoSY3QDUGfWcFQX5UAnieDKuokjzoU0nl3N4R9PkVQkWcFm70zLpzDnB4IJNo3bnr6RwVAZQ/OxZF9B", - "ekEMpxiCaz7Sx/Q3CPYxA/rWcMJPZeO/F3Q2I0LvCmyErLkaNdvakdNs+lKmRyNG/nn2VjuM/KzVQjR3", - "IdJ5JVDF2L04BgX6Jjewns4vaSOWiX10s+yLX/VHLfsuR/n7B2Ef3XGWf5bSY+eF4Oq2yPo5hz81kPvC", - "yEtbtZSgsB6OoHVGwkNmCLTGHfhmcANPGWUAl9IOmuAEvj9GGDxudtxjw2w/bd4qoQSUCus0pEqth+/8", - "LjjwYXA7v3F26C1wO7+rfCXAXfx2eaPfVaZSyQ/oiof86ZE5HypBycBzAoxFU4KSkXo2kGClofTRvtPO", - "TLIt/pk0eHv3fAP93ZH9h9XfwmQoEMvvsjO50Q63hcSJWrrLRT6tXABK+hmSMXzAD1kMwcPhLdziev3+", - "2MPxaePl+o96Wo92f58XHT45fvpFtIp7rnSwbOlTp4dFMKcL0ux0L+9gS6JEkF7CE7hcCQ3BLD3cWaaw", - "6M8+I9u8xaqy/0LUQRyTEIVUkEBFS0SZ4iARTB9/kUhwbQnAcy6WPmd6cee+Ejw+tLNZcx7aPWWdYfmd", - "b7zshVjh3sJJmxUutDvctLu7bS3wEGXo9c9og1wrYRB30VRbPohOM5KS64CQUAJPbhYHPBw0eDbpZzKe", - "TdqMcgV28luLTY2CVCoeu7U/OUYbUGxhRpheC63qT0GTTQRf0NAUIs2JuuCRoeqwgaA39btqpSKrlOGM", - "CzO4b6LDtDmQZp9pUhYLJnShc9CZUIZhcGtRist7yiRU6f4whbSGfO84zun8OMKs5bfhjB3NidrIcURU", - "nBtovM0fx9xTPuaKganuTCuddu1KRbaLVW0ZQvoQgLlZHPPjuq0/fj/hlVQ+ychK6zpfZAZpk9v8+2LB", - "weOdD4/tLv/4hMPxXxNnfBdc5dCAbtHHML/xAEcoJAsS8QSqSJp3O91OKqLOQWeuVHKwtRXp9+ZcqoPn", - "g+eDztdPX///AAAA//9DeBEen5ABAA==", + "B7VWY9Gc8h7eP4fXWwfC6w/XOlJa0J2Rq8cgOmQK9DTb9iTDycNQfFUsW+ZryF+CU1jQkPQR7C4IqnGZ", + "iZWddq54kpAw8//0R8xGkmc/SXODoj80dFBzQgXigs5oueOyg+0hg+JuwoqOm27NjsUP6xoqPITwjeZN", + "j6fKoDxcumQtUsycsovQ6XbOM0wMK4nKpHmX4YrUKJIHedaG+Prsw01D2xLBp9SHdASxEPaptcxc0Ndv", + "u4Pz3vD/MQGcmt9ARaPMxE/EPKxAWNj32508r88+nDWNKQOVQMXR1eaURbysgtVyFLGXSvZW0lowjv31", + "wZJ1kuveL3y67FTgmEzS6ZSIcexxrr3Sz5F5wYQ2UYZOfy7rs1pvbms1n5UWB8zmKQ4sJkA76nsccpVp", + "dAvU/ORfrnfEHMNNmYR6qYR9xyYT9tGbDMYDvT77IFEepeTx1JWXtzFS/2y+lDTAkWnRJAZTVnSwAXO2", + "1pDP8g+tK9KjJ/vRX9xGQBuLWZLCNjx/1zt5+3ErDsmiWxoTRBbNeUT0uDcL0mLh8gnztIKSkFg0eToM", + "Y8i2G6hAq2wHtyZSYb96qKO4wtFYRtwXrPFeP0TwEG18fGXyvfQIuigpLaX+vUCFEn/ve3eMlkhN3Z5D", + "h1WXaWmDe23HMg6nca8Uplfq1LdVTJB9XcepQy/xy/JC88v1cD+mkeZ+j1weQMWpbRVJZNIFEKQLuPtn", + "ZD41XmvrGpEkwQIrEi2NZpEdfRGdkmAZRGaPk/pVIrkmwQ0SEV7q17+aPOJUkLGaCyLnPCpfQO9067Bv", + "EoIgF8TCb5g5FRzviqMYi0s4GJ0ijVJmKFBOOthZB2k5Vyq5waR+ef/+zFjXioiFiSgrBvnK2tXqMYnw", + "Ek2IuiKEualgiTB6zTOck2pWjmyAQBBqnBBBeZmGnR1Pv+cmMBPNBA4IMl85FEa7JBJOzbaktL14oLyC", + "gEjZsL7DVetrP52mUbs19g1ruBa0NLjJAr8/OnOp/BkqoSPzdp3KZ0T0zJZz8ISrl3ZbrgaVcF0xzki9", + "M8EnBFAmbOJNMUXIBZYAiob+vJSWUxAOspgPY/uBXWBI1TX7/FMrZa+63X0X6TFmoQ/d0UQ8m9zHWRrr", + "BdFDFincHdHQRJ2YdEyjlhcDtwXBIWVEykriWJCKqNPt9KZ2VgdbWxEPcDTnUh3s7gyfb62O31sZuGnj", + "VMYhXWXfuWgWE+/g8qsMyiJMuswSWzhJWnjADB3XnA8gnuqBYoDoqM+2gobnIocHg1oO3DUOlAPGAZdY", + "6coTF7dtolmyNB9oMIMV3nvxorg/B9476BzV27H/Vo339cxMMJnmEeNuqNDRMPlnr0bFhQ/9hAtlM1wm", + "xEWbZeehi+Wylyqlzp4Pnhdn2Qo8GYRNZZvbfeeZqnm7nPVGCuS2+9c2AHExZZ3DbenVWGmaLmt4SkvE", + "GkutJihPCLsRPfd2d7ZvRs+2EzlxSXkVueTLuT86PTY6UcCZwpQRgWKisEWKLwgZ8CVpKaMN/BCTGPIU", + "pv+xWrQ0xC8Uk+gbb8CPanBrD3L73QAT9M7Eb4YoxoxOtUC2bxZ7lnO8vbd/YMDMQjLd3dvv9/s3TS1+", + "mecSt1qKLZM+Wcgy7sv53dbhATKI28zlS+fs8P0vWpClUphDa0tOKDso/Dv7Z/4A/jD/nFDmzTxuhX9H", + "pzXcu3I8mDb4ze8HBahyp/e0giP2O4MhLBTgDrwwMQrPtG5jOO6ueDC3RozLYUtVASmumE3TAjWOfl59", + "jey8SvCO7TNlikY5oF79AvlWkIhyJWpUDTEqISzDiYoi81fA2ULvCh9oVOkkcs/uFHyxUvX6e13jWr/f", + "nOK1hm39XrZM/rUFy7OQNp6T6JtL/dsEKJV7fzv7zz/+X3n27B/DP377+PG/Fq//8/gN/a+P0dnbO2Wv", + "r0Yz+qaQRPeGQgRROSUoorasdIpV4PFGaUOngcL2ibGtVTDvoyPwmh+MWA/9RhURODpAo04lv2rUQRsE", + "bAL4Sit2uimbJrqpPz4zd2f64y9O4ftabSO0+aDCLkiWRS7TSchjTNnmiI2YbQu5iUjQgPVfIQpwosBx", + "QRnSlt4STQRUY7F3G3nnXfQFJ8nXzRGD6wFyrYSeQYKFyuDXXA/AFHZUJuDSvk5Ch+djrhdGLDuXMjgf", + "c8HVz9RcCGyopqv4ibLaUrE2wvOBD/gIQub1QkZUKqNsZ5yt2SiL5UfPB5t1y2WNNp3x0Ar2g51QL9Pk", + "mLLFXjIMDF0bwT12zrg1gQhaNpk9gsBWUhz+e45cQzktsiU2HnKTQCHNBauKZCF1YrPjBYmH1W05IXPD", + "CJ9FLVKuX5rcmve/nSNFROyyHTcCTc4pDfT8IHaSSplqVqQYHR6dvtzst6gzBbTNxr9iHd9nM6xmxtob", + "x6aL1NywwzHpopNjyG2yOzRX4CAm+RUXKDICJt/XB+iDJBUbEbLXICTSrGS0zK8tzQkw6my6FpOqpDhA", + "7zK9EWdDKdW2Kt+E5vsSmrVRKyZgutZ6t1Y1Rji7yIo2CI/GKsuw0ydusyho76iwFIc9XzGrb7y3izfJ", + "jUZzYe3vG+zu/tWdnZupO65wQTLH0sfd8+JVCLy0oloQrfh3RfO9f6nftQV7su6gBpE++oqf+0qt7PWG", + "w/fD3Zvb/DfFKSsDehSwZDKosvYYYw+B1VW3f6+pGjdGhCL92MZ/Oivv4ymaY8n+ouBhxdYb7jxrhTqv", + "e20bS1mMouRTM6RMSjl0kCwG0OCkXNIoMqG1ks4YjtALtHF+8vrXk99+20Q99PbtaXUpVn3hW58WkGVO", + "VLw++wDXaViOXThScwYOzrPYyDWVStbBU1pF9d0FIs182g5W3E3StJHji6/GWfulhIXmRcPZvEeANBeK", + "WSPjY0Cffcscl+8Pdm0lUNpd0c6s8fJAYGeNwt0HFFaW8+bn+4Ute5DhrC3SVzzrXQLirXHCuh3qSb46", + "lFoEkxCdnOVQ5bmT0TVfmZOteDkcDPrDQRuXa4yDFX2fHh6173ywbVSLAzw5CMIDMr2Dy9cytlHGcXSF", + "lxKNnLk06hj7rGCYFbatNalaXU/X4dhuh75WVWj8ctoodu7eX9ryNk3KTYvsmirYS5xGJoGzWBKnXuVT", + "JgbZzGACZ7rsiMEAuxZhJavMiYNApLk/w5VcM5pvmli+HzFBZMKZNIUY++hXspQopnCHkHUPkUMSZUHc", + "4YhtCBfwn0X2JziVJNQ/QDRt10Vt6qFRBdjH+oMRk/MUKsht9tERZzKNibCuHjSh4IfeRDI1xh2MF6gB", + "9UwlDYkYMf2aBzvtS6aoH+wPBoNBVtmuc7Cj/z3wcdMd8fPc5+1UDvt5rmv4eXgd3F47HL27gpetKodz", + "Xi6E09qiu0O1yFbB5k6ns2Hm9qvxTe7KCAp4GoXaTJjog8F4cUhonU2SqLzGEJwlH9gl0+xcmroNclMc", + "/ZESsUQfT09LF2yCTG0JlRYThw3VsA48udEybK8xrNeO5paQdo8BY1c91AvK1L2D1hU9/C7dzXBoC09/", + "blx5Q5QpM0uj+WTFnCo+2pAsxmnq09n1I5fk/uHDyXGJOTDeHz4fPH/Rez4Z7vd2w8Gwh4c7+73tPTyY", + "7gTPdhqKmLVPUbh91oHXQvNVpXKxh2MXA+ktPNwQgVo5Im1U3RVlIb9qVbI7692GUK3rvh4g2XoI3rBq", + "qHQMLTVIidNCPWMTVlipWdTgedp/Pxiu8Ty1K8PcIH/fi5QFBn0KJHHm0y0WYC4uVr0I683FKQzIhS+v", + "o1ax8/ZEGxzsvTjYuyvRXAjuujFW2ekRF7cp4MBBGFZifF2eScF/UahgDPqGcbPakOBOt5NFLcPfcNBW", + "IuKyx61C8Zs2bNcvRlbJ74aMtJOS2gy3sAbJKDzQWkCWzDRJFcoSHbV6cRTxNEQF348BdoGLkZOCCq2b", + "gXsK6xoyCF0mpFar2gAJCYDElGlBDBdCuhGbvnaAXsO78AjHxrqwg8AsrNyF4HBp7oL1/nJdG11/9ZDP", + "rZoP32idH0FhcT1tTQbrIlzdhNF8DtAbDt9kRgfjVV+jeR20/frrVb/khs0rc/nH0JlV4w7Qq0x1y5Q/", + "q+xtSGL/HFuBlaf9b5aSL+2KdzS35CtXyCvsdgxFO92OIxTkH9YzET/kXF/bf0VW9AVJEBzBXs4zvVJF", + "I4tyCTOhUFrbhgDrxW3SLywgPAnHxjBpCnky6UPWeMk+curLx1O0AXhWf0PWkar/tZmFR5XOuu0Xuy/2", + "n22/2G+FWpEPcL3aeQTJbfXBrdVBgyQdu6q1DVM/OvtgTPDAGLfgmbdzLyQJJ4Jr0aNnnpfBzTt/0X9R", + "BOsIeWpKgtshWWSfr4XC9ytrFjfE+PxBowWdTtkfn4PL7X8IGg+v9+X2xOvQzCvse70/J8Xb3pqrlEx6", + "poiHH08BGErIRsiRd0TCDNA5UQj4p4dwAKZDlpNmWc4Bk1iKexlrd2dn5/mzve1WfGVHV9g4Y/AFeQ5l", + "O4LCFoM30ca783O0VWA406ZL1AWAUGbNSv8+Q7YU2KCskPaHgx0flzQc3DnX2LYXcSPJP1rTzE7KEh1S", + "6zKzrbbLvdTe2Rk82917vtduG1sv5Vhcr5YwLvDckMfi2BZXfgO0yfeHZwjSuqY4KPtNhts7u3v7z57f", + "aFTqRqMCDGaDnXqDgT1/tr+3u7M9bIed44sCsKhQpQ1bll2eTedhCs9qeEhRF73dptPCp04ZBntHggjT", + "+DBwEbyV08dgpI6FeS1fhDYHg3WM1w6uFt+2chxVSk4b1YALlLIMmbu//grwdjd6zWLanAfrxXjdso8w", + "0+SyIA+mFMctaJcIsqA8lffQEFcm1WkacS5u9G2ThfKOyDRS5tqNSvTx9C8gRDRzIalIUjaaLPutgMK4", + "5eRutIFLPOHn6iZitVqNNku/asLdhm3aXZUHXdr+jYgzoRZVKVsffneEoyAF8HmcraeeFWBHQCZnkkRL", + "E6gaRZwzFMwxmxEoI2hKXbAZwmjOo7DvDR7UT8ZT77U9v0IRN1iZl4QkFpfdDEJ/pnUWuiBoo5BJigwr", + "Vcoz7cVGqljk7TI37sX+QkBY+pIfsgxGTU+seAHG0XxS8jFGfCbBClQQgtuvogcnWJjIWsxMnYFFbIzH", + "MvTOtj7tPUOsSG/fEWqOTj61Fq3VMSA/0FASB4JLiUhEZ4Bp//G0kna2IociSz5bH1JXHmwL1jUXaZ6z", + "C8402bocie9A9ASn3+VIBB6GHJQVwWrOGxljlgJSe4GRyXVChWGPdgFpcy7VOIMTueFgpRoDCncqSI45", + "lCVLZg4g9473XHSi7TbkspGft/q6xlX+ppoG2CxTvRT1U6ub8aCPjeuAKisxXHJQmCoCyE0gf3LYZiqh", + "VVpAm0EbjKuSWCpAD2+2Cc7w26i6n5p5aqtM/bY7OG+LxrMafOcMq/kJm3JPyvYNriGd69nGCyZExBRw", + "6FFIGCWhMx6z+0jr24I8v0gSFKbEUs4opAJbgmOzvSHtmjmnGGWziqyvdtjGH2zGsBqkG/q1L7YJs5H+", + "7LD3IgVamcA4iXCeJ9YqypDKsf/+qt6wILM0wgJVEadWDFku44iyyzaty2U84RENkP6gesk85VHEr8b6", + "kfwJ5rLZanb6g3Ge5lC5NDaDs0kuZkEq/eZT+EnPcrOSYgeuly3z/RYk+reJWvLG6r6iEbGgTB8YvS4w", + "ehnFdnd70JR92dBoKe+yDuh1U8ltWda34x3W1mFWNNNzS2mibit3pWVH5NqbPgjrXpVrWnfFoA0XCOVQ", + "gst0LaD1tvKEtIssr4b8udFsSRKUe999vvdsvyVc8p18nSaF/b49m4t4hUezYaVO27jNnu89f/FiZ3fv", + "xfaNHFQuOrRhfZoiRIvrU6mNW3Ga7UEk1eBGgzLxof4hNcSIlgdUqnN76wF9XbF1m4IL8r3ZdMkZFVfS", + "3bOUPaDtfIwrtKXDkspVqAK/QaZTAkbl2NCtlw+mkiHYagwBTnBA1dLjMMFXpmRf9koF7q2NN608WA9J", + "bdsWGkhLLplO8iSKDdc5+qtxrVd44Xlr1HWZTprc+G+rvRonfu4DKl4RtbihyQtD1t0F2XyusCzFmum/", + "A4iezKv8V2NmzRuroaeqgB9wCWiLCxQiKXyQhZXzz35UXP7KchbcviUluUrxVUdo8xa8kQ3tOZE9JnSw", + "PhOmIh/sAXi7r8aTYj2ElQUnSsUT8lP35v22SPapg4VmJ9jN+yukPdzkwyowFvCjHYMled52t8QSDdxU", + "CNP1mCM8Ir0szsHG8CKZGv+q3vMWa9GThxFc8um0DPi01wwQCMh8ELLtesFKkThRXUSuwUwnYQ1dztXR", + "3pOjDuICjTrDeNSpOAG9SRAxvh7bDsq5yoNViH1ZJaDqIKWbwSTiwaWB+leCEtlHAxQTzCRKGWz+io9y", + "OFjta+t2ksLaZPh4xNwQ18QWjGlC5nhBAZbVeqhmpTgWck2VhHgbaOcAhRzs23KdIztD/ZpJUTjIJw2H", + "DmZL27BuUL/HmQsIyt8Fc2kK1ZXYZyJ41ybfaYn99u1p19z/QOSGGVgpPMRN1IxAC8isiwrGaP67P/xq", + "EpExjLuKWRnX6VhMJQM/tSCSKGlB7HJ2qDABCnjKVBXMMm4XwlmOeK8fSSmDWAl7ewaQDbZ3Wx0xJIGt", + "QFt3L5UY/RbMXQm7tJT2xV3u+FgYNgV45vye93fWvV4dgERYkGKJMtNOMSzO+FzHUnGLaZ/t6jG5DggJ", + "q3A8/lfahhraL72hhr9hm/OeVQ+zb0O4WH12/YeLPYexNlG7GBLJOOtBtq1bUpsZa1BDbO51mdFK+HuF", + "DNSxD/zI90KbvClyvZrWb8i1ApDAMI00eZtY14oqexito/itky6aNjQX64ttPkDtBBOud6vqCTbS71EK", + "KNifH6RoQm05zoly755bvmmulVmCNy55BF2ApHulfEVpeKeL7ImOhvFmhed25343iAXoapkvwnBMxokg", + "U3q9glvMC8YSLmcj5zunVE9Voo0YX6PdZyiYYyErY2d0NlfRsnx/uesBA7hTKRFBFGHqBpVn89V0H9aD", + "BexyFlv3acPnhdR9f0VcEo5XwdgdZa+569gEL8Ft0+hjfbazOxjsbA9uhWN3X4V6C+00pSAUvrP3JKWo", + "nmILWcJXvZrTlaCQN5aRSSpBcHwAgcoJDgiKyBRQXrIqemsPi1rXqwdvNSibR57xv1sou27uCqMMFJ11", + "ZSEA3TQ6LgCqDB1QfF4f9goomEzMBDVMGE+Owk5vsP9+uHOwt38wHD4E8F1GpKbo2Gefh1fPom083Y2e", + "L5/9MZw/m23HO17D6wFqQpdzPqslou0cEiKqZbqq5e0kiSgjPZlFlK9P61ghC0yQxtr9fzPHvpnBSmXh", + "vDzJos6AVU6cEmc9Ek6GHf3K24nq8E+OVw/7ViHa1YH4Gaw6FOCndoMBINbhXVE/U9by3PlQeLH1ybMy", + "bWDd2ePL8oSt7V3lBor7+LkkGEs7bNWJXT/VPN7RGRdUzePVx0P2WoYhCHFmn6UKy7gMfXQyY1CUr/hz", + "FlZQNJP0x51uJ/q8W94z9vf2CB0WMy9jQLvURTWgxbU71HxcTQV4JTcthIn809a4HvNPw97wBQS/RZ93", + "fxr0XvTR3wtBeF1DrSL5hu7t0q+DNjQsVrpwCOnDFzeKUHP0XMVBv1JfnYb8ILZoepbH84po7qxwGUml", + "Bc4f19a4AiPQqHHeVbWzp9m4qCWFJMJLXx3rgitWVuzDIpOhCZlRJtt4ZncGmWt2Lx51+ujQYlCCtZoX", + "vCw1D6UOC3xC45iEVCuVxrhvjvjcbultqxoPNwMmdl951LO+Xz97sT6HdF2A+rpjsn+HhKU7mbvtTNxV", + "6c3gOXM2KZRAgRe7iE4RZpU6PLbqrM00hMwRQJg5cIAqOctaGSBzxc95QrpoxhXKcwxbetRS1uz5y8ZP", + "rsGjuiKp2DDE9r1kjGfoJXSV+Do5RongYRrkCTYRDDpPiRZpBcxxhVa/PoTpIR0akLk25QKtd2g0eTDa", + "eSCb1rvifdQM27zUw8H6pX4QL0i3kybhehlmXmonwW6ENbomZcPjkymTvaIJFibzqYVEf1ekYN3INd7i", + "QKtEaeKuUDRP1TnJc6EClwg+2L1jEhF9TNUbQTwK86hSKnMpul6kDvefz5suMeHOqT6QXwlJtK0CABHQ", + "X4zZ0juwapFVtDFwlbOkudLqGaxyS63y4J6t1cQal6p9vdqKV9sklBfLA2egm/dbrNZ+ubaU+EP44b6l", + "kvbWXi5U4NUc+F8GV+r6NxkzAM/Iosp5veu7gPexxXtrGTdBxlWzZopu5sPefxu3Mhr3D7Z++tv/3fv0", + "V697uWI3SyJ6IZlCKNElWfYA7x5pG71fBkwDrF6tTNva/orgGJxGwSUxTqoYXxfHuzfIhMbyDY5rU4AY", + "rJiy7N9rJ/S3f2uOYCqQ8QPIybUse2co64eoE6S4O442YiJmrpKxC7zf7I8YxPJfkqVEhWIEVqVxjPoX", + "mX2iVXRwWuIIXRg1sE/Y4gJNKNR0kSOmrVocBCTR1oQFZaemGB8H6SMIjort2KIILlHOXjmaiAGCPp7W", + "0Pbefnj/89sPb47Hb89evjk8Gf/68r8giOOqZ3oIe5r3dvf2bQm+IiWHniW+A/DvnVD8fOxmsMA8/AVJ", + "KVDX0KMwUwkppS6goPAy2iBxopau2JDLbdm8GTbZYdagN5ztnlHYBy/uo+jMh5VVZhY86mmNugFE1+vA", + "NLTwhmNDUybMvdPk2J55yoCfW2/ijM6wx5ftrUB6H8Vh3IDWgsbV1r+xtIM/OP64ijBspIEhVQURt2KX", + "StVrjp2PtSI1zms9liMyUmbTS2ghYKucSxIztWWLOPlSWkN98q5OKMp3mUMs6sFH6/NkVqryhZkVRtK8", + "NqdOY63o1CsIdKZJczUnghQWAj7IkVtvSDKb7NEiUdqUWkmIyAMhXaYI1D4XFLJHMmeDI0GWEFT3wK5G", + "5j3F11kP4L3HsnbHBfPIMfKHr38edTb76J0rVUqnrgkYRsWe8AOhlrloFU0cV9UXo8hV9Xmb970bz8qq", + "FdKvaW9VmDPvo8SaPn78O6bqFRdggTSnJT84nipYNyERgMtSRUttBTVKYxKOs4LNTfvf1Wg2OclZOey8", + "jJOztjAwsRZy60vtuMTZfAx1SmtykCAVVC3PobCriRAmWBBxmJoN7+rD2p/zjqEq0tev4KecerIQXhNG", + "BA3Q4dkJ7McYM1DS0cfTQiUTU9SmhqEG6uXboxNr4ToYPrBYqALWc8F8h2cnnW5nQYSx8jqD/nZ/AJs5", + "IQwntHPQ2ekP+4OOqegLU9yCIorwp00wzCylk9DqQT+bV/RXAsdEESE7B797EvUgkA1eBn0XzwoWS4Kp", + "sCZLEkH6oGEVqr8FYF13lB6Y89gW5G3toJNqaZMpSPLWLusnUCdh18AUtwcDCzOq7MELqSAm/nzrHzYY", + "Me+3lT4H5PGgzNYsCqdTWpJ/7XZ2B8MbjWfVMGDH+rr9wHCq5lzQzwSGuXdDItyq0xNmMryQwQqzgTbF", + "fQYsVNxhv3/S6yXTOMZi6ciV0yrhskkZJhJhxMiVrQj6Dz7pI3v9AGVj5JynkZYmyKSvOUeDwqI/+4yw", + "COZ0QUbMntNxGimaYAFuhBjp89kYTOWtYbo2q58BC/zMw2WFullzW7q5nnM65wSupiVIMgYc4nFTud/c", + "3UwZ02ISS2JraWR1L+vRPFpcjmXAfflE7wnDTPVkQgI6pQGCl/XutR5tb4OtoPm0wINlIQIQs5yHZnvT", + "n5MKVT/86dzH2TNkyVtWJxhcAwVRGuY6l0uTwmKCo8iL3TSL+ARHY0OfS+JRUV/DG5YoxQIpTrlhPCSm", + "2EWyVHPOzN/pJGUqNX9PBL+SRGgVyBYxs7QmoSlbZlj3CrA+YygkZkqk6j63zBC3vlyS5df+iB2GsSt/", + "K80nOJJcn5q26KDNCzBb2vCuvyxLQ1zJUSoVjy1LZRUY82HyVCWpsnfqkihbeQ1epxIlqZyTcMQUR18E", + "mVGpxPLr1pe8x69guxAcaj4pvGKmtPWFhl+bRi3HWM9+DK96rD8CBBh19Oky6ui/ZwJr2yWVc3CiSHCc", + "zIpLupHh6Wi9cLNK4QAzlPDEYBEBU82xZrlSGxCPjqMIKdhK7lutbcJKNszHphfHk8bcYpMMWtlGlKHT", + "nwubabD73L+fJAkE8Tk4/vP87RsER5VeA/Na7rAyl9pMn6IoTEGTh977I/YSB3Nk9CbAmx11aDjqZNZF", + "uAljTaWN0O71QMX9SQ/tJ9NNl4Y/9fu6KaM9H6Dfv5hWDvReSuKx4peEjTpfu6jwYEbVPJ1kzz75CdqU", + "onleEgRow8j+TVeDGKCi8mPQnBuYhYhbWRstEUa5BCr6USaUYbGygLKH9JaC2pTHM1kkxpcR+G5HnYOR", + "896OOt1Rh7AF/GZdvKPOVz8FrBLdDG5qakg7XTtjov3BYHM9doKlr0eFLr2ot9/Xmva1fW+Kh1W66oqH", + "mZxDZtYraKqBG3XrETSfn3Ho6kv+UPHWqHjWc1FQ3uD74jlg2DcixsCtaGDano2cBrbSOjFsAQUTwOJw", + "SCfG4KBOg8uZt2h+VM35ulmx27TLAhhi5Phv9xH4D/rNShGafl88Vr84ApxxB1z/xNgRFssxYtdvEb8m", + "6nvguMFjiVILiv4t+fep8M9rYvW+nGgVabZFFu6+yY/nBMkm0rZiXta26jmMqXdOmEIv4de+/a+zeCBb", + "+iLis4sDZEgY8RmKKLP3gIXbIn0oWlrCRybfJPvOpp84MM0Nc37+83/+FwZF2eyf//O/Wps2f8F23zKJ", + "kwC+fzEnWKgJweriAP1KSNLDEV0QNxmAxyYLIpZoZwBqZiLgUbGkktVN5IiN2DuiUsEK96UG11LaBsH0", + "YDAfylIibb6OfpFOLeiWcTB7THi3lw0pH3VHdz2JzjCDwgT0qeh4ALJEbfk1a391/N4zM+eS/6zqK695", + "TNfLF0WuleHenhngDQUMkNi37+CBnTTaOD9/udlHYGMYrgBgNdCY82as8tz/IZPWyyQjUcoCBahsZJNJ", + "Ylvt/z2277RzANsW/0weYIu1eQMXsHF5QO66W4EftkILd7Cfbs417PPPHrsszWYH7e3nW+zCxTG1MoTv", + "b50d79Vpbp4USPYtTGC04aLhwY3IBTo7OnE1bTe/GdM/yqmhZ2pr9WVHB+IMsNcezSw74mwa0UChnhsL", + "FGaKSWaqlRnkqYiDd3bUCLt5VSGMi+fbVgmRr/Gky8D58iPv4U+PSqc3OUZymOWc136cJOtY55jKgOtv", + "C9zSC3AChHTqS7ZPi1y0ziFlguuzI2elumTF88mx25CP55qyXaesejY8glA8rgjEbygIK8XjC8DkT4mb", + "P2Sr6BApVniuvi/WHDyeFvTYXiwfmz8lN1ZYIZuWgiaou/EAfU2UCeXuPOBC2x48Ez8nwu1qV0cCZp1N", + "y3xqCquaCcGF9Grb98S80s70Ne39mSxfIM9NNBZL8h8qSgtjN6fVKgP3xJYsfzj7Fnq4kXl7f/e8lsE8", + "RIZgk4nzWJu6u1guWbD5p7rqfZTTzBD7SR5mZ2kUuRuPBREKvT06MTureAZsfYGwpPW6vdttK4+DD+9+", + "6xEWcIhDy2Ko/EqUfXLPGr5ZMDOVH2zSxiY0adHUnWdNGs4d1t+ECyIT4din/N+3X0V0IrBY/vv2Kxwl", + "lJF/3zmMsCJSbT4YswweSzQ/tsb9hJlPK9y0TDQQTQzqva/TULO3Wiqp7v0/lZ5qJn0jTTWj6w9ltY2y", + "WiTXSn3VLsWDaqymj290JZMxm4/a8MjFJ/7JNNXH9fJZjnRA0VSWrz1skT0uwM8LjyhDqSRPMICSZhxX", + "PDZauqvzDbny+HCse3LcBUJCXQRAbbIJIo/kvHbjeHTl1vb7+J7rw3hCZylPZTH3JMYqmBNpk5UiUhbA", + "T03tzo/nRsX7O+bSwWMeHY+uV//g+wfS+KsLaoS3uYFap/O7t9rq/PZ9rfObFGqbu2ahpboOdnCzIajQ", + "JVG3ZeNSrnk92NE3Lp8tgj5oQyU3FxBYEAcj9n+0/fG7Ijj+9JNLkkkHg+19+J2wxaefXJ4MO3WsQhhU", + "PAKU2MM3x3DtN4PscwCSzVPyquMwlSeA9Rx0zr+cgZTffLa3kBwX/rCQWllIBXKttpDsWjysiVSG33p0", + "G8nxm4/gFsTkh5X0GFaSTKdTGlDCVF7ztBYkZksmP8HcMmbvhwrBHaWDtrWVlG3KNQpoXhbg0QN7TnIc", + "xMc2jlwFgqcZI88TC+ltzZH8MGy2R743fhg8rnB+fDvkKbOYUfjrpEu0TukrpQ0Yk3EKZSFRjhACUZ9I", + "2PqPrsU+yitYyzRJuFDS4FSCAmyQ7OdaAfZhWpZhKn24lIDFSInsjhhUKtCPTS7/1iVZGhRKylkGOFmt", + "x+rLvSqjgH7TbXT/OpYf4rSVjvXI29iCVn87HeubiY5H0bROSrUANrKNAQblhGQ7mWfJffQzZbPNJxWB", + "aoRVNrcCnpFH1dqCSn8W13dLZtVkmw7aArSvLT37L3ji1ifp09odFm2BgCikeMa4VDQoVt4twoP+OKFb", + "n9CrKevl5qktke436F9xcdn2iPPUFXsCJ11xht+hL0EPD9DAvr1LAYxtcxpopnn0U7BWLO5bpmDQ6rkY", + "RGkIVentgehUyang8dj+aPBq9a6waKDgoghsq99a2OjeH8Fh9IYrROMkIlqLJyHqGW7Sq2lVfwc3T2Wh", + "tOLNhKHeNsWEGANGJ11pIisi4XLNLdgG3LPXl8srNSM+Ww+CkXXuEB88KBgjZuDwicPOv0CZkIXiXSQi", + "gUJXcxrMAREDCnpBRVcAq8BJcpFBYG0eoNewU4tIYND5hiRCG0IBZ5JHxABdLOL44qCO2Prx9BQ+MmAY", + "Bpv14iAruZ4dEFK/VUS4yGoevbG4HRuakwSPIrOiF9pqLMxv02Jf5BBlI+bDwWDkyjZIp+iiAIlx0YCJ", + "4QTqb3z2zbStbjOwpJmL4kgA4QxvEhZ2mi5iaORHwxgOvPVgWiJzmGE8MDBHbTC/8VkGalliZZwkbdnX", + "DhO4eBHHK3gYbRSKs0oV8lT9TaqQCAEfW+5uYm60gQPzD4UvNaPawkJZeVtgP+91o0GZ85JKC9VCFR3z", + "r0Ucd7odO54COt0NtPc1CCfVBuvXYnplCjAmP/TumwCUlIV9AaGkcnLY8v/NKvc788Kf3j9rCRX+Gbws", + "5fusfBSU5QWgBFRwd1W4nhTSASxkTRczhZF8e8TNsicLxUPbXW/Vyo5+B0bruluvrIZkVuDysa+/6iN4", + "ykkwsjabKRfV9Ph192LfPSPd35LUptqGQ37w5s3dc60YM0lX1BKFUqgS/HxQXxNwnYM557LA9hMyxwvK", + "hUVgt17XjDPBZWGsRxs9d6FZ9cL6by+sen5gfU0IFx/ZPvrwuY2583/hHuVfvCpY25nE7zqVGlAgJcJo", + "IiiZogSnkmhtKY0JMhVGLJA3wcHcVQvvj9j7OUG2PmbBgZCVU6YSXQzjiy6apApFWMzA2jEPTSSdIAGP", + "Y8JCU/N2xOYEL6g21QSKsCIsWPYkgRrIC5IXMNGmu72hNKW2syqrXeSK84KD4aJQevcCJYIAExlzmZXq", + "3I6YSNl/GORK3eyFG+gFIlLhSUTlPKsVEeCQsMALC3n+fYux+3finhNVr077Te4sbyVLv+UlZtGXmdUH", + "/y7uN59YoBYXrrJmCzG/QumVzaZhOfLxPK/I+y+4pc1c3Ry/0c1MRuJVu/j7uJIpleT/cS2j7JYMU9Md", + "KZet/9PeteR1pFNWum6xPtnbXrhklRAyMt9I5m19cX+e3MJH9p1Iwm6jYd+EuZ1P+nsQuZaqt5K538g5", + "aH1JBa/YNxTBdlDfTn3ioiDlvgsxbDZcJo2LMkcJDDYVZz+EcVUY2/CA2wpj53GtXYAXxDNlvSTCTXI5", + "r1rvF8DWIfAvGv1amV1BEH5zwZffCDyasDvJxJsReAleRhz/2e9lAi6ESei05YifDqBYwRdYuGDaAI9b", + "N5MQXZdN8vH0dLNJSgi1UkYI9YQlRLmsaRB7qjW+XRAhaOhKRx6dHtvoVSqRSFkfvY0p1HO8JCSBQjGU", + "pxJBZm5fz8+lttaL4JVyWLsdwpRYJpwytXYU+asPM5ivtyqd98hy0kIq/ukvj8EL//SEFMgOra7YCay2", + "IhVWjcF4LjiNMlPvUmtbeMJT3bqWLK7Q7gzOtimNiFxKRWITmTdNI9hEALprazLZ70xGaRdRJZHeD13I", + "wEuIiKmUlDM5Yrb8e0KE7lt/DsV/8yAjr/Ne4UxqnhnR930EsOnBmJgtrJqoBtACUAe0c9DZwkmyBeWi", + "/UFSdnh3GNIriEhDchlPeEQDFFF2KdFGRC+N0YEWEkX6j82VIW1j+O6+K07dfmdpSp+wKfcW5TA8mzHz", + "nyMJqSzW3CXikxNrr0lxszj5AwvtF2tyrVwTBEc9RWOSJb+jVNGIfjaiTjdCpaKByavJUy+hCLPNvhyx", + "U6KEfgcLggIeRSRQzrmylQgebI3SwWAnSCiglOwQGBwIvObHMfR4dPYB3jOForsjpv8BDb8/PDM3sVNs", + "fQSFgTKirri4RCdbb9cE+Z4Dmf6Fo+TMBFfmQHoX/Mf13c0zmxv3kGzYojxZZQDx5E8fxmk1uB/egqfp", + "LQBoiWw2GzOBA1CK5TxVIb9ifs/AgkdprP9h/jhZB1CicDD/CK9+N9quGc7abtwEn8SmtHMKiSka9E0u", + "KAzBnmp8qSacmwIoMaXIPe8pcKj+jNx9/075Ih2/w6tJS1FXkOu72VuPffLZMTjcrSI9nso2N5zmZqL4", + "au/TFabN3qefIx5cSpQyRaMSqIG22wAHVP+Y4zbaiz9QEyA70pUSR+Q6oQIQbCrwCIjoGUuEkSIipgxH", + "WzBn0wggUDovFl5wCknKQUQhTYyGBCU8igBl52pOGNKzAUeVa6BwTyttBYjiO8UrRsXRhAQ8Jg6Vc9Nn", + "uv0dU/WKizLE5vciF98X6K/no6eq57kGVbS5xzuhjJ7iawhrDlN7TexGtPGa5z8aV1AXwdqMOjsDOep0", + "0aizHY86egWOMLhQsUJ7KKYsVUT20bHxb0Ea6v4ASRJwFkoHDuo8eDsD2ZSUatiyIcNxH757TLXHchWQ", + "8p3txCce9HtIfw8JNmijuOHsngy7sOlCxFMFAdxuX9m3QqLAPbL56DewhT3yw7ZvI8n/brdvSUbBKmtx", + "WVh6I9kz+Mi1XjeXVDHnMkedRAFOcEDVsotwFPEg9x6kMrsd6GVDmQiCL7UN1R+xdxlwpU2EQEdnH7rO", + "aYZCKi9NC9Yv1kdvF0TIdJINDoE0MB48WAwSjpjiKMBRkEaab8l0SgLIYYhoTJVs8KtlQ3nIMoh5J56F", + "dw8z2Jqn5Uzy8wSsXs4WssJxW2aptwQJIkzjolOpShxQfeFKF9y+E90o18fwNLLXW4HgUiLbVI9EdEYn", + "kb2skX30XqscOCYjlkSYMSJQKk3ckR56LxFEytQkxugGoM6s4aguyoFOEsGVdRNHnAtpPLuawz+eIqlI", + "soLN3pmWT2HODwQTbBq3PX0jg6EyhuZjyb6C9IIYTjEE13ykj+lvEOxjBvSt4YSfysZ/L+hsRoTeFdgI", + "WXM1ara1I6fZ9KVMj0aM/PPsrXYY+VmrhWjuQqTzSqCKsXtxDAr0TW5gPZ1f0kYsE/voZtkXv+qPWvZd", + "jvL3D8I+uuMs/yylx84LwdVtkfVzDn9qIPeFkZe2ailBYT0cQeuMhIfMEGiNO/DN4AaeMsoALqUdNMEJ", + "fH+MMHjc7LjHhtl+2rxVQgkoFdZpSJVaD9/5XXDgw+B2fuPs0Fvgdn5X+UqAu/jt8ka/q0ylkh/QFQ/5", + "0yNzPlSCkoHnBBiLpgQlI/VsIMFKQ+mjfaedmWRb/DNp8Pbu+Qb6uyP7D6u/hclQIJbfZWdyox1uC4kT", + "tXSXi3xauQCU9DMkY/iAH7IYgofDW7jF9fr9sYfj08bL9R/1tB7t/j4vOnxy/PSLaBX3XOlg2dKnTg+L", + "YE4XpNnpXt7BlkSJIL2EJ3C5EhqCWXq4s0xh0Z99RrZ5i1Vl/4WogzgmIQqpIIGKlogyxUEimD7+IpHg", + "2hKA51wsfc704s59JXh8aGez5jy0e8o6w/I733jZC7HCvYWTNitcaHe4aXd321rgIcrQ65/RBrlWwiDu", + "oqm2fBCdZiQl1wEhoQSe3CwOeDho8GzSz2Q8m7QZ5Qrs5LcWmxoFqVQ8dmt/cow2oNjCjDC9FlrVn4Im", + "mwi+oKEpRJoTdcEjQ9VhA0Fv6nfVSkVWKcMZF2Zw30SHaXMgzT7TpCwWTOhC56AzoQzD4NaiFJf3lEmo", + "0v1hCmkN+d5xnNP5cYRZy2/DGTuaE7WR44ioODfQeJs/jrmnfMwVA1PdmVY67dqVimwXq9oyhPQhAHOz", + "OObHdVt//H7CK6l8kpGV1nW+yAzSJrf598WCg8c7Hx7bXf7xCYfjvybO+C64yqEB3aKPYX7jAY5QSBYk", + "4glUkTTvdrqdVESdg85cqeRgayvS7825VAfPB88Hna+fvv7/AQAA//8QadjqX44BAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index e6d03230..c6bce872 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -690,13 +690,6 @@ components: Optional final state for the forked instance. Default is the source instance state at fork time. For example, forking from Running defaults the fork result to Running. - wait_for_network: - type: boolean - description: | - When the fork result is Running, wait for guest networking to be applied before returning. - Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. - default: true - example: true ForkTargetState: type: string @@ -882,13 +875,6 @@ components: Optional hypervisor override. Allowed only when forking from a Stopped snapshot. Standby snapshots must fork with their original hypervisor. example: cloud-hypervisor - wait_for_network: - type: boolean - description: | - When the fork result is Running, wait for guest networking to be applied before returning. - Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. - default: true - example: true SnapshotScheduleRetention: type: object From 209147e71528b89377c8a335e1de2081593a93e2 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:39:08 +0000 Subject: [PATCH 09/16] Extract resume network handoff flow --- lib/instances/restore.go | 49 +++------------- lib/instances/resume_network_handoff.go | 74 +++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 42 deletions(-) create mode 100644 lib/instances/resume_network_handoff.go diff --git a/lib/instances/restore.go b/lib/instances/restore.go index b2ca8063..c8645e0c 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -248,39 +248,13 @@ func (m *manager) restoreInstance( proxyRegistered = true } - var resumeNetworkAckWaiter *guestResumeNetworkUDPWaiter - var resumeNetworkAckCfg *guestNetworkConfig - resumeNetworkMailboxPatched := false - if allocatedNet != nil && !stored.SkipGuestAgent && guestInitiatedResumeNetworkMailbox(stored) { - resumeNetworkCfg, cfgErr := guestNetworkReconfigureConfig(allocatedNet) - if cfgErr != nil { - log.WarnContext(ctx, "failed to build guest resume network mailbox payload; falling back to host-initiated reconfigure", "instance_id", id, "error", cfgErr) - } else { - payload := newGuestResumeNetworkPayload(resumeNetworkCfg) - var waitErr error - resumeNetworkAckWaiter, waitErr = startGuestResumeNetworkUDPWaiter() - if waitErr != nil { - log.ErrorContext(ctx, "failed to start guest resume network UDP ack waiter", "instance_id", id, "error", waitErr) - releaseNetwork() - return nil, fmt.Errorf("start guest resume network UDP ack waiter: %w", waitErr) - } - resumeNetworkAckCfg = resumeNetworkCfg - payload.AckPort = resumeNetworkAckWaiter.Port() - if patchErr := patchGuestResumeNetworkMailbox(snapshotDir, guestInitiatedResumeNetworkMailboxToken(stored), &payload); patchErr != nil { - if resumeNetworkAckWaiter != nil { - resumeNetworkAckWaiter.Close() - resumeNetworkAckWaiter = nil - resumeNetworkAckCfg = nil - } - log.WarnContext(ctx, "failed to patch guest resume network mailbox; falling back to host-initiated reconfigure", "instance_id", id, "error", patchErr) - } else { - resumeNetworkMailboxPatched = true - } - } - } - if resumeNetworkAckWaiter != nil { - defer resumeNetworkAckWaiter.Close() + resumeNetworkHandoff, err := m.prepareResumeNetworkHandoff(ctx, stored, allocatedNet, snapshotDir) + if err != nil { + log.ErrorContext(ctx, "failed to prepare guest resume network handoff", "instance_id", id, "error", err) + releaseNetwork() + return nil, fmt.Errorf("prepare guest resume network handoff: %w", err) } + defer resumeNetworkHandoff.Close() // 5. Transition: Standby → Paused (start hypervisor + restore) restoreCtx, restoreSpanEnd := m.startLifecycleStep(ctx, "restore_from_snapshot", @@ -336,16 +310,7 @@ func (m *manager) restoreInstance( attribute.String("hypervisor", string(stored.HypervisorType)), attribute.String("operation", "reconfigure_guest_network"), ) - var reconfigureErr error - if resumeNetworkMailboxPatched { - reconfigureErr = m.waitForGuestResumeNetworkUDPAck(reconfigureCtx, resumeNetworkAckWaiter, stored, resumeNetworkAckCfg) - if reconfigureErr != nil { - log.ErrorContext(ctx, "guest resume network UDP ack wait failed; falling back to host-initiated reconfigure", "instance_id", id, "error", reconfigureErr) - reconfigureErr = reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet) - } - } else { - reconfigureErr = reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet) - } + reconfigureErr := resumeNetworkHandoff.AfterResume(reconfigureCtx) reconfigureSpanEnd(reconfigureErr) if reconfigureErr != nil { log.ErrorContext(ctx, "failed to configure guest network after restore", "instance_id", id, "error", reconfigureErr) diff --git a/lib/instances/resume_network_handoff.go b/lib/instances/resume_network_handoff.go new file mode 100644 index 00000000..9b48a8fa --- /dev/null +++ b/lib/instances/resume_network_handoff.go @@ -0,0 +1,74 @@ +package instances + +import ( + "context" + "fmt" + + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/network" +) + +type resumeNetworkHandoff struct { + manager *manager + stored *StoredMetadata + allocatedNet *network.Allocation + ackWaiter *guestResumeNetworkUDPWaiter + ackCfg *guestNetworkConfig + patched bool +} + +func (m *manager) prepareResumeNetworkHandoff(ctx context.Context, stored *StoredMetadata, allocatedNet *network.Allocation, snapshotDir string) (*resumeNetworkHandoff, error) { + h := &resumeNetworkHandoff{ + manager: m, + stored: stored, + allocatedNet: allocatedNet, + } + if allocatedNet == nil || stored.SkipGuestAgent || !guestInitiatedResumeNetworkMailbox(stored) { + return h, nil + } + + log := logger.FromContext(ctx) + resumeNetworkCfg, err := guestNetworkReconfigureConfig(allocatedNet) + if err != nil { + log.WarnContext(ctx, "failed to build guest resume network mailbox payload; falling back to host-initiated reconfigure", "instance_id", stored.Id, "error", err) + return h, nil + } + + waiter, err := startGuestResumeNetworkUDPWaiter() + if err != nil { + return nil, fmt.Errorf("start guest resume network UDP ack waiter: %w", err) + } + + payload := newGuestResumeNetworkPayload(resumeNetworkCfg) + payload.AckPort = waiter.Port() + if err := patchGuestResumeNetworkMailbox(snapshotDir, guestInitiatedResumeNetworkMailboxToken(stored), &payload); err != nil { + waiter.Close() + log.WarnContext(ctx, "failed to patch guest resume network mailbox; falling back to host-initiated reconfigure", "instance_id", stored.Id, "error", err) + return h, nil + } + + h.ackWaiter = waiter + h.ackCfg = resumeNetworkCfg + h.patched = true + return h, nil +} + +func (h *resumeNetworkHandoff) Close() { + if h != nil && h.ackWaiter != nil { + h.ackWaiter.Close() + } +} + +func (h *resumeNetworkHandoff) AfterResume(ctx context.Context) error { + if h == nil || h.allocatedNet == nil || h.stored.SkipGuestAgent { + return nil + } + if h.patched { + err := h.manager.waitForGuestResumeNetworkUDPAck(ctx, h.ackWaiter, h.stored, h.ackCfg) + if err == nil { + return nil + } + logger.FromContext(ctx).ErrorContext(ctx, "guest resume network UDP ack wait failed; falling back to host-initiated reconfigure", "instance_id", h.stored.Id, "error", err) + } + return reconfigureGuestNetwork(ctx, h.stored, h.allocatedNet) +} From d8e6a3ba9997befddefd93447e759bf4ee223586 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:39:25 +0000 Subject: [PATCH 10/16] Close stale guest connections after network RPC errors --- lib/guest/client.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/guest/client.go b/lib/guest/client.go index d2974bf4..b6dcd2e8 100644 --- a/lib/guest/client.go +++ b/lib/guest/client.go @@ -161,7 +161,11 @@ type ReconfigureNetworkOptions struct { func ReconfigureNetworkInInstance(ctx context.Context, dialer hypervisor.VsockDialer, opts ReconfigureNetworkOptions) error { if opts.WaitForAgent == 0 { - return reconfigureNetworkOnce(ctx, dialer, opts) + err := reconfigureNetworkOnce(ctx, dialer, opts) + if err != nil && isRetryableConnectionError(err) { + CloseConn(dialer.Key()) + } + return err } ctx, span := otel.Tracer("hypeman/guest").Start(ctx, "guest.reconfigure_network", trace.WithAttributes( From 7d0b8c8833b9364b275433e00c696702cf7e8cea Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:39:53 +0000 Subject: [PATCH 11/16] Trace resume network mailbox patching --- lib/instances/resume_network_handoff.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/instances/resume_network_handoff.go b/lib/instances/resume_network_handoff.go index 9b48a8fa..bd80c4a0 100644 --- a/lib/instances/resume_network_handoff.go +++ b/lib/instances/resume_network_handoff.go @@ -6,6 +6,7 @@ import ( "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" + "go.opentelemetry.io/otel/attribute" ) type resumeNetworkHandoff struct { @@ -41,7 +42,14 @@ func (m *manager) prepareResumeNetworkHandoff(ctx context.Context, stored *Store payload := newGuestResumeNetworkPayload(resumeNetworkCfg) payload.AckPort = waiter.Port() - if err := patchGuestResumeNetworkMailbox(snapshotDir, guestInitiatedResumeNetworkMailboxToken(stored), &payload); err != nil { + _, patchSpanEnd := m.startLifecycleStep(ctx, "guest.resume_network.mailbox_patch", + attribute.String("instance_id", stored.Id), + attribute.String("hypervisor", string(stored.HypervisorType)), + attribute.String("operation", "guest_resume_network_mailbox_patch"), + ) + err = patchGuestResumeNetworkMailbox(snapshotDir, guestInitiatedResumeNetworkMailboxToken(stored), &payload) + patchSpanEnd(err) + if err != nil { waiter.Close() log.WarnContext(ctx, "failed to patch guest resume network mailbox; falling back to host-initiated reconfigure", "instance_id", stored.Id, "error", err) return h, nil From b301e744572df9d60194e308064313217d79ede9 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:45:15 +0000 Subject: [PATCH 12/16] Move mailbox contract package --- lib/instances/guest_resume_network.go | 32 +++++++++---------- lib/instances/restore_egress_test.go | 28 ++++++++-------- lib/mailbox/README.md | 5 +++ lib/{resumenetwork => mailbox}/mailbox.go | 2 +- .../mailbox_test.go | 2 +- lib/system/guest_agent/resume_network.go | 26 +++++++-------- lib/system/guest_agent/resume_network_test.go | 4 +-- 7 files changed, 52 insertions(+), 47 deletions(-) create mode 100644 lib/mailbox/README.md rename lib/{resumenetwork => mailbox}/mailbox.go (98%) rename lib/{resumenetwork => mailbox}/mailbox_test.go (97%) diff --git a/lib/instances/guest_resume_network.go b/lib/instances/guest_resume_network.go index 2311b405..0d880eb6 100644 --- a/lib/instances/guest_resume_network.go +++ b/lib/instances/guest_resume_network.go @@ -14,7 +14,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" - "github.com/kernel/hypeman/lib/resumenetwork" + "github.com/kernel/hypeman/lib/mailbox" "github.com/nrednav/cuid2" "go.opentelemetry.io/otel/attribute" "golang.org/x/sys/unix" @@ -40,15 +40,15 @@ func guestInitiatedResumeNetworkMailbox(stored *StoredMetadata) bool { token := guestInitiatedResumeNetworkMailboxToken(stored) return stored != nil && stored.HypervisorType == hypervisor.TypeFirecracker && - strings.TrimSpace(stored.Env[resumenetwork.MailboxEnv]) == "1" && - resumenetwork.ValidToken(token) + strings.TrimSpace(stored.Env[mailbox.MailboxEnv]) == "1" && + mailbox.ValidToken(token) } func guestInitiatedResumeNetworkMailboxToken(stored *StoredMetadata) string { if stored == nil { return "" } - return strings.TrimSpace(stored.Env[resumenetwork.MailboxTokenEnv]) + return strings.TrimSpace(stored.Env[mailbox.MailboxTokenEnv]) } func ensureGuestInitiatedResumeNetworkMailbox(stored *StoredMetadata) { @@ -61,14 +61,14 @@ func ensureGuestInitiatedResumeNetworkMailbox(stored *StoredMetadata) { if stored.Env == nil { stored.Env = make(map[string]string) } - stored.Env[resumenetwork.MailboxEnv] = "1" - if token := guestInitiatedResumeNetworkMailboxToken(stored); !resumenetwork.ValidToken(token) { - stored.Env[resumenetwork.MailboxTokenEnv] = cuid2.Generate() + stored.Env[mailbox.MailboxEnv] = "1" + if token := guestInitiatedResumeNetworkMailboxToken(stored); !mailbox.ValidToken(token) { + stored.Env[mailbox.MailboxTokenEnv] = cuid2.Generate() } } -func newGuestResumeNetworkPayload(cfg *guestNetworkConfig) resumenetwork.Payload { - return resumenetwork.Payload{ +func newGuestResumeNetworkPayload(cfg *guestNetworkConfig) mailbox.Payload { + return mailbox.Payload{ InterfaceName: "eth0", MAC: cfg.mac, IPv4: cfg.ip, @@ -163,8 +163,8 @@ func (m *manager) waitForGuestResumeNetworkUDPAck(ctx context.Context, waiter *g return nil } -func patchGuestResumeNetworkMailbox(snapshotDir, token string, payload *resumenetwork.Payload) error { - payloadBytes, err := resumenetwork.MarshalPayload(payload) +func patchGuestResumeNetworkMailbox(snapshotDir, token string, payload *mailbox.Payload) error { + payloadBytes, err := mailbox.MarshalPayload(payload) if err != nil { return err } @@ -183,7 +183,7 @@ func patchGuestResumeNetworkMailbox(snapshotDir, token string, payload *resumene return fmt.Errorf("resume network mailbox memory file is empty") } - marker, err := resumenetwork.Marker(token) + marker, err := mailbox.Marker(token) if err != nil { return err } @@ -191,20 +191,20 @@ func patchGuestResumeNetworkMailbox(snapshotDir, token string, payload *resumene if err != nil { return err } - if idx+int64(resumenetwork.MailboxPayloadOffset)+int64(len(payloadBytes)) > info.Size() { + if idx+int64(mailbox.MailboxPayloadOffset)+int64(len(payloadBytes)) > info.Size() { return fmt.Errorf("resume network mailbox marker is too close to end of memory file") } - if _, err := file.WriteAt(payloadBytes, idx+int64(resumenetwork.MailboxPayloadOffset)); err != nil { + if _, err := file.WriteAt(payloadBytes, idx+int64(mailbox.MailboxPayloadOffset)); err != nil { return fmt.Errorf("write resume network mailbox payload: %w", err) } var u32 [4]byte binary.LittleEndian.PutUint32(u32[:], uint32(len(payloadBytes))) - if _, err := file.WriteAt(u32[:], idx+int64(resumenetwork.MailboxLengthOffset)); err != nil { + if _, err := file.WriteAt(u32[:], idx+int64(mailbox.MailboxLengthOffset)); err != nil { return fmt.Errorf("write resume network mailbox payload length: %w", err) } binary.LittleEndian.PutUint32(u32[:], 1) - if _, err := file.WriteAt(u32[:], idx+int64(resumenetwork.MailboxSeqOffset)); err != nil { + if _, err := file.WriteAt(u32[:], idx+int64(mailbox.MailboxSeqOffset)); err != nil { return fmt.Errorf("write resume network mailbox sequence: %w", err) } return nil diff --git a/lib/instances/restore_egress_test.go b/lib/instances/restore_egress_test.go index 3696f745..c3983780 100644 --- a/lib/instances/restore_egress_test.go +++ b/lib/instances/restore_egress_test.go @@ -7,8 +7,8 @@ import ( "testing" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/mailbox" "github.com/kernel/hypeman/lib/network" - "github.com/kernel/hypeman/lib/resumenetwork" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -88,11 +88,11 @@ func TestPatchGuestResumeNetworkMailbox(t *testing.T) { dir := t.TempDir() token := "test-token" mem := make([]byte, 4096) - copy(mem[512:], resumenetwork.MailboxMagic) - copy(mem[512+len(resumenetwork.MailboxMagic):], token) + copy(mem[512:], mailbox.MailboxMagic) + copy(mem[512+len(mailbox.MailboxMagic):], token) require.NoError(t, os.WriteFile(dir+"/"+firecrackerSnapshotMemoryFile, mem, 0644)) - payload := &resumenetwork.Payload{ + payload := &mailbox.Payload{ InterfaceName: "eth0", MAC: "02:00:00:85:17:c8", IPv4: "10.102.146.62", @@ -106,12 +106,12 @@ func TestPatchGuestResumeNetworkMailbox(t *testing.T) { require.NoError(t, err) offset := 512 - require.Equal(t, uint32(1), binary.LittleEndian.Uint32(patched[offset+resumenetwork.MailboxSeqOffset:])) - payloadLen := binary.LittleEndian.Uint32(patched[offset+resumenetwork.MailboxLengthOffset:]) + require.Equal(t, uint32(1), binary.LittleEndian.Uint32(patched[offset+mailbox.MailboxSeqOffset:])) + payloadLen := binary.LittleEndian.Uint32(patched[offset+mailbox.MailboxLengthOffset:]) require.NotZero(t, payloadLen) - var decoded resumenetwork.Payload - err = json.Unmarshal(patched[offset+resumenetwork.MailboxPayloadOffset:offset+resumenetwork.MailboxPayloadOffset+int(payloadLen)], &decoded) + var decoded mailbox.Payload + err = json.Unmarshal(patched[offset+mailbox.MailboxPayloadOffset:offset+mailbox.MailboxPayloadOffset+int(payloadLen)], &decoded) require.NoError(t, err) assert.Equal(t, *payload, decoded) } @@ -125,10 +125,10 @@ func TestEnsureGuestInitiatedResumeNetworkMailbox(t *testing.T) { } ensureGuestInitiatedResumeNetworkMailbox(stored) - require.Equal(t, "1", stored.Env[resumenetwork.MailboxEnv]) - token := stored.Env[resumenetwork.MailboxTokenEnv] + require.Equal(t, "1", stored.Env[mailbox.MailboxEnv]) + token := stored.Env[mailbox.MailboxTokenEnv] require.NotEmpty(t, token) - require.LessOrEqual(t, len(token), resumenetwork.MailboxTokenMaxLen) + require.LessOrEqual(t, len(token), mailbox.MailboxTokenMaxLen) assert.True(t, guestInitiatedResumeNetworkMailbox(stored)) } @@ -139,13 +139,13 @@ func TestEnsureGuestInitiatedResumeNetworkMailboxPreservesToken(t *testing.T) { HypervisorType: hypervisor.TypeFirecracker, NetworkEnabled: true, Env: map[string]string{ - resumenetwork.MailboxTokenEnv: "existing-token", + mailbox.MailboxTokenEnv: "existing-token", }, } ensureGuestInitiatedResumeNetworkMailbox(stored) - assert.Equal(t, "1", stored.Env[resumenetwork.MailboxEnv]) - assert.Equal(t, "existing-token", stored.Env[resumenetwork.MailboxTokenEnv]) + assert.Equal(t, "1", stored.Env[mailbox.MailboxEnv]) + assert.Equal(t, "existing-token", stored.Env[mailbox.MailboxTokenEnv]) } func TestEnsureGuestInitiatedResumeNetworkMailboxRequiresEligibleGuest(t *testing.T) { diff --git a/lib/mailbox/README.md b/lib/mailbox/README.md new file mode 100644 index 00000000..3e834b7c --- /dev/null +++ b/lib/mailbox/README.md @@ -0,0 +1,5 @@ +# mailbox + +The resume network handoff uses a small pre-armed mailbox in guest memory to avoid making the first post-resume operation a host-initiated guest RPC. When a Firecracker guest boots with mailbox env enabled, the guest agent allocates and locks a fixed-size buffer containing a magic marker and token. That buffer is captured in standby snapshots. + +Before restore, the host finds the marker in the snapshot memory file, writes a JSON network payload into the buffer, and flips the sequence field. After resume, VMGenID wakes the guest watcher, which reads the payload, applies the new network identity locally, and sends a UDP applied ack to the host. If the mailbox is missing, cannot be patched, or does not ack in time, restore falls back to the host-initiated network reconfigure path. diff --git a/lib/resumenetwork/mailbox.go b/lib/mailbox/mailbox.go similarity index 98% rename from lib/resumenetwork/mailbox.go rename to lib/mailbox/mailbox.go index 11342130..832f852f 100644 --- a/lib/resumenetwork/mailbox.go +++ b/lib/mailbox/mailbox.go @@ -1,4 +1,4 @@ -package resumenetwork +package mailbox import ( "encoding/json" diff --git a/lib/resumenetwork/mailbox_test.go b/lib/mailbox/mailbox_test.go similarity index 97% rename from lib/resumenetwork/mailbox_test.go rename to lib/mailbox/mailbox_test.go index 4f036181..db0b5c2b 100644 --- a/lib/resumenetwork/mailbox_test.go +++ b/lib/mailbox/mailbox_test.go @@ -1,4 +1,4 @@ -package resumenetwork +package mailbox import ( "encoding/binary" diff --git a/lib/system/guest_agent/resume_network.go b/lib/system/guest_agent/resume_network.go index cb4cd723..f8708976 100644 --- a/lib/system/guest_agent/resume_network.go +++ b/lib/system/guest_agent/resume_network.go @@ -18,7 +18,7 @@ import ( "unsafe" pb "github.com/kernel/hypeman/lib/guest" - "github.com/kernel/hypeman/lib/resumenetwork" + "github.com/kernel/hypeman/lib/mailbox" "golang.org/x/sys/unix" ) @@ -31,7 +31,7 @@ type vmGenIDResumeWaiter struct { } func startResumeNetworkWatcher(s *guestServer) { - if strings.TrimSpace(os.Getenv(resumenetwork.MailboxEnv)) != "1" { + if strings.TrimSpace(os.Getenv(mailbox.MailboxEnv)) != "1" { return } @@ -44,15 +44,15 @@ func startResumeNetworkWatcher(s *guestServer) { } func newResumeNetworkMailbox() []byte { - token := strings.TrimSpace(os.Getenv(resumenetwork.MailboxTokenEnv)) - if !resumenetwork.ValidToken(token) { - log.Printf("[guest-agent] resume network mailbox disabled: invalid %s", resumenetwork.MailboxTokenEnv) + token := strings.TrimSpace(os.Getenv(mailbox.MailboxTokenEnv)) + if !mailbox.ValidToken(token) { + log.Printf("[guest-agent] resume network mailbox disabled: invalid %s", mailbox.MailboxTokenEnv) return nil } - buf := make([]byte, resumenetwork.MailboxSize) - copy(buf, resumenetwork.MailboxMagic) - copy(buf[len(resumenetwork.MailboxMagic):resumenetwork.MailboxSeqOffset], token) + buf := make([]byte, mailbox.MailboxSize) + copy(buf, mailbox.MailboxMagic) + copy(buf[len(mailbox.MailboxMagic):mailbox.MailboxSeqOffset], token) if err := unix.Mlock(buf); err != nil { log.Printf("[guest-agent] resume network mailbox mlock failed: %v", err) } @@ -94,7 +94,7 @@ func waitAndApplyResumeNetworkMailbox(s *guestServer, buf []byte) error { func waitAndApplyResumeNetworkMailboxWithTimeout(s *guestServer, buf []byte, timeout time.Duration) error { deadline := time.Now().Add(timeout) for { - seq := atomicLoadUint32(buf[resumenetwork.MailboxSeqOffset:]) + seq := atomicLoadUint32(buf[mailbox.MailboxSeqOffset:]) if seq == 0 { if time.Now().After(deadline) { return fmt.Errorf("resume network mailbox payload was not patched within %s", timeout) @@ -103,8 +103,8 @@ func waitAndApplyResumeNetworkMailboxWithTimeout(s *guestServer, buf []byte, tim continue } - payloadLen := binary.LittleEndian.Uint32(buf[resumenetwork.MailboxLengthOffset:]) - payload, err := resumenetwork.DecodePayloadFrame(buf, payloadLen) + payloadLen := binary.LittleEndian.Uint32(buf[mailbox.MailboxLengthOffset:]) + payload, err := mailbox.DecodePayloadFrame(buf, payloadLen) if err != nil { return err } @@ -120,12 +120,12 @@ func waitAndApplyResumeNetworkMailboxWithTimeout(s *guestServer, buf []byte, tim return err } sendResumeNetworkAck(payload, "applied") - atomicStoreUint32(buf[resumenetwork.MailboxSeqOffset:], 0) + atomicStoreUint32(buf[mailbox.MailboxSeqOffset:], 0) return nil } } -func sendResumeNetworkAck(payload resumenetwork.Payload, stage string) { +func sendResumeNetworkAck(payload mailbox.Payload, stage string) { if payload.AckPort == 0 || payload.Gateway == "" { return } diff --git a/lib/system/guest_agent/resume_network_test.go b/lib/system/guest_agent/resume_network_test.go index 17c77834..9e30a533 100644 --- a/lib/system/guest_agent/resume_network_test.go +++ b/lib/system/guest_agent/resume_network_test.go @@ -6,12 +6,12 @@ import ( "testing" "time" - "github.com/kernel/hypeman/lib/resumenetwork" + "github.com/kernel/hypeman/lib/mailbox" "github.com/stretchr/testify/require" ) func TestWaitAndApplyResumeNetworkMailboxTimesOutWhenPayloadMissing(t *testing.T) { - buf := make([]byte, resumenetwork.MailboxSize) + buf := make([]byte, mailbox.MailboxSize) err := waitAndApplyResumeNetworkMailboxWithTimeout(&guestServer{}, buf, 5*time.Millisecond) From 362c3577d90f887a399b3f8be6805bcae2ae74fd Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:52:11 +0000 Subject: [PATCH 13/16] Fall back when resume network ack listener fails --- lib/instances/resume_network_handoff.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/instances/resume_network_handoff.go b/lib/instances/resume_network_handoff.go index bd80c4a0..073fc92e 100644 --- a/lib/instances/resume_network_handoff.go +++ b/lib/instances/resume_network_handoff.go @@ -2,7 +2,6 @@ package instances import ( "context" - "fmt" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" @@ -37,7 +36,8 @@ func (m *manager) prepareResumeNetworkHandoff(ctx context.Context, stored *Store waiter, err := startGuestResumeNetworkUDPWaiter() if err != nil { - return nil, fmt.Errorf("start guest resume network UDP ack waiter: %w", err) + log.WarnContext(ctx, "failed to start guest resume network UDP ack waiter; falling back to host-initiated reconfigure", "instance_id", stored.Id, "error", err) + return h, nil } payload := newGuestResumeNetworkPayload(resumeNetworkCfg) From 431328a473bd6bc1996ef8b6aa6d7d505419cb67 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:57:13 +0000 Subject: [PATCH 14/16] Update resume network handoff docs --- lib/forkvm/README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/forkvm/README.md b/lib/forkvm/README.md index 88474c72..d224b220 100644 --- a/lib/forkvm/README.md +++ b/lib/forkvm/README.md @@ -24,13 +24,12 @@ memory. After resume, VMGenID tells the guest-agent that this is a restored VM; the guest-agent reads the mailbox and applies the new MAC, address, route, and neighbor state with netlink. -For API calls that return a running fork, `wait_for_network` defaults to true. -In that mode Hypeman waits for a guest UDP "applied" ack before returning, so -the fast path still avoids making host-initiated guest RPC/vsock contact as the -first post-resume dependency. If `wait_for_network=false`, the API returns after -resume once the mailbox has been patched and the guest finishes the network -handoff asynchronously. If the mailbox path is unavailable, restore falls back -to the older host-initiated guest network reconfigure path. +For API calls that return a running fork, Hypeman waits for a guest UDP +"applied" ack before returning, so the fast path still avoids making +host-initiated guest RPC/vsock contact as the first post-resume dependency. If +the mailbox path is unavailable, cannot start its ack listener, or does not ack +in time, restore falls back to the older host-initiated guest network +reconfigure path. ## Fork data copy behavior From 5967319558b8fdb180ad05a0a86ca89aa633a0bb Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 1 Jun 2026 18:05:17 +0000 Subject: [PATCH 15/16] Require guest readiness after fork starts --- lib/instances/fork.go | 32 ++++++++++++++++++++------------ lib/instances/fork_test.go | 18 ++++++++++++++++++ lib/instances/manager.go | 17 ++++++----------- 3 files changed, 44 insertions(+), 23 deletions(-) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 4e4e8e99..cc27ace9 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -406,14 +406,11 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe lock.Lock() defer lock.Unlock() - returnWithReadiness := func(inst *Instance, err error) (*Instance, error) { + returnWithReadiness := func(inst *Instance, err error, guestReady bool) (*Instance, error) { if err != nil { return nil, err } - if inst != nil && (inst.State == StateRunning || inst.State == StateInitializing) { - if guestInitiatedResumeNetworkMailbox(&inst.StoredMetadata) { - return inst, nil - } + if forkReturnNeedsGuestAgentReady(inst, guestReady) { if err := ensureGuestAgentReadyForForkPhase(ctx, &inst.StoredMetadata, "before returning running fork instance"); err != nil { return nil, fmt.Errorf("wait for forked guest agent readiness: %w", err) } @@ -426,24 +423,24 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe return nil, err } if current.State == target || (target == StateRunning && current.State == StateInitializing) { - return returnWithReadiness(current, nil) + return returnWithReadiness(current, nil, false) } switch current.State { case StateStopped: switch target { case StateRunning: - return returnWithReadiness(m.startInstance(ctx, forkID, StartInstanceRequest{})) + return returnWithReadiness(m.startInstance(ctx, forkID, StartInstanceRequest{}), false) case StateStandby: if _, err := m.startInstance(ctx, forkID, StartInstanceRequest{}); err != nil { return nil, fmt.Errorf("start forked instance for standby transition: %w", err) } - return returnWithReadiness(m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false)) + return returnWithReadiness(m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false), false) } case StateStandby: switch target { case StateRunning: - return returnWithReadiness(m.restoreInstance(ctx, forkID)) + return returnWithReadiness(m.restoreInstance(ctx, forkID), current.NetworkEnabled && !current.SkipGuestAgent) case StateStopped: if err := os.RemoveAll(m.paths.InstanceSnapshotLatest(forkID)); err != nil { return nil, fmt.Errorf("remove fork snapshot: %w", err) @@ -456,20 +453,31 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe if err := m.saveMetadata(meta); err != nil { return nil, fmt.Errorf("save stopped fork metadata: %w", err) } - return returnWithReadiness(m.getInstance(ctx, forkID)) + return returnWithReadiness(m.getInstance(ctx, forkID), false) } case StateRunning: switch target { case StateStandby: - return returnWithReadiness(m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false)) + return returnWithReadiness(m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false), false) case StateStopped: - return returnWithReadiness(m.stopInstance(ctx, forkID)) + return returnWithReadiness(m.stopInstance(ctx, forkID), false) } } return nil, fmt.Errorf("%w: cannot transition forked instance from %s to %s", ErrInvalidState, current.State, target) } +func forkTargetStateAlreadyApplied(inst *Instance, target State) bool { + if inst == nil { + return false + } + return inst.State == target || (target == StateRunning && inst.State == StateInitializing) +} + +func forkReturnNeedsGuestAgentReady(inst *Instance, guestReady bool) bool { + return inst != nil && (inst.State == StateRunning || inst.State == StateInitializing) && !guestReady +} + func (m *manager) cleanupForkInstanceOnError(ctx context.Context, forkID string) error { lock := m.getInstanceLock(forkID) lock.Lock() diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 606b7671..1268d90c 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -319,6 +319,24 @@ func TestApplyForkTargetStateStoppedRefreshesSnapshotForkCID(t *testing.T) { assert.Equal(t, generateVsockCID(forkID), updated.StoredMetadata.VsockCID) } +func TestForkReturnReadinessDoesNotUseMailboxEligibility(t *testing.T) { + t.Parallel() + + stored := StoredMetadata{ + HypervisorType: hypervisor.TypeFirecracker, + NetworkEnabled: true, + } + ensureGuestInitiatedResumeNetworkMailbox(&stored) + require.True(t, guestInitiatedResumeNetworkMailbox(&stored)) + + inst := &Instance{ + StoredMetadata: stored, + State: StateInitializing, + } + assert.True(t, forkReturnNeedsGuestAgentReady(inst, false)) + assert.False(t, forkReturnNeedsGuestAgentReady(inst, true)) +} + func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { t.Parallel() startedAt := time.Now().Add(-2 * time.Minute) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 58a1877a..0ded26ea 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -389,19 +389,14 @@ func (m *manager) ForkInstance(ctx context.Context, id string, req ForkInstanceR return nil, err } - inst, err := m.applyForkTargetState(ctx, forked.Id, targetState) - if err != nil { - if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { - return nil, fmt.Errorf("apply fork target state: %w; additionally failed to cleanup forked instance %s: %v", err, forked.Id, cleanupErr) - } - return nil, fmt.Errorf("apply fork target state: %w", err) - } - if inst.State == StateRunning && !guestInitiatedResumeNetworkMailbox(&inst.StoredMetadata) { - if err := ensureGuestAgentReadyForForkPhase(ctx, &inst.StoredMetadata, "before returning running fork instance"); err != nil { + inst := forked + if !forkTargetStateAlreadyApplied(inst, targetState) { + inst, err = m.applyForkTargetState(ctx, forked.Id, targetState) + if err != nil { if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { - return nil, fmt.Errorf("wait for fork guest agent readiness: %w; additionally failed to cleanup forked instance %s: %v", err, forked.Id, cleanupErr) + return nil, fmt.Errorf("apply fork target state: %w; additionally failed to cleanup forked instance %s: %v", err, forked.Id, cleanupErr) } - return nil, fmt.Errorf("wait for fork guest agent readiness: %w", err) + return nil, fmt.Errorf("apply fork target state: %w", err) } } m.notifyLifecycleEvent(ctx, LifecycleEventFork, inst) From 02bda12e5eaadbba818e01f3b8443dabf2faf0d9 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 1 Jun 2026 18:07:00 +0000 Subject: [PATCH 16/16] Fix fork readiness build --- lib/instances/fork.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index cc27ace9..1d4f72a8 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -430,17 +430,20 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe case StateStopped: switch target { case StateRunning: - return returnWithReadiness(m.startInstance(ctx, forkID, StartInstanceRequest{}), false) + inst, err := m.startInstance(ctx, forkID, StartInstanceRequest{}) + return returnWithReadiness(inst, err, false) case StateStandby: if _, err := m.startInstance(ctx, forkID, StartInstanceRequest{}); err != nil { return nil, fmt.Errorf("start forked instance for standby transition: %w", err) } - return returnWithReadiness(m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false), false) + inst, err := m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false) + return returnWithReadiness(inst, err, false) } case StateStandby: switch target { case StateRunning: - return returnWithReadiness(m.restoreInstance(ctx, forkID), current.NetworkEnabled && !current.SkipGuestAgent) + inst, err := m.restoreInstance(ctx, forkID) + return returnWithReadiness(inst, err, current.NetworkEnabled && !current.SkipGuestAgent) case StateStopped: if err := os.RemoveAll(m.paths.InstanceSnapshotLatest(forkID)); err != nil { return nil, fmt.Errorf("remove fork snapshot: %w", err) @@ -453,14 +456,17 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe if err := m.saveMetadata(meta); err != nil { return nil, fmt.Errorf("save stopped fork metadata: %w", err) } - return returnWithReadiness(m.getInstance(ctx, forkID), false) + inst, err := m.getInstance(ctx, forkID) + return returnWithReadiness(inst, err, false) } case StateRunning: switch target { case StateStandby: - return returnWithReadiness(m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false), false) + inst, err := m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false) + return returnWithReadiness(inst, err, false) case StateStopped: - return returnWithReadiness(m.stopInstance(ctx, forkID), false) + inst, err := m.stopInstance(ctx, forkID) + return returnWithReadiness(inst, err, false) } }