GCS: periodically update cgroup limits

cmd/gcs/main.go

@@ -113,6 +113,58 @@ func readMemoryEvents(startTime time.Time, efdFile *os.File, cgName string, thre
 	}
 }
 
+// Sanity checks the memory limit value
+func calculateContainersMemoryLimit(totalMemoryBytes, rootMemReserveBytes uint64) (int64, error) {
+	if totalMemoryBytes <= rootMemReserveBytes {
+		return 0, errors.Errorf("total memory %d must be greater than root memory reserve %d", totalMemoryBytes, rootMemReserveBytes)
+	}
+
+	limit := totalMemoryBytes - rootMemReserveBytes
+	const maxInt64 = uint64(1<<63 - 1)
+	if limit > maxInt64 {
+		return 0, errors.Errorf("containers cgroup memory limit %d exceeds maximum supported value %d", limit, maxInt64)
+	}
+	return int64(limit), nil
+}
+
+// Returns the total system memory in bytes.
+func currentTotalMemoryBytes() (uint64, error) {
+	sinfo := syscall.Sysinfo_t{}
+	if err := syscall.Sysinfo(&sinfo); err != nil {
+		return 0, errors.Wrap(err, "failed to get sys info")
+	}
+
+	totalMemoryBytes := uint64(sinfo.Totalram)
+	unit := uint64(sinfo.Unit)
+
+	// If the unit is non-zero, multiply the total memory by the unit to get the actual memory in bytes.
+	if unit != 0 {
+		const maxUint64 = ^uint64(0)
+		if totalMemoryBytes > maxUint64/unit {
+			return 0, errors.Errorf("total memory %d with unit %d exceeds maximum supported value", totalMemoryBytes, unit)
+		}
+		totalMemoryBytes *= unit
+	}
+	return totalMemoryBytes, nil
+}
+
+// Start a periodic goroutine to monitor and update the pods cgroup memory limit.
+func monitorCgroupMemoryLimits(h *hcsv2.Host, interval time.Duration) {
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for range ticker.C {
+		podLimit, updated, err := h.UpdateCgroupMemoryLimits()
+		if err != nil {
+			logrus.WithError(err).Error("failed to refresh cgroup memory limits")
+			continue
+		}
+		if !updated {
+			continue
+		}
+		logrus.WithField("memoryLimitBytes", podLimit).Debug("refreshed cgroup memory limits")
+	}
+}
+
 // runWithRestartMonitor starts a command with given args and waits for it to exit. If the
 // command exit code is non-zero the command is restarted with with some back off delay.
 // Any stdout or stderr of the command will be split into lines and written as a log with
@@ -351,11 +403,14 @@ func main() {
 	//
 	// The gcs cgroup is not limited but an event will get logged if memory
 	// usage exceeds 50 MB.
-	sinfo := syscall.Sysinfo_t{}
-	if err := syscall.Sysinfo(&sinfo); err != nil {
-		logrus.WithError(err).Fatal("failed to get sys info")
+	totalMemoryBytes, err := currentTotalMemoryBytes()
+	if err != nil {
+		logrus.WithError(err).Fatal("failed to get total system memory")
+	}
+	podsLimit, err := calculateContainersMemoryLimit(totalMemoryBytes, *rootMemReserveBytes)
+	if err != nil {
+		logrus.WithError(err).Fatal("failed to calculate pods cgroup memory limit")
 	}
-	podsLimit := int64(sinfo.Totalram - *rootMemReserveBytes)
 	podsControl, err := cgroup.NewManager("/pods", &oci.LinuxResources{
 		Memory: &oci.LinuxMemory{
 			Limit: &podsLimit,
@@ -382,6 +437,10 @@ func main() {
 	}
 
 	h := hcsv2.NewHost(rtime, tport, initialEnforcer, logWriter)
+	h.InitializeCgroupMemoryLimitUpdater(podsControl, *rootMemReserveBytes, podsLimit)
+
+	// Start periodic task to update the pods cgroup memory limit.
+	go monitorCgroupMemoryLimits(h, 60*time.Second)
 
 	// During live migration the VM is frozen and only wakes up when the host
 	// shim is ready, so the vsock port should be immediately available. We

cmd/gcs/main_test.go

@@ -0,0 +1,42 @@
+//go:build linux
+// +build linux
+
+package main
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestCalculateContainersMemoryLimit(t *testing.T) {
+	tests := []struct {
+		name        string
+		total       uint64
+		reserve     uint64
+		want        int64
+		wantErrText string
+	}{
+		{name: "subtracts reserve", total: 4096, reserve: 1024, want: 3072},
+		{name: "rejects equal reserve", total: 1024, reserve: 1024, wantErrText: "must be greater"},
+		{name: "rejects below reserve", total: 512, reserve: 1024, wantErrText: "must be greater"},
+		{name: "rejects int64 overflow", total: uint64(1 << 63), wantErrText: "exceeds maximum"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := calculateContainersMemoryLimit(test.total, test.reserve)
+			if test.wantErrText != "" {
+				if err == nil || !strings.Contains(err.Error(), test.wantErrText) {
+					t.Fatalf("expected error containing %q, got %v", test.wantErrText, err)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("calculateContainersMemoryLimit returned error: %v", err)
+			}
+			if got != test.want {
+				t.Fatalf("calculateContainersMemoryLimit returned %d, want %d", got, test.want)
+			}
+		})
+	}
+}

internal/guest/prot/protocol.go

@@ -599,6 +599,8 @@ func UnmarshalContainerModifySettings(b []byte) (*containerModifySettings, error
 			return &request, errors.Wrap(err, "failed to unmarshal settings as SecurityPolicyFragment")
 		}
 		msr.Settings = fragment
+	case guestresource.ResourceTypePodCgroupMemoryLimit:
+		msr.Settings = nil
 	default:
 		return &request, errors.Errorf("invalid ResourceType '%s'", msr.ResourceType)
 	}

internal/guest/runtime/hcsv2/uvm.go

@@ -95,9 +95,13 @@ type pod struct {
 // and processes.
 type Host struct {
 	// containersMutex guards both containers and pods maps.
-	containersMutex sync.Mutex
-	containers      map[string]*Container
-	pods            map[string]*pod
+	containersMutex           sync.Mutex
+	containers                map[string]*Container
+	pods                      map[string]*pod
+	cgroupMemoryMutex         sync.Mutex
+	podsCgroup                cgroupUpdater
+	rootMemReserveBytes       uint64
+	lastAppliedCgroupMemLimit int64
 
 	externalProcessesMutex sync.Mutex
 	externalProcesses      map[int]*externalProcess
@@ -131,6 +135,89 @@ type Host struct {
 	uvmError uvmConsistencyError
 }
 
+type cgroupUpdater interface {
+	Update(resources *specs.LinuxResources) error
+}
+
+func calculateCgroupMemoryLimit(totalMemoryBytes, rootMemReserveBytes uint64) (int64, error) {
+	if totalMemoryBytes <= rootMemReserveBytes {
+		return 0, errors.Errorf("total memory %d must be greater than root memory reserve %d", totalMemoryBytes, rootMemReserveBytes)
+	}
+
+	limit := totalMemoryBytes - rootMemReserveBytes
+	const maxInt64 = uint64(1<<63 - 1)
+	if limit > maxInt64 {
+		return 0, errors.Errorf("pods cgroup memory limit %d exceeds maximum supported value %d", limit, maxInt64)
+	}
+	return int64(limit), nil
+}
+
+func currentTotalMemoryBytes() (uint64, error) {
+	sinfo := syscall.Sysinfo_t{}
+	if err := syscall.Sysinfo(&sinfo); err != nil {
+		return 0, errors.Wrap(err, "failed to get sys info")
+	}
+
+	totalMemoryBytes := uint64(sinfo.Totalram)
+	unit := uint64(sinfo.Unit)
+	if unit != 0 {
+		const maxUint64 = ^uint64(0)
+		if totalMemoryBytes > maxUint64/unit {
+			return 0, errors.Errorf("total memory %d with unit %d exceeds maximum supported value", totalMemoryBytes, unit)
+		}
+		totalMemoryBytes *= unit
+	}
+	return totalMemoryBytes, nil
+}
+
+func setCgroupMemoryLimit(podsCgroup cgroupUpdater, totalMemoryBytes, rootMemReserveBytes uint64, lastAppliedLimit int64) (int64, bool, error) {
+	limit, err := calculateCgroupMemoryLimit(totalMemoryBytes, rootMemReserveBytes)
+	if err != nil {
+		return lastAppliedLimit, false, err
+	}
+	if limit == lastAppliedLimit {
+		return limit, false, nil
+	}
+
+	resources := &specs.LinuxResources{Memory: &specs.LinuxMemory{Limit: &limit}}
+	if err := podsCgroup.Update(resources); err != nil {
+		return lastAppliedLimit, false, errors.Wrap(err, "failed to update pods cgroup memory limit")
+	}
+	return limit, true, nil
+}
+
+// InitializeCgroupMemoryLimitUpdater configures the cgroup updated when the UVM memory changes.
+func (h *Host) InitializeCgroupMemoryLimitUpdater(podsCgroup cgroup.Manager, rootMemReserveBytes uint64, initialLimit int64) {
+	h.cgroupMemoryMutex.Lock()
+	defer h.cgroupMemoryMutex.Unlock()
+
+	h.podsCgroup = podsCgroup
+	h.rootMemReserveBytes = rootMemReserveBytes
+	h.lastAppliedCgroupMemLimit = initialLimit
+}
+
+// UpdateCgroupMemoryLimits refreshes the pods cgroup limit from the UVM's current memory size.
+func (h *Host) UpdateCgroupMemoryLimits() (int64, bool, error) {
+	h.cgroupMemoryMutex.Lock()
+	defer h.cgroupMemoryMutex.Unlock()
+
+	if h.podsCgroup == nil {
+		return h.lastAppliedCgroupMemLimit, false, errors.New("cgroup memory limit updater is not initialized")
+	}
+	totalMemoryBytes, err := currentTotalMemoryBytes()
+	if err != nil {
+		return h.lastAppliedCgroupMemLimit, false, err
+	}
+	limit, updated, err := setCgroupMemoryLimit(h.podsCgroup, totalMemoryBytes, h.rootMemReserveBytes, h.lastAppliedCgroupMemLimit)
+	if err != nil {
+		return h.lastAppliedCgroupMemLimit, false, err
+	}
+	if updated {
+		h.lastAppliedCgroupMemLimit = limit
+	}
+	return limit, updated, nil
+}
+
 type uvmConsistencyError struct {
 	mu sync.Mutex
 	// The error describing why the UVM has entered an inconsistent state.  If
@@ -925,6 +1012,14 @@ func (h *Host) modifyHostSettings(ctx context.Context, containerID string, req *
 			return errors.New("the request settings are not of type SecurityPolicyFragment")
 		}
 		return h.securityOptions.InjectFragment(ctx, r)
+	case guestresource.ResourceTypePodCgroupMemoryLimit:
+		switch req.RequestType {
+		case guestrequest.RequestTypeUpdate:
+			_, _, err := h.UpdateCgroupMemoryLimits()
+			return err
+		default:
+			return newInvalidRequestTypeError(req.RequestType)
+		}
 	default:
 		return errors.Errorf("the ResourceType %q is not supported for UVM", req.ResourceType)
 	}

internal/guest/runtime/hcsv2/uvm_state_test.go

@@ -4,9 +4,99 @@
 package hcsv2
 
 import (
+	"context"
+	"io"
+	"strings"
 	"testing"
+
+	"github.com/Microsoft/hcsshim/internal/protocol/guestrequest"
+	"github.com/Microsoft/hcsshim/internal/protocol/guestresource"
+	"github.com/Microsoft/hcsshim/pkg/securitypolicy"
+	oci "github.com/opencontainers/runtime-spec/specs-go"
 )
 
+func TestCalculateCgroupMemoryLimit(t *testing.T) {
+	tests := []struct {
+		name        string
+		total       uint64
+		reserve     uint64
+		want        int64
+		wantErrText string
+	}{
+		{name: "subtracts reserve", total: 4096, reserve: 1024, want: 3072},
+		{name: "rejects equal reserve", total: 1024, reserve: 1024, wantErrText: "must be greater"},
+		{name: "rejects below reserve", total: 512, reserve: 1024, wantErrText: "must be greater"},
+		{name: "rejects int64 overflow", total: uint64(1 << 63), wantErrText: "exceeds maximum"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := calculateCgroupMemoryLimit(test.total, test.reserve)
+			if test.wantErrText != "" {
+				if err == nil || !strings.Contains(err.Error(), test.wantErrText) {
+					t.Fatalf("expected error containing %q, got %v", test.wantErrText, err)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("calculateCgroupMemoryLimit returned error: %v", err)
+			}
+			if got != test.want {
+				t.Fatalf("calculateCgroupMemoryLimit returned %d, want %d", got, test.want)
+			}
+		})
+	}
+}
+
+func TestSetCgroupMemoryLimit(t *testing.T) {
+	podsControl := &testCgroupUpdater{}
+	limit, updated, err := setCgroupMemoryLimit(podsControl, 4096, 1024, 2048)
+	if err != nil {
+		t.Fatalf("setCgroupMemoryLimit returned error: %v", err)
+	}
+	if !updated || limit != 3072 || podsControl.limit != 3072 {
+		t.Fatalf("setCgroupMemoryLimit returned limit=%d, updated=%t, applied=%d; want 3072, true, 3072", limit, updated, podsControl.limit)
+	}
+
+	limit, updated, err = setCgroupMemoryLimit(podsControl, 4096, 1024, 3072)
+	if err != nil {
+		t.Fatalf("setCgroupMemoryLimit returned error: %v", err)
+	}
+	if updated || limit != 3072 || podsControl.updates != 1 {
+		t.Fatalf("unchanged set returned limit=%d, updated=%t, updates=%d; want 3072, false, 1", limit, updated, podsControl.updates)
+	}
+}
+
+func TestModifyHostSettingsRejectsInvalidPodCgroupMemoryLimitRequestType(t *testing.T) {
+	for _, requestType := range []guestrequest.RequestType{
+		guestrequest.RequestTypeAdd,
+		guestrequest.RequestTypeRemove,
+		guestrequest.RequestTypePreAdd,
+	} {
+		t.Run(string(requestType), func(t *testing.T) {
+			host := NewHost(nil, nil, &securitypolicy.OpenDoorSecurityPolicyEnforcer{}, io.Discard)
+			err := host.modifyHostSettings(context.Background(), UVMContainerID, &guestrequest.ModificationRequest{
+				ResourceType: guestresource.ResourceTypePodCgroupMemoryLimit,
+				RequestType:  requestType,
+			})
+			if err == nil || !strings.Contains(err.Error(), "RequestType") {
+				t.Fatalf("modifyHostSettings returned %v, want invalid RequestType error", err)
+			}
+		})
+	}
+}
+
+type testCgroupUpdater struct {
+	limit   int64
+	updates int
+}
+
+func (cgroup *testCgroupUpdater) Update(resources *oci.LinuxResources) error {
+	cgroup.updates++
+	cgroup.limit = *resources.Memory.Limit
+	return nil
+}
+
 func Test_Add_Remove_RWDevice(t *testing.T) {
 	hm := newHostMounts()
 	mountPath := "/run/gcs/c/abcd"

internal/protocol/guestresource/resources.go

@@ -55,6 +55,8 @@ const (
 	ResourceTypeSecurityPolicy guestrequest.ResourceType = "SecurityPolicy"
 	// ResourceTypePolicyFragment is the modify resource type for injecting policy fragments.
 	ResourceTypePolicyFragment guestrequest.ResourceType = "SecurityPolicyFragment"
+	// ResourceTypePodCgroupMemoryLimit is used to force a cgroup memory limit update in the guest.
+	ResourceTypePodCgroupMemoryLimit guestrequest.ResourceType = "PodCgroupMemoryLimit"
 )
 
 // This class is used by a modify request to add or remove a combined layers

internal/uvm/update_uvm.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 
 	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
+	"github.com/Microsoft/hcsshim/internal/protocol/guestrequest"
+	"github.com/Microsoft/hcsshim/internal/protocol/guestresource"
 	"github.com/Microsoft/hcsshim/pkg/annotations"
 	"github.com/Microsoft/hcsshim/pkg/ctrdtaskapi"
 	"github.com/opencontainers/runtime-spec/specs-go"
@@ -54,6 +56,14 @@ func (uvm *UtilityVM) Update(ctx context.Context, data interface{}, annots map[s
 		if err := uvm.UpdateMemory(ctx, *memoryLimitInBytes); err != nil {
 			return err
 		}
+		if uvm.operatingSystem == "linux" {
+			if err := uvm.GuestRequest(ctx, guestrequest.ModificationRequest{
+				ResourceType: guestresource.ResourceTypePodCgroupMemoryLimit,
+				RequestType:  guestrequest.RequestTypeUpdate,
+			}); err != nil {
+				return err
+			}
+		}
 	}
 	if processorLimits != nil {
 		if err := uvm.UpdateCPULimits(ctx, processorLimits); err != nil {