chore(manifests): disable --auto-gomemlimit for Prometheus on SNO until we can ensure it won't result in excessive CPU usage

Author: machine424

pkg/manifests/manifests.go

@@ -1456,7 +1456,7 @@ func (f *Factory) PrometheusK8s(grpcTLS *v1.Secret, telemetrySecret *v1.Secret)
 	}
 
 	setupAlerting(p, platformAlertmanagerService, f.namespace)
-	f.setupGoGC(p)
+	f.adjustGoGCRelatedConfig(p)
 
 	for i, container := range p.Spec.Containers {
 		switch container.Name {
@@ -1495,7 +1495,7 @@ func (f *Factory) PrometheusK8s(grpcTLS *v1.Secret, telemetrySecret *v1.Secret)
 	return p, nil
 }
 
-func (f *Factory) setupGoGC(p *monv1.Prometheus) {
+func (f *Factory) adjustGoGCRelatedConfig(p *monv1.Prometheus) {
 	if f.infrastructure.HighlyAvailableInfrastructure() {
 		return
 	}
@@ -1513,6 +1513,11 @@ func (f *Factory) setupGoGC(p *monv1.Prometheus) {
 		// be savvy on CPU hence set GOGC=100 (Go runtime default) in this
 		// case.
 		p.Spec.Containers[i].Env = append(p.Spec.Containers[i].Env, v1.EnvVar{Name: "GOGC", Value: "100"})
+
+		// Until we're certain setting GOMEMLIMIT to 0.9 (default ratio) of detected maximum
+		// container or system memory won't result in excessive CPU usage, we're disabling the
+		// auto setting for SNO.
+		p.Spec.Containers[i].Args = append(p.Spec.Containers[i].Args, "--no-auto-gomemlimit")
 	}
 }
 
@@ -1805,7 +1810,7 @@ func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret) (*monv1.Prometheus,
 		}
 	}
 
-	f.setupGoGC(p)
+	f.adjustGoGCRelatedConfig(p)
 
 	if f.config.UserWorkloadConfiguration.Alertmanager.Enabled {
 		setupAlerting(p, userWorkloadAlertmanagerService, f.namespaceUserWorkload)

pkg/manifests/manifests_test.go

@@ -4288,44 +4288,47 @@ func TestThanosRulerRetentionConfig(t *testing.T) {
 	}
 }
 
-func TestPrometheusGoGC(t *testing.T) {
+func TestPrometheusGoGCRelatedConfig(t *testing.T) {
 	for _, tc := range []struct {
 		ir    InfrastructureReader
 		promf func(*Factory) (*monv1.Prometheus, error)
 
-		exp string
+		expectedGOGC           string
+		autoGOMEMLIMITDisabled bool
 	}{
 		{
 			ir: &fakeInfrastructureReader{highlyAvailableInfrastructure: false},
 			promf: func(f *Factory) (*monv1.Prometheus, error) {
 				return f.PrometheusK8s(&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, nil)
 			},
 
-			exp: "100",
+			expectedGOGC:           "100",
+			autoGOMEMLIMITDisabled: true,
 		},
 		{
 			ir: &fakeInfrastructureReader{highlyAvailableInfrastructure: true},
 			promf: func(f *Factory) (*monv1.Prometheus, error) {
 				return f.PrometheusK8s(&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, nil)
 			},
 
-			exp: "",
+			expectedGOGC: "",
 		},
 		{
 			ir: &fakeInfrastructureReader{highlyAvailableInfrastructure: false},
 			promf: func(f *Factory) (*monv1.Prometheus, error) {
 				return f.PrometheusUserWorkload(&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}})
 			},
 
-			exp: "100",
+			expectedGOGC:           "100",
+			autoGOMEMLIMITDisabled: true,
 		},
 		{
 			ir: &fakeInfrastructureReader{highlyAvailableInfrastructure: true},
 			promf: func(f *Factory) (*monv1.Prometheus, error) {
 				return f.PrometheusUserWorkload(&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}})
 			},
 
-			exp: "",
+			expectedGOGC: "",
 		},
 	} {
 		t.Run("", func(t *testing.T) {
@@ -4341,14 +4344,16 @@ func TestPrometheusGoGC(t *testing.T) {
 				}
 			}
 			require.NotNil(t, c)
-			if tc.exp == "" {
+			if tc.expectedGOGC == "" {
 				for _, env := range c.Env {
 					require.NotEqual(t, env.Name, "GOGC")
 				}
 				return
 			}
 
-			require.Contains(t, c.Env, v1.EnvVar{Name: "GOGC", Value: tc.exp})
+			require.Contains(t, c.Env, v1.EnvVar{Name: "GOGC", Value: tc.expectedGOGC})
+
+			require.Equal(t, tc.autoGOMEMLIMITDisabled, argumentPresent(*c, "--no-auto-gomemlimit"))
 		})
 	}
 }