From 329dd20a30bc8ae26c6207f525e4ec04aeeddc70 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Thu, 23 Jul 2026 09:00:24 -0700 Subject: [PATCH] fix(noderesource): raise cosmos-exporter memory ceiling 384Mi -> 512Mi (stopgap) cosmos-exporter sidecars are OOMKilling roughly every ~2h fleet-wide (observed on atlantic-2/archive-0-0-0: 7 restarts over 14h). Root cause is believed to be a memory leak in the goroutine-per-event fan-out in HandleBankTransferEvent (sei-cosmos-exporter/events.go) -- NOT the eventstream Reset()/Run() reconnect path, which was independently verified to have no leak (single in-flight goroutine at all times, clean handoff). This raises the OOM ceiling as a stopgap to reduce restart frequency while the upstream leak gets fixed -- it does not fix the leak. Filed as a follow-up: [upstream sei-cosmos-exporter issue to be filed]. Reaches every live SeiNode on its next controller reconcile (the StatefulSet template is re-rendered unconditionally each pass, independent of the image-drift gate that controls automatic pod replacement) -- a pod that isn't automatically rolled (no image drift) picks up the new limit on its next natural recreation (its own OOM-restart cycle, or a manual `kubectl delete pod`). Cross-reviewed via /xreview: idiom pass confirmed this matches the file's existing comment/resource-quantity conventions; a controller-boundary review confirmed the rollout-trigger mechanism (template re-renders every reconcile; only automatic pod-replace is image-drift-gated) and corrected an earlier assumption that this specific stopgap couldn't reach Git-detached SeiNode CRs -- it does, since the controller reconciles off the live CR, not off Git. Co-Authored-By: Claude Sonnet 5 --- internal/noderesource/noderesource.go | 6 +++++- internal/noderesource/noderesource_test.go | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/noderesource/noderesource.go b/internal/noderesource/noderesource.go index 467e1b7..56cbb3d 100644 --- a/internal/noderesource/noderesource.go +++ b/internal/noderesource/noderesource.go @@ -852,7 +852,11 @@ func defaultCosmosExporterResources() corev1.ResourceRequirements { corev1.ResourceMemory: resource.MustParse("64Mi"), }, Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("384Mi"), + // Temporary: raises the OOM ceiling to reduce restart frequency + // while a suspected leak in sei-cosmos-exporter (goroutine-per- + // event fan-out in HandleBankTransferEvent, not the eventstream + // reconnect path) gets fixed upstream. Does not fix the leak. + corev1.ResourceMemory: resource.MustParse("512Mi"), }, } } diff --git a/internal/noderesource/noderesource_test.go b/internal/noderesource/noderesource_test.go index 8c8977d..a403468 100644 --- a/internal/noderesource/noderesource_test.go +++ b/internal/noderesource/noderesource_test.go @@ -1771,14 +1771,14 @@ func TestCosmosExporter_DefaultResources(t *testing.T) { sts := mustGenerateStatefulSet(t, node, platformtest.Config()) ce := findContainer(sts.Spec.Template.Spec.Containers, containerNameCosmosExporter) - // 50m/64Mi requests, 256Mi memory limit, no CPU limit (see + // 50m/64Mi requests, 512Mi memory limit, no CPU limit (see // defaultCosmosExporterResources — scrape pulls would throttle). cpuReq := ce.Resources.Requests[corev1.ResourceCPU] memReq := ce.Resources.Requests[corev1.ResourceMemory] memLim := ce.Resources.Limits[corev1.ResourceMemory] g.Expect(cpuReq.String()).To(Equal("50m")) g.Expect(memReq.String()).To(Equal("64Mi")) - g.Expect(memLim.String()).To(Equal("384Mi")) + g.Expect(memLim.String()).To(Equal("512Mi")) _, hasCPULimit := ce.Resources.Limits[corev1.ResourceCPU] g.Expect(hasCPULimit).To(BeFalse()) }