From b2dc20db6b47a75e9393fd6692bc67c8ac69210a Mon Sep 17 00:00:00 2001 From: bdchatham Date: Mon, 27 Jul 2026 11:39:51 -0700 Subject: [PATCH 1/6] feat(seed): add seed node mode to SeiNode A seed is a peer-discovery server: seid runs only the P2P transport and the PEX reactor, binding no RPC, gRPC, REST or EVM listener. It joins no consensus and stores no chain state. SeedSpec carries one field, a required NodeKey. Unlike the validator's optional key this is mandatory because a seed's NodeID is published -- operators dial NodeID@host:port and the handshake verifies the pinned value, so a regenerated key silently breaks every client carrying the old one. It is declared on SeedSpec rather than hoisted to SeiNodeSpec because ValidatorSpec's key-pairing CEL rules could no longer reach a top-level field; SeiNodeSpec.NodeKeySecret() resolves the split so downstream callers stay mode-blind. seedPlanner mirrors the archive planner. A nil snapshot source selects the genesis progression and, via needsStateSyncWitnesses, omits configure-state-sync; validate-node-key pre-flights the identity Secret before the StatefulSet is applied. The pod needed shaping. Readiness gated on seid's /lag_status and the cosmos-exporter container blocked on gRPC, so on a seed the pod would never reach Ready and the exporter would CrashLoopBackOff. servesSeidRPC gates both; ValidateSeedProbes fails the render closed if a seid probe ever targets RPC or gRPC again. GOMEMLIMIT is set from the resolved memory limit -- with request==limit the Go GC, blind to the cgroup, turns an allocation burst into an OOMKill rather than a throttle. scheduling.nodepoolSeed has no fallback: the default pool is RPC-class, so a 2Gi seed landing there costs an order of magnitude more than a seed is worth and karpenter.sh/do-not-disrupt then pins that node against consolidation. GenerateStatefulSet rejects a seed without it, which keeps the requirement off Config.Validate so clusters running no seeds still boot on an app-config file predating the key. storage.sizeSeed stays genuinely optional -- 20Gi is a correct value, not a placeholder. Co-Authored-By: Claude Opus 5 (1M context) --- api/v1alpha1/seed_types.go | 48 ++++++ api/v1alpha1/seinode_types.go | 37 ++++- api/v1alpha1/zz_generated.deepcopy.go | 21 +++ config/crd/sei.io_seinodes.yaml | 65 +++++++- .../node/envtest/seed_validation_test.go | 99 +++++++++++ internal/controller/node/workflow.go | 2 + internal/noderesource/noderesource.go | 140 ++++++++++++++-- internal/noderesource/noderesource_test.go | 148 ++++++++++++++++- internal/noderesource/probe_guard.go | 31 ++++ internal/noderesource/probe_guard_test.go | 56 +++++++ internal/planner/planner.go | 15 +- internal/planner/seed.go | 78 +++++++++ internal/planner/seed_test.go | 155 ++++++++++++++++++ internal/platform/load.go | 3 + internal/platform/platform.go | 50 +++++- internal/platform/platform_test.go | 36 +++- internal/platform/platformtest/config.go | 1 + internal/task/bootstrap_resources.go | 7 + manifests/sei.io_seinodes.yaml | 65 +++++++- 19 files changed, 1016 insertions(+), 41 deletions(-) create mode 100644 api/v1alpha1/seed_types.go create mode 100644 internal/controller/node/envtest/seed_validation_test.go create mode 100644 internal/planner/seed.go create mode 100644 internal/planner/seed_test.go diff --git a/api/v1alpha1/seed_types.go b/api/v1alpha1/seed_types.go new file mode 100644 index 00000000..293f5763 --- /dev/null +++ b/api/v1alpha1/seed_types.go @@ -0,0 +1,48 @@ +package v1alpha1 + +// SeedSpec configures a CometBFT seed node: a peer-discovery server running the +// P2P transport and PEX reactor only. It hands a dialing node a spread of peer +// addresses, then drops the connection. +// +// A seed joins no consensus, syncs no blocks, and serves no queries — seid binds +// no RPC, gRPC, REST or EVM listener in this mode. Hence no snapshot source +// (nothing to restore), no snapshotGeneration (no history to snapshot), and no +// signing key or operator keyring (it signs nothing). Its data volume holds the +// peer store plus the block and state DBs seid opens and never writes. +// +// Peers come from SeiNodeSpec.Peers as in every mode; a seed with none is a +// valid bootstrap root that others dial without dialing out itself. +// +// A seed reachable from outside the cluster needs three things this spec does +// not provide: SeiNodeSpec.ExternalAddress set to the published `host:port` +// (otherwise seid advertises its in-cluster listen address over PEX and every +// node that learns the seed by gossip gets an unroutable peer), a load balancer +// and DNS record for TCP 26656 that outlive pod and node churn, and an ingress +// allowance for that port. The readiness probe cannot detect any of them +// missing — see readinessProbeForNode. +// +// A seed serves no RPC, so it has no /status and no `catching_up`: sync +// freshness is not a property a seed has. Monitor it on P2P metrics +// (tendermint_p2p_*) at the metrics port instead. +type SeedSpec struct { + // NodeKey supplies the P2P identity (node_key.json) this seed presents. + // + // Required, where the validator's NodeKey is optional, because a seed's + // NodeID is published: operators dial `NodeID@host:port` and the + // secret-connection handshake verifies the pinned value, so a changed NodeID + // silently breaks every client carrying the old one. A Secret-sourced key + // survives pod recreation and PVC loss; one left to `seid init` regenerates + // onto the data volume. Carrying the identity to another cluster means + // replicating the Secret there — the controller reads it, it does not move + // it. Treat the NodeID as a one-way door. + // + // The Secret name is immutable, so rotating the identity means deleting and + // recreating the SeiNode — and announcing the new NodeID. There is no fast + // path for a leaked key: the old NodeID stays dialable until every client + // ships a new default, so a thief keeps impersonating a bootstrap anchor. + // + // Give each seed a distinct Secret. Nothing rejects two seeds sharing one, + // and they would present the same NodeID — collapsing the redundant anchors + // into a single entry in every dialer's peer store. + NodeKey NodeKeySource `json:"nodeKey"` +} diff --git a/api/v1alpha1/seinode_types.go b/api/v1alpha1/seinode_types.go index e8fe069f..44584c1d 100644 --- a/api/v1alpha1/seinode_types.go +++ b/api/v1alpha1/seinode_types.go @@ -7,9 +7,9 @@ import ( ) // SeiNodeSpec defines the desired state of a standalone Sei node. -// Exactly one mode sub-spec (fullNode, archive, replayer, validator) must be set; -// the populated field determines the node's operating mode. -// +kubebuilder:validation:XValidation:rule="(has(self.fullNode) ? 1 : 0) + (has(self.archive) ? 1 : 0) + (has(self.replayer) ? 1 : 0) + (has(self.validator) ? 1 : 0) == 1",message="exactly one of fullNode, archive, replayer, or validator must be set" +// Exactly one mode sub-spec (fullNode, archive, replayer, validator, seed) must +// be set; the populated field determines the node's operating mode. +// +kubebuilder:validation:XValidation:rule="(has(self.fullNode) ? 1 : 0) + (has(self.archive) ? 1 : 0) + (has(self.replayer) ? 1 : 0) + (has(self.validator) ? 1 : 0) + (has(self.seed) ? 1 : 0) == 1",message="exactly one of fullNode, archive, replayer, validator, or seed must be set" // +kubebuilder:validation:XValidation:rule="!has(self.replayer) || (has(self.peers) && size(self.peers) > 0)",message="peers is required when replayer mode is set" type SeiNodeSpec struct { // ChainID of the chain this node belongs to. @@ -73,6 +73,10 @@ type SeiNodeSpec struct { // +optional Validator *ValidatorSpec `json:"validator,omitempty"` + // Seed configures a peer-discovery seed node (P2P + PEX only). + // +optional + Seed *SeedSpec `json:"seed,omitempty"` + // ExternalAddress is the routable P2P host:port written into seid's // `p2p.external_address`. SeiNetwork-managed nodes get this stamped by the // SeiNetwork reconciler when TCP networking is enabled. Standalone SeiNodes @@ -138,6 +142,27 @@ func (s *SeiNodeSpec) SnapshotSource() *SnapshotSource { } } +// NodeKeySecret returns the Secret supplying this node's P2P identity +// (node_key.json), or nil when the node has none and `seid init` generates one +// onto the data volume. +// +// Two modes carry a node key on differing terms — a validator's is optional and +// paired with a signing key, a seed's is required and standalone — so each +// sub-spec declares its own field rather than hoisting one to SeiNodeSpec, out of +// reach of the validator's key-pairing CEL rules. This accessor resolves that +// split in one place, so callers needing only "which Secret holds the node key" +// stay mode-blind. Mirrors SnapshotSource. +func (s *SeiNodeSpec) NodeKeySecret() *SecretNodeKeySource { + switch { + case s.Validator != nil && s.Validator.NodeKey != nil: + return s.Validator.NodeKey.Secret + case s.Seed != nil: + return s.Seed.NodeKey.Secret + default: + return nil + } +} + // --------------------------------------------------------------------------- // Status // --------------------------------------------------------------------------- @@ -294,9 +319,9 @@ const ( // SeiNodes with spec.validator.signingKey. ConditionSigningKeyReady = "SigningKeyReady" - // ConditionNodeKeyReady indicates whether a referenced validator - // node-key Secret passes all validation requirements. Only set on - // SeiNodes with spec.validator.nodeKey. + // ConditionNodeKeyReady indicates whether a referenced node-key Secret + // passes all validation requirements. Only set on SeiNodes that source a + // node key from a Secret — spec.validator.nodeKey or spec.seed.nodeKey. ConditionNodeKeyReady = "NodeKeyReady" // ConditionOperatorKeyringReady indicates whether a referenced diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 1a926cd9..93f90a4c 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -846,6 +846,22 @@ func (in *SecretSigningKeySource) DeepCopy() *SecretSigningKeySource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SeedSpec) DeepCopyInto(out *SeedSpec) { + *out = *in + in.NodeKey.DeepCopyInto(&out.NodeKey) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SeedSpec. +func (in *SeedSpec) DeepCopy() *SeedSpec { + if in == nil { + return nil + } + out := new(SeedSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SeiNetwork) DeepCopyInto(out *SeiNetwork) { *out = *in @@ -1110,6 +1126,11 @@ func (in *SeiNodeSpec) DeepCopyInto(out *SeiNodeSpec) { *out = new(ValidatorSpec) (*in).DeepCopyInto(*out) } + if in.Seed != nil { + in, out := &in.Seed, &out.Seed + *out = new(SeedSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SeiNodeSpec. diff --git a/config/crd/sei.io_seinodes.yaml b/config/crd/sei.io_seinodes.yaml index 5646ce81..62ba6dc7 100644 --- a/config/crd/sei.io_seinodes.yaml +++ b/config/crd/sei.io_seinodes.yaml @@ -52,8 +52,8 @@ spec: spec: description: |- SeiNodeSpec defines the desired state of a standalone Sei node. - Exactly one mode sub-spec (fullNode, archive, replayer, validator) must be set; - the populated field determines the node's operating mode. + Exactly one mode sub-spec (fullNode, archive, replayer, validator, seed) must + be set; the populated field determines the node's operating mode. properties: archive: description: Archive configures an archive node with full history @@ -446,6 +446,60 @@ spec: required: - snapshot type: object + seed: + description: Seed configures a peer-discovery seed node (P2P + PEX + only). + properties: + nodeKey: + description: |- + NodeKey supplies the P2P identity (node_key.json) this seed presents. + + Required, where the validator's NodeKey is optional, because a seed's + NodeID is published: operators dial `NodeID@host:port` and the + secret-connection handshake verifies the pinned value, so a changed NodeID + silently breaks every client carrying the old one. A Secret-sourced key + survives pod recreation and PVC loss; one left to `seid init` regenerates + onto the data volume. Carrying the identity to another cluster means + replicating the Secret there — the controller reads it, it does not move + it. Treat the NodeID as a one-way door. + + The Secret name is immutable, so rotating the identity means deleting and + recreating the SeiNode — and announcing the new NodeID. There is no fast + path for a leaked key: the old NodeID stays dialable until every client + ships a new default, so a thief keeps impersonating a bootstrap anchor. + + Give each seed a distinct Secret. Nothing rejects two seeds sharing one, + and they would present the same NodeID — collapsing the redundant anchors + into a single entry in every dialer's peer store. + properties: + secret: + description: |- + Secret loads the node key from a Kubernetes Secret in the + SeiNode's namespace. + properties: + secretName: + description: |- + SecretName is the name of a Secret in the SeiNode's namespace. + The controller never creates, mutates, or deletes this Secret. + Immutable: re-pointing the node ID on a running validator costs + peer reputation and forces a peer-graph reset; force delete-and-recreate. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + x-kubernetes-validations: + - message: secretName is immutable + rule: self == oldSelf + required: + - secretName + type: object + type: object + x-kubernetes-validations: + - message: exactly one node key source must be set + rule: '(has(self.secret) ? 1 : 0) == 1' + required: + - nodeKey + type: object sidecar: description: Sidecar configures the sei-sidecar container. properties: @@ -827,10 +881,11 @@ spec: - image type: object x-kubernetes-validations: - - message: exactly one of fullNode, archive, replayer, or validator must - be set + - message: exactly one of fullNode, archive, replayer, validator, or seed + must be set rule: '(has(self.fullNode) ? 1 : 0) + (has(self.archive) ? 1 : 0) + - (has(self.replayer) ? 1 : 0) + (has(self.validator) ? 1 : 0) == 1' + (has(self.replayer) ? 1 : 0) + (has(self.validator) ? 1 : 0) + (has(self.seed) + ? 1 : 0) == 1' - message: peers is required when replayer mode is set rule: '!has(self.replayer) || (has(self.peers) && size(self.peers) > 0)' diff --git a/internal/controller/node/envtest/seed_validation_test.go b/internal/controller/node/envtest/seed_validation_test.go new file mode 100644 index 00000000..005d945c --- /dev/null +++ b/internal/controller/node/envtest/seed_validation_test.go @@ -0,0 +1,99 @@ +//go:build envtest + +package envtest_test + +import ( + "testing" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1" +) + +// seedNode returns a seed whose identity comes from the named Secret. An empty +// secretName yields a seed with no pinned identity, which admission must reject. +func seedNode(ns, name, secretName string) *seiv1alpha1.SeiNode { + spec := seiv1alpha1.SeiNodeSpec{ + ChainID: "envtest-1", + Image: "sei:latest", + Seed: &seiv1alpha1.SeedSpec{}, + } + if secretName != "" { + spec.Seed.NodeKey = seiv1alpha1.NodeKeySource{ + Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: secretName}, + } + } + return &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Spec: spec, + } +} + +func TestSeed_WithNodeKey_Accepted(t *testing.T) { + g := NewWithT(t) + ns := makeNamespace(t) + g.Expect(testCli.Create(testCtx, seedNode(ns, "seed-ok", "seed-0-node-key"))).To(Succeed()) +} + +// A seed's NodeID is published, so an unpinned identity is rejected at admission +// rather than left to regenerate onto the data volume at boot. +func TestSeed_WithoutNodeKey_Rejected(t *testing.T) { + g := NewWithT(t) + ns := makeNamespace(t) + + err := testCli.Create(testCtx, seedNode(ns, "seed-no-key", "")) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("nodeKey")) +} + +// The mode sub-specs stay mutually exclusive with seed added to the set. +func TestSeed_WithSecondMode_Rejected(t *testing.T) { + g := NewWithT(t) + ns := makeNamespace(t) + + node := seedNode(ns, "seed-plus-full", "seed-0-node-key") + node.Spec.FullNode = &seiv1alpha1.FullNodeSpec{} + + err := testCli.Create(testCtx, node) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("exactly one of")) +} + +// Widening the exactly-one rule must not reject the four pre-existing modes. +func TestSeed_OtherModesStillAccepted(t *testing.T) { + ns := makeNamespace(t) + + modes := map[string]func(*seiv1alpha1.SeiNodeSpec){ + "fullnode": func(s *seiv1alpha1.SeiNodeSpec) { s.FullNode = &seiv1alpha1.FullNodeSpec{} }, + "archive": func(s *seiv1alpha1.SeiNodeSpec) { s.Archive = &seiv1alpha1.ArchiveSpec{} }, + "validator": func(s *seiv1alpha1.SeiNodeSpec) { s.Validator = &seiv1alpha1.ValidatorSpec{} }, + } + for name, set := range modes { + t.Run(name, func(t *testing.T) { + g := NewWithT(t) + spec := seiv1alpha1.SeiNodeSpec{ChainID: "envtest-1", Image: "sei:latest"} + set(&spec) + node := &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: "mode-" + name, Namespace: ns}, + Spec: spec, + } + g.Expect(testCli.Create(testCtx, node)).To(Succeed()) + }) + } +} + +// The Secret backs a published NodeID, so re-pointing it in place is refused; +// rotating the identity means delete-and-recreate. +func TestSeed_NodeKeySecretName_Immutable(t *testing.T) { + g := NewWithT(t) + ns := makeNamespace(t) + + node := seedNode(ns, "seed-immutable", "seed-0-node-key") + g.Expect(testCli.Create(testCtx, node)).To(Succeed()) + + node.Spec.Seed.NodeKey.Secret.SecretName = "seed-0-node-key-v2" + err := testCli.Update(testCtx, node) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("immutable")) +} diff --git a/internal/controller/node/workflow.go b/internal/controller/node/workflow.go index 6844d4f4..684357a0 100644 --- a/internal/controller/node/workflow.go +++ b/internal/controller/node/workflow.go @@ -622,6 +622,8 @@ func ineligibleWorkflowRole(node *seiv1alpha1.SeiNode) string { return "archive" case node.Spec.Replayer != nil: return "replayer" + case node.Spec.Seed != nil: + return "seed" // stores no chain state to re-bootstrap default: return "non-full" // unset/unknown mode — refused by the allowlist } diff --git a/internal/noderesource/noderesource.go b/internal/noderesource/noderesource.go index 467e1b76..84e4c593 100644 --- a/internal/noderesource/noderesource.go +++ b/internal/noderesource/noderesource.go @@ -62,6 +62,7 @@ const ( roleArchive = "archive" roleReplayer = "replayer" roleFullNode = "node" + roleSeed = "seed" // Per-mode seid-container footprint: memory and CPU request are both set to // the full sei-infra / verified-prod instance envelope for the role (memory @@ -70,9 +71,17 @@ const ( memValidator = "128Gi" // full r5.4xlarge envelope; provisioned on the next size up (r5.8xlarge) memRPCClass = "256Gi" // full r7i.8xlarge envelope (fullNode + replayer); provisioned on r7i.12xlarge memArchive = "512Gi" // full r7i.16xlarge envelope (verified live prod); provisioned on r7i.24xlarge + memSeed = "2Gi" // peer buffers + idle DBs only; see defaultNodeResourceProfiles cpuValidator = "16" // r5.4xlarge vCPU; request-only, no CPU limit cpuRPCClass = "32" // r7i.8xlarge vCPU (fullNode + replayer); request-only, no CPU limit cpuArchive = "64" // r7i.16xlarge vCPU; request-only, no CPU limit + cpuSeed = "1" // PEX gossip is light and spiky; request-only, no CPU limit + + // goMemLimitPercentOfLimit is the seed's GOMEMLIMIT as a percentage of its + // memory limit. The 13% margin absorbs the non-heap RSS the Go soft limit + // does not govern: thread stacks, mmap'd memIAVL pages, and cgo/kernel + // allocations. See goMemLimitEnv. + goMemLimitPercentOfLimit = 87 dataDir = platform.DataDir @@ -273,6 +282,8 @@ func deriveRole(node *seiv1alpha1.SeiNode) string { return roleArchive case node.Spec.Replayer != nil: return roleReplayer + case node.Spec.Seed != nil: + return roleSeed default: return roleFullNode } @@ -286,6 +297,8 @@ func NodeMode(node *seiv1alpha1.SeiNode) string { return string(seiconfig.ModeArchive) case node.Spec.Validator != nil: return string(seiconfig.ModeValidator) + case node.Spec.Seed != nil: + return string(seiconfig.ModeSeed) case node.Spec.Replayer != nil: return string(seiconfig.ModeFull) default: @@ -293,6 +306,18 @@ func NodeMode(node *seiv1alpha1.SeiNode) string { } } +// servesSeidRPC reports whether seid binds its RPC and gRPC listeners. True for +// every mode but seed, which runs the P2P transport and PEX reactor only. +// +// It gates the two pod-spec decisions that poll those ports — the readiness probe +// target, and whether cosmos-exporter is attached. On a seed both wait forever: +// readiness never passes, and cosmos-exporter's wait loop never reaches its own +// listener, so its liveness probe kills it. Named for the property rather than +// the mode so the call sites read as intent. +func servesSeidRPC(node *seiv1alpha1.SeiNode) bool { + return node.Spec.Seed == nil +} + // NeedsLongStartup returns true when the node's bootstrap strategy involves // replaying blocks, requiring extended startup probe thresholds. func NeedsLongStartup(node *seiv1alpha1.SeiNode) bool { @@ -318,6 +343,11 @@ func DefaultStorageForMode(mode string, p PlatformConfig) (storageClass string, return p.StorageClassArchive, p.StorageSizeArchive case string(seiconfig.ModeFull), string(seiconfig.ModeValidator): return p.StorageClassPerf, p.StorageSizeDefault + case string(seiconfig.ModeSeed): + // A seed stores a peer store and two DBs it never writes, so it needs + // orders of magnitude less than StorageSizeDefault (sized for a chain's + // full state) and no performance class. + return p.StorageClassDefault, p.SeedStorageSize() default: return p.StorageClassDefault, p.StorageSizeDefault } @@ -377,6 +407,14 @@ var defaultNodeResourceProfiles = map[string]nodeResourceProfile{ // Full r7i.16xlarge envelope — the live-prod snapshotter shape (the sei-infra // TF's m7i.8xlarge is stale). roleArchive: {cpuRequest: cpuArchive, memory: memArchive}, + // Peer discovery only: no block execution, no state commitment, no query + // surface. The working set is P2P connection buffers plus the block/state DBs + // seid opens and never writes. The exception to the "request == the reference + // instance's full envelope" rule above: a seed targets the ~2 vCPU / 2-4 GiB + // t-class shape RFC 006 budgets, and on a node that small the fixed + // kube/system reservation is a large fraction of capacity — so the request + // sits well below advertised RAM rather than equal to it. + roleSeed: {cpuRequest: cpuSeed, memory: memSeed}, } // overrideForRole returns the app-config resource override for a node's mode. @@ -388,6 +426,8 @@ func overrideForRole(role string, p PlatformConfig) platform.ResourceOverride { return p.NodeResourcesArchive case roleReplayer: return p.NodeResourcesReplayer + case roleSeed: + return p.NodeResourcesSeed default: return p.NodeResourcesNode } @@ -438,6 +478,13 @@ func GenerateStatefulSet(node *seiv1alpha1.SeiNode, p PlatformConfig) (*appsv1.S if p.KubeRBACProxyImage == "" { return nil, fmt.Errorf("images.kubeRBACProxy is not configured in the app-config file") } + // Seeds need their own small pool. Required here rather than in + // Config.Validate so a cluster that runs no seeds keeps booting on an + // app-config file that predates the key — and so the alternative, silently + // scheduling a 2Gi seed onto the RPC-class default pool, is impossible. + if !servesSeidRPC(node) && p.NodepoolForMode(NodeMode(node)) == "" { + return nil, fmt.Errorf("scheduling.nodepoolSeed is not configured in the app-config file; a seed requires a dedicated small-instance nodepool") + } one := int32(1) labels := ResourceLabels(node) podSpec, err := buildNodePodSpec(node, p) @@ -622,6 +669,8 @@ func ServicePorts() []corev1.ServicePort { ports := make([]corev1.ServicePort, len(np)) for i, p := range np { ports[i] = corev1.ServicePort{Name: p.Name, Port: p.Port, TargetPort: intstr.FromInt32(p.Port), Protocol: corev1.ProtocolTCP} + // Literal, not seiconfig.PortNameGRPC: that constant ships in an + // unreleased sei-config. Adopt it at the next module bump. if p.Name == "grpc" { h2c := "kubernetes.io/h2c" ports[i].AppProtocol = &h2c @@ -723,10 +772,6 @@ func buildNodePodSpec(node *seiv1alpha1.SeiNode, p PlatformConfig) (corev1.PodSp buildSidecarContainer(node, p), buildRBACProxyContainer(node, p), } - ceContainer, err := buildCosmosExporterContainer(p) - if err != nil { - return corev1.PodSpec{}, err - } seidContainer := buildSidecarMainContainer(node, p) // Refuse to render a gated seid container that carries an RPC-based // liveness/startup probe: a held seid answers neither port, so such a probe @@ -734,9 +779,21 @@ func buildNodePodSpec(node *seiv1alpha1.SeiNode, p PlatformConfig) (corev1.PodSp if err := ValidateGatedSeidProbes(seidContainer); err != nil { return corev1.PodSpec{}, err } - spec.Containers = []corev1.Container{ - seidContainer, - ceContainer, + // A seed's readiness must not depend on seid's RPC/gRPC, which it never binds. + if err := ValidateSeedProbes(node, seidContainer); err != nil { + return corev1.PodSpec{}, err + } + spec.Containers = []corev1.Container{seidContainer} + + // cosmos-exporter scrapes staking and validator state over seid's gRPC. A + // seed serves neither, so attaching it there would leave it blocked in its + // wait loop until its own liveness probe killed it. + if servesSeidRPC(node) { + ceContainer, err := buildCosmosExporterContainer(p) + if err != nil { + return corev1.PodSpec{}, err + } + spec.Containers = append(spec.Containers, ceContainer) } return spec, nil @@ -817,6 +874,9 @@ func buildSidecarMainContainer(node *seiv1alpha1.SeiNode, p PlatformConfig) core container := buildNodeMainContainer(node) container.Command, container.Args = sidecarWaitCommand(node) container.Resources = ResourcesForNode(node, p) + if env, ok := goMemLimitEnv(node, container.Resources); ok { + container.Env = append(container.Env, env) + } // Sidecar binds loopback; gate startup on the proxy's /v0/healthz // (a bypass path that forwards through to the sidecar). container.StartupProbe = &corev1.Probe{ @@ -828,7 +888,62 @@ func buildSidecarMainContainer(node *seiv1alpha1.SeiNode, p PlatformConfig) core PeriodSeconds: 5, FailureThreshold: 86400, } - container.ReadinessProbe = &corev1.Probe{ + container.ReadinessProbe = readinessProbeForNode(node) + return container +} + +// goMemLimitEnv returns the GOMEMLIMIT env var for a seed's seid container. +// +// The Go runtime targets a heap roughly twice the live set and cannot see the +// cgroup limit, so with memory request==limit an allocation burst is an OOMKill +// rather than a throttle. A soft limit below the hard cap makes the GC spend CPU +// to stay resident instead. Scoped to seed: it is the only mode whose footprint +// (2Gi) is small enough for ordinary GC slack to cross the cap — the r-class +// modes carry absolute headroom in the tens of GiB. +// +// Derived from the resolved limit, so an app-config resources.seed override +// keeps it consistent. Reports false when no memory limit is set. +func goMemLimitEnv(node *seiv1alpha1.SeiNode, res corev1.ResourceRequirements) (corev1.EnvVar, bool) { + if servesSeidRPC(node) { + return corev1.EnvVar{}, false + } + lim, ok := res.Limits[corev1.ResourceMemory] + if !ok { + return corev1.EnvVar{}, false + } + soft := lim.Value() / 100 * goMemLimitPercentOfLimit + return corev1.EnvVar{Name: "GOMEMLIMIT", Value: fmt.Sprintf("%dB", soft)}, true +} + +// readinessProbeForNode returns the seid readiness probe for the node's mode. +// +// Chain-following modes gate on seid's /lag_status, which reports sync distance +// — the meaningful "ready to serve" signal. A seed serves no RPC, so its only +// in-band signal is that the P2P transport is bound: it distinguishes a crashed +// seid or a failed bind from a live one, and nothing more. +// +// Two limits an operator must know. It proves nothing about EXTERNAL +// reachability — a seed with no DNS record, no load balancer, or a closed +// security group reads Ready while no stranger can dial it. And each probe +// completes a TCP connect without the secret handshake, so it increments +// tendermint_p2p_new_connections{direction="in"} and books a handshake failure +// roughly 6/min; alerts on those series need that baseline subtracted, and +// inbound connections alone do not prove a seed is publicly reachable. +func readinessProbeForNode(node *seiv1alpha1.SeiNode) *corev1.Probe { + if !servesSeidRPC(node) { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt32(seiconfig.PortP2P), + }, + }, + InitialDelaySeconds: 30, + PeriodSeconds: 10, + FailureThreshold: 3, + TimeoutSeconds: 5, + } + } + return &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/lag_status", @@ -840,7 +955,6 @@ func buildSidecarMainContainer(node *seiv1alpha1.SeiNode, p PlatformConfig) core FailureThreshold: 3, TimeoutSeconds: 5, } - return container } // defaultCosmosExporterResources: no CPU limit — cosmos-exporter calls @@ -1189,11 +1303,11 @@ func nodeKeyMounts(node *seiv1alpha1.SeiNode) []corev1.VolumeMount { }} } +// nodeKeySecretSource returns the Secret holding this node's P2P identity, from +// whichever mode sub-spec carries one (validator or seed). Mode-blind by +// delegation, so the node-key volume and mount below serve both. func nodeKeySecretSource(node *seiv1alpha1.SeiNode) *seiv1alpha1.SecretNodeKeySource { - if node.Spec.Validator == nil || node.Spec.Validator.NodeKey == nil { - return nil - } - return node.Spec.Validator.NodeKey.Secret + return node.Spec.NodeKeySecret() } // operatorKeyringVolumes projects the operator-keyring Secret as a directory diff --git a/internal/noderesource/noderesource_test.go b/internal/noderesource/noderesource_test.go index 8c8977d6..670b1205 100644 --- a/internal/noderesource/noderesource_test.go +++ b/internal/noderesource/noderesource_test.go @@ -841,6 +841,12 @@ func nodeForRole(role string) *seiv1alpha1.SeiNode { spec.Archive = &seiv1alpha1.ArchiveSpec{} case roleReplayer: spec.Replayer = &seiv1alpha1.ReplayerSpec{} + case roleSeed: + spec.Seed = &seiv1alpha1.SeedSpec{ + NodeKey: seiv1alpha1.NodeKeySource{ + Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: "seed-node-key"}, + }, + } default: spec.FullNode = &seiv1alpha1.FullNodeSpec{} } @@ -861,6 +867,7 @@ func TestResourcesForNode_DefaultsPerMode(t *testing.T) { {"", cpuRPCClass, memRPCClass}, // fullNode / rpc {roleReplayer, cpuRPCClass, memRPCClass}, {roleArchive, cpuArchive, memArchive}, + {roleSeed, cpuSeed, memSeed}, } for _, tc := range cases { t.Run("mode="+tc.role, func(t *testing.T) { @@ -1645,7 +1652,7 @@ func TestGenerateStatefulSet_SeidMount_PassesGuard(t *testing.T) { // --- Cosmos exporter --- -func TestCosmosExporter_AlwaysPresent(t *testing.T) { +func TestCosmosExporter_PresentOnModesServingGRPC(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("ce-0", "default") @@ -1656,6 +1663,32 @@ func TestCosmosExporter_AlwaysPresent(t *testing.T) { g.Expect(sts.Spec.Template.Spec.Containers).To(HaveLen(2)) } +// cosmos-exporter's wait loop blocks until seid's gRPC accepts, and a seed never +// binds it — so attaching the container would leave it waiting until its own +// liveness probe killed it, in CrashLoopBackOff for the pod's life. +func TestCosmosExporter_AbsentOnSeed(t *testing.T) { + g := NewWithT(t) + + sts := mustGenerateStatefulSet(t, nodeForRole(roleSeed), platformtest.Config()) + + containers := sts.Spec.Template.Spec.Containers + g.Expect(findContainer(containers, containerNameCosmosExporter)).To(BeNil()) + g.Expect(containers).To(HaveLen(1)) + g.Expect(containers[0].Name).To(Equal(containerNameSeid)) +} + +// A seed needs the peer store and two DBs it never writes — not the full-state +// volume StorageSizeDefault sizes for. +func TestDefaultStorageForMode_Seed(t *testing.T) { + g := NewWithT(t) + cfg := platformtest.Config() + + sc, size := DefaultStorageForMode(string(seiconfig.ModeSeed), cfg) + g.Expect(sc).To(Equal(cfg.StorageClassDefault)) + g.Expect(size).To(Equal(cfg.SeedStorageSize())) + g.Expect(size).NotTo(Equal(cfg.StorageSizeDefault)) +} + func TestCosmosExporter_DefaultImage(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("ce-0", "default") @@ -1927,3 +1960,116 @@ func TestCosmosExporter_NonRootSecurityContext(t *testing.T) { g.Expect(*ce.SecurityContext.RunAsNonRoot).To(BeTrue()) g.Expect(*ce.SecurityContext.RunAsUser).To(Equal(int64(65532))) } + +// --- Seed identity --- + +// A seed's NodeID is published, so it must come from the Secret rather than be +// regenerated onto the data volume. The mount uses subPath, so kubelet never +// hot-swaps the identity under a running seid. +func TestSeed_NodeKeySecretMountedOnPodTemplate(t *testing.T) { + g := NewWithT(t) + + sts := mustGenerateStatefulSet(t, nodeForRole(roleSeed), platformtest.Config()) + + vol := findVolume(sts.Spec.Template.Spec.Volumes, nodeKeyVolumeName) + g.Expect(vol).NotTo(BeNil(), "seed node-key volume must be present") + g.Expect(vol.Secret).NotTo(BeNil()) + g.Expect(vol.Secret.SecretName).To(Equal("seed-node-key")) + g.Expect(vol.Secret.Items[0].Key).To(Equal(nodeKeyDataKey)) + + seid := findContainer(sts.Spec.Template.Spec.Containers, containerNameSeid) + g.Expect(seid).NotTo(BeNil()) + mount := findVolumeMount(seid.VolumeMounts, nodeKeyVolumeName) + g.Expect(mount).NotTo(BeNil(), "seed seid container must mount its node key") + g.Expect(mount.SubPath).To(Equal(nodeKeyDataKey)) + g.Expect(mount.ReadOnly).To(BeTrue()) +} + +// deriveRole feeds the sei.io/role pod label the platform PodMonitor lifts into +// sei_role, and NodeMode drives storage and nodepool selection. Both default to +// full-node values, so a missing seed arm is silently wrong rather than a +// compile error. +func TestSeed_RoleAndModeAreNotFullNodeDefaults(t *testing.T) { + g := NewWithT(t) + node := nodeForRole(roleSeed) + + g.Expect(deriveRole(node)).To(Equal(roleSeed)) + g.Expect(NodeMode(node)).To(Equal(string(seiconfig.ModeSeed))) + g.Expect(ResourceLabels(node)).To(HaveKeyWithValue(roleLabel, roleSeed)) + // A seed produces no snapshots, so the publish selector must not match it. + g.Expect(ResourceLabels(node)).NotTo(HaveKey(snapshotPublishLabel)) + // Nothing to replay: startup thresholds stay at the short default. + g.Expect(NeedsLongStartup(node)).To(BeFalse()) +} + +// Memory request==limit makes an allocation burst an OOMKill, and the Go GC +// cannot see the cgroup limit. A soft limit below the cap trades CPU to stay +// resident. Seed-only: the r-class modes carry tens of GiB of absolute headroom. +func TestSeed_GoMemLimitBelowMemoryLimit(t *testing.T) { + g := NewWithT(t) + cfg := platformtest.Config() + + sts := mustGenerateStatefulSet(t, nodeForRole(roleSeed), cfg) + seid := findContainer(sts.Spec.Template.Spec.Containers, containerNameSeid) + g.Expect(seid).NotTo(BeNil()) + + env := findEnvVar(seid.Env, "GOMEMLIMIT") + g.Expect(env).NotTo(BeNil(), "a seed must carry GOMEMLIMIT") + + limit := seid.Resources.Limits[corev1.ResourceMemory] + soft, err := resource.ParseQuantity(strings.TrimSuffix(env.Value, "B")) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(soft.Value()).To(BeNumerically("<", limit.Value())) + g.Expect(soft.Value()).To(BeNumerically(">", limit.Value()/2), "soft limit should not waste the footprint") +} + +// Chain-following modes keep the runtime's own heap sizing; their headroom is +// large enough that GC slack cannot cross the cap. +func TestNonSeed_HasNoGoMemLimit(t *testing.T) { + g := NewWithT(t) + + sts := mustGenerateStatefulSet(t, newSnapshotNode("rpc-0", "default"), platformtest.Config()) + seid := findContainer(sts.Spec.Template.Spec.Containers, containerNameSeid) + + g.Expect(findEnvVar(seid.Env, "GOMEMLIMIT")).To(BeNil()) +} + +func findEnvVar(env []corev1.EnvVar, name string) *corev1.EnvVar { + for i := range env { + if env[i].Name == name { + return &env[i] + } + } + return nil +} + +// The alternative to failing closed is silently scheduling a 2Gi seed onto the +// RPC-class default pool, which costs an order of magnitude more than a seed is +// worth. Clusters running no seeds are unaffected — the check is per-render, not +// at startup. +func TestSeed_RenderFailsWithoutSeedNodepool(t *testing.T) { + g := NewWithT(t) + cfg := platformtest.Config() + cfg.NodepoolSeed = "" + + _, err := GenerateStatefulSet(nodeForRole(roleSeed), cfg) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("scheduling.nodepoolSeed")) + + // Every other mode still renders on the same config. + _, err = GenerateStatefulSet(newSnapshotNode("rpc-0", "default"), cfg) + g.Expect(err).NotTo(HaveOccurred()) +} + +// A configured seed pool lands on the pod, not the default pool. +func TestSeed_SchedulesOnSeedNodepool(t *testing.T) { + g := NewWithT(t) + cfg := platformtest.Config() + + sts := mustGenerateStatefulSet(t, nodeForRole(roleSeed), cfg) + spec := sts.Spec.Template.Spec + + terms := spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + g.Expect(terms[0].MatchExpressions[0].Values).To(ConsistOf(cfg.NodepoolSeed)) + g.Expect(spec.Tolerations[0].Value).To(Equal(cfg.NodepoolSeed)) +} diff --git a/internal/noderesource/probe_guard.go b/internal/noderesource/probe_guard.go index 5a93bc98..7ed8d4bf 100644 --- a/internal/noderesource/probe_guard.go +++ b/internal/noderesource/probe_guard.go @@ -5,6 +5,8 @@ import ( seiconfig "github.com/sei-protocol/sei-config" corev1 "k8s.io/api/core/v1" + + seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1" ) // ValidateGatedSeidProbes enforces the probe contract the workflow hold leans @@ -31,6 +33,35 @@ func ValidateGatedSeidProbes(c corev1.Container) error { return nil } +// ValidateSeedProbes is the seed counterpart of the gated-seid contract: a seed's +// seid container must carry no probe against seid's RPC or gRPC port, since seed +// mode binds neither for the pod's whole life. A permanently-failing readiness +// probe holds the pod out of Service endpoints and out of any NLB target group +// fronting it — leaving the seed unreachable, which is its only job. +// +// Readiness is checked here where ValidateGatedSeidProbes skips it: during a hold +// a failing readiness is correct and temporary, on a seed it is permanent. No-op +// for every other mode. +func ValidateSeedProbes(node *seiv1alpha1.SeiNode, c corev1.Container) error { + if servesSeidRPC(node) { + return nil + } + probes := []struct { + name string + probe *corev1.Probe + }{ + {"readiness", c.ReadinessProbe}, + {"liveness", c.LivenessProbe}, + {"startup", c.StartupProbe}, + } + for _, p := range probes { + if probeTargetsSeidRPC(p.probe) { + return fmt.Errorf("seed seid container %q %s probe must not target seid's RPC/gRPC port; seed mode binds neither", c.Name, p.name) + } + } + return nil +} + // probeTargetsSeidRPC reports whether a probe polls seid's RPC or gRPC port // (via HTTPGet or TCPSocket). A nil probe or one targeting any other port // (e.g. the RBAC-proxy healthz port) returns false. diff --git a/internal/noderesource/probe_guard_test.go b/internal/noderesource/probe_guard_test.go index f8189bbd..641a5521 100644 --- a/internal/noderesource/probe_guard_test.go +++ b/internal/noderesource/probe_guard_test.go @@ -54,3 +54,59 @@ func TestValidateGatedSeidProbes_RejectsRPCProbes(t *testing.T) { }} g.Expect(ValidateGatedSeidProbes(withGRPCStartup)).To(HaveOccurred()) } + +// A seed serves only P2P, so its readiness gates on the transport being bound — +// the failure most worth catching, since a loopback-bound seed boots clean and +// accepts nothing. +func TestSeedContainer_ReadinessProbesP2PNotRPC(t *testing.T) { + g := NewWithT(t) + c := buildSidecarMainContainer(nodeForRole(roleSeed), platformtest.Config()) + + g.Expect(c.ReadinessProbe).NotTo(BeNil()) + g.Expect(c.ReadinessProbe.HTTPGet).To(BeNil(), "a seed serves no RPC to probe over HTTP") + g.Expect(c.ReadinessProbe.TCPSocket).NotTo(BeNil()) + g.Expect(c.ReadinessProbe.TCPSocket.Port.IntVal).To(Equal(seiconfig.PortP2P)) + g.Expect(ValidateSeedProbes(nodeForRole(roleSeed), c)).To(Succeed()) +} + +// Chain-following modes keep the /lag_status readiness gate, which reports sync +// distance rather than mere liveness. +func TestNonSeedContainer_ReadinessProbesLagStatus(t *testing.T) { + g := NewWithT(t) + node := newGenesisNode("mynet-0", "default") + c := buildSidecarMainContainer(node, platformtest.Config()) + + g.Expect(c.ReadinessProbe.HTTPGet).NotTo(BeNil()) + g.Expect(c.ReadinessProbe.HTTPGet.Path).To(Equal("/lag_status")) + g.Expect(c.ReadinessProbe.HTTPGet.Port.IntVal).To(Equal(seiconfig.PortRPC)) + // The seed guard is a no-op for modes that do bind RPC. + g.Expect(ValidateSeedProbes(node, c)).To(Succeed()) +} + +// A permanently-failing readiness probe would hold a seed out of Service +// endpoints and any NLB target group fronting it, so the render fails closed. +func TestValidateSeedProbes_RejectsRPCProbes(t *testing.T) { + seed := nodeForRole(roleSeed) + base := buildSidecarMainContainer(seed, platformtest.Config()) + + rpcProbe := func(port int32) *corev1.Probe { + return &corev1.Probe{ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt32(port)}, + }} + } + + cases := map[string]func(*corev1.Container){ + "readiness on RPC": func(c *corev1.Container) { c.ReadinessProbe = rpcProbe(seiconfig.PortRPC) }, + "liveness on RPC": func(c *corev1.Container) { c.LivenessProbe = rpcProbe(seiconfig.PortRPC) }, + "startup on gRPC": func(c *corev1.Container) { c.StartupProbe = rpcProbe(seiconfig.PortGRPC) }, + "readiness on gRPC": func(c *corev1.Container) { c.ReadinessProbe = rpcProbe(seiconfig.PortGRPC) }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + g := NewWithT(t) + c := base + mutate(&c) + g.Expect(ValidateSeedProbes(seed, c)).NotTo(Succeed()) + }) + } +} diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 376c3efc..fa78c3b0 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -310,6 +310,8 @@ func (r *NodeResolver) plannerForMode(node *seiv1alpha1.SeiNode) (NodePlanner, e return &replayerPlanner{platform: r.Platform}, nil case node.Spec.Validator != nil: return &validatorPlanner{platform: r.Platform}, nil + case node.Spec.Seed != nil: + return &seedPlanner{platform: r.Platform}, nil default: return nil, fmt.Errorf("no mode sub-spec set on SeiNode %s/%s", node.Namespace, node.Name) } @@ -379,12 +381,13 @@ func validateSigningKeyParams(node *seiv1alpha1.SeiNode) any { } } +// needsValidateNodeKey reports whether the node sources its P2P identity from a +// Secret and so needs the pre-flight validation task. Mode-blind: it asks the +// spec which Secret holds the node key, not which mode is set, so validator and +// seed are covered by the same gate. func needsValidateNodeKey(node *seiv1alpha1.SeiNode) bool { - if node.Spec.Validator == nil || node.Spec.Validator.NodeKey == nil { - return false - } - return node.Spec.Validator.NodeKey.Secret != nil && - node.Spec.Validator.NodeKey.Secret.SecretName != "" + s := node.Spec.NodeKeySecret() + return s != nil && s.SecretName != "" } func validateNodeKeyParams(node *seiv1alpha1.SeiNode) any { @@ -392,7 +395,7 @@ func validateNodeKeyParams(node *seiv1alpha1.SeiNode) any { return nil } return &task.ValidateNodeKeyParams{ - SecretName: node.Spec.Validator.NodeKey.Secret.SecretName, + SecretName: node.Spec.NodeKeySecret().SecretName, Namespace: node.Namespace, } } diff --git a/internal/planner/seed.go b/internal/planner/seed.go new file mode 100644 index 00000000..8f67ce6e --- /dev/null +++ b/internal/planner/seed.go @@ -0,0 +1,78 @@ +package planner + +import ( + "fmt" + + seiconfig "github.com/sei-protocol/sei-config" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1" + "github.com/sei-protocol/sei-k8s-controller/internal/platform" + "github.com/sei-protocol/sei-k8s-controller/internal/task" +) + +type seedPlanner struct { + platform platform.Config +} + +func (p *seedPlanner) Mode() string { return string(seiconfig.ModeSeed) } + +func (p *seedPlanner) Validate(node *seiv1alpha1.SeiNode) error { + if node.Spec.Seed == nil { + return fmt.Errorf("seed sub-spec is nil") + } + // CEL already requires the field and a Secret variant, but an in-memory + // spec that never went through admission can still reach here. A seed with + // no pinned identity would silently generate a fresh NodeID on the data + // volume, so fail loudly rather than serve an unstable address. + if s := node.Spec.NodeKeySecret(); s == nil || s.SecretName == "" { + return fmt.Errorf("seed: nodeKey.secret.secretName is required — a seed's NodeID is published and must not be regenerated") + } + return nil +} + +// BuildPlan drives a seed through the plain genesis progression. A seed never +// bootstraps from a snapshot — it stores no chain state to restore — so the nil +// SnapshotSource selects the genesis task sequence and, via +// needsStateSyncWitnesses, omits ConfigureStateSync entirely. +func (p *seedPlanner) BuildPlan(node *seiv1alpha1.SeiNode) (*seiv1alpha1.TaskPlan, error) { + if node.Status.Phase == seiv1alpha1.PhaseRunning { + return p.buildRunningPlan(node) + } + // No per-mode controller overrides: every seed-specific config key comes + // from sei-config's applySeedOverrides, resolved sidecar-side off + // ConfigIntent.Mode. commonOverrides still supplies the shared keys. + intent := &seiconfig.ConfigIntent{ + Mode: seiconfig.ModeSeed, + Overrides: mergeOverrides(commonOverrides(node), node.Spec.Overrides), + } + return buildBasePlan(node, nil, intent) +} + +// buildRunningPlan returns the update plan for a Running seed. Same shape as +// the other modes, with the node-key gate ahead of any StatefulSet mutation so +// a missing or malformed identity Secret fails controller-side rather than as a +// kubelet volume-mount error on the recreated pod. +func (p *seedPlanner) buildRunningPlan(node *seiv1alpha1.SeiNode) (*seiv1alpha1.TaskPlan, error) { + if imageDrifted(node) || sidecarImageDrifted(node, p.platform) { + setNodeUpdateCondition(node, metav1.ConditionTrue, "UpdateStarted", imageDriftMessage(node, p.platform)) + prog := make([]string, 0, 8) + if needsValidateNodeKey(node) { + prog = append(prog, task.TaskTypeValidateNodeKey) + } + prog = append(prog, + task.TaskTypeApplyStatefulSet, + task.TaskTypeApplyService, + TaskConfigPatch, + TaskConfigValidate, + task.TaskTypeReplacePod, + task.TaskTypeObserveImage, + TaskMarkReady, + ) + return assembleUpdatePlan(node, prog, p2pConfigPatch(node)) + } + if sidecarNeedsReapproval(node) { + return buildMarkReadyPlan(node) + } + return nil, nil +} diff --git a/internal/planner/seed_test.go b/internal/planner/seed_test.go new file mode 100644 index 00000000..8e64444e --- /dev/null +++ b/internal/planner/seed_test.go @@ -0,0 +1,155 @@ +package planner + +import ( + "encoding/json" + "slices" + "testing" + + seiconfig "github.com/sei-protocol/sei-config" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1" + "github.com/sei-protocol/sei-k8s-controller/internal/task" +) + +// seedImage and its predecessor exercise the image-drift update path. +const ( + seedImage = "seid:v6.4.1" + seedPrevImage = "seid:v6.4.0" +) + +func seedNode() *seiv1alpha1.SeiNode { + return &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: "seed-0", Namespace: sourceChainID}, + Spec: seiv1alpha1.SeiNodeSpec{ + ChainID: sourceChainID, + Image: seedImage, + Seed: &seiv1alpha1.SeedSpec{ + NodeKey: seiv1alpha1.NodeKeySource{ + Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: "seed-0-node-key"}, + }, + }, + }, + } +} + +// A seed takes the genesis progression and validates its pinned identity before +// the StatefulSet is applied. The absences are the point: a seed stores no chain +// state, so restoring or state-syncing one is meaningless, and it signs nothing. +func TestSeedPlanner_GenesisProgression(t *testing.T) { + p := &seedPlanner{} + plan, err := p.BuildPlan(seedNode()) + if err != nil { + t.Fatalf("BuildPlan: %v", err) + } + + got := planTaskTypes(plan) + want := []string{ + task.TaskTypeEnsureDataPVC, + task.TaskTypeValidateNodeKey, + task.TaskTypeApplyRBACProxyConfig, + task.TaskTypeApplyStatefulSet, + task.TaskTypeApplyService, + TaskConfigureGenesis, + TaskConfigApply, + TaskConfigValidate, + TaskMarkReady, + } + if !slices.Equal(got, want) { + t.Errorf("seed init progression:\n got %v\nwant %v", got, want) + } + + for _, absent := range []string{ + TaskSnapshotRestore, + TaskConfigureStateSync, + task.TaskTypeValidateSigningKey, + task.TaskTypeValidateOperatorKeyring, + } { + if slices.Contains(got, absent) { + t.Errorf("seed plan must not contain %s, got %v", absent, got) + } + } +} + +// The config-apply payload carries the mode; sei-config's applySeedOverrides +// resolves every seed-specific key from it sidecar-side. +func TestSeedPlanner_ConfigIntentCarriesSeedMode(t *testing.T) { + p := &seedPlanner{} + plan, err := p.BuildPlan(seedNode()) + if err != nil { + t.Fatalf("BuildPlan: %v", err) + } + + idx := slices.IndexFunc(plan.Tasks, func(pt seiv1alpha1.PlannedTask) bool { + return pt.Type == TaskConfigApply + }) + if idx < 0 { + t.Fatalf("plan has no %s task: %v", TaskConfigApply, planTaskTypes(plan)) + } + + var intent seiconfig.ConfigIntent + if err := json.Unmarshal(plan.Tasks[idx].Params.Raw, &intent); err != nil { + t.Fatalf("unmarshaling config-apply params: %v", err) + } + if intent.Mode != seiconfig.ModeSeed { + t.Errorf("config-apply mode = %q, want %q", intent.Mode, seiconfig.ModeSeed) + } +} + +// A seed's NodeID is published, so an unpinned identity must fail the plan +// rather than boot a node that regenerates its key onto the data volume. +func TestSeedPlanner_ValidateRequiresNodeKey(t *testing.T) { + node := seedNode() + node.Spec.Seed.NodeKey = seiv1alpha1.NodeKeySource{} + + p := &seedPlanner{} + if err := p.Validate(node); err == nil { + t.Error("Validate should reject a seed with no nodeKey Secret") + } + + if err := p.Validate(seedNode()); err != nil { + t.Errorf("Validate on a well-formed seed: %v", err) + } +} + +// An image roll re-validates the identity Secret before touching the +// StatefulSet, so a broken Secret surfaces controller-side rather than as a +// kubelet mount failure on the replacement pod. +func TestSeedPlanner_UpdatePlanValidatesNodeKeyFirst(t *testing.T) { + node := seedNode() + node.Status.Phase = seiv1alpha1.PhaseRunning + node.Status.CurrentImage = seedPrevImage + + p := &seedPlanner{} + plan, err := p.BuildPlan(node) + if err != nil { + t.Fatalf("BuildPlan: %v", err) + } + if plan == nil { + t.Fatal("expected an update plan for a drifted image") + } + + got := planTaskTypes(plan) + if got[0] != task.TaskTypeValidateNodeKey { + t.Errorf("update plan should validate the node key first, got %v", got) + } + if slices.Contains(got, TaskConfigureStateSync) { + t.Errorf("update plan must not contain configure-state-sync, got %v", got) + } +} + +// Steady state builds nothing: no drift, no plan. +func TestSeedPlanner_NoPlanWithoutDrift(t *testing.T) { + node := seedNode() + node.Status.Phase = seiv1alpha1.PhaseRunning + node.Status.CurrentImage = node.Spec.Image + + p := &seedPlanner{} + plan, err := p.BuildPlan(node) + if err != nil { + t.Fatalf("BuildPlan: %v", err) + } + if plan != nil { + t.Errorf("expected no plan in steady state, got %v", planTaskTypes(plan)) + } +} diff --git a/internal/platform/load.go b/internal/platform/load.go index b64eb068..a95d000e 100644 --- a/internal/platform/load.go +++ b/internal/platform/load.go @@ -41,6 +41,7 @@ func Load() (Config, error) { NodepoolName: file.Scheduling.NodepoolName, NodepoolArchive: file.Scheduling.NodepoolArchive, NodepoolValidator: file.Scheduling.NodepoolValidator, + NodepoolSeed: file.Scheduling.NodepoolSeed, TolerationKey: file.Scheduling.TolerationKey, ServiceAccount: file.Scheduling.ServiceAccount, @@ -49,11 +50,13 @@ func Load() (Config, error) { StorageClassArchive: file.Storage.ClassArchive, StorageSizeDefault: file.Storage.SizeDefault, StorageSizeArchive: file.Storage.SizeArchive, + StorageSizeSeed: file.Storage.SizeSeed, NodeResourcesValidator: file.Resources.Validator, NodeResourcesNode: file.Resources.Node, NodeResourcesReplayer: file.Resources.Replayer, NodeResourcesArchive: file.Resources.Archive, + NodeResourcesSeed: file.Resources.Seed, SnapshotBucket: file.Snapshot.Bucket, SnapshotRegion: file.Snapshot.Region, diff --git a/internal/platform/platform.go b/internal/platform/platform.go index 6d81b129..ad55437d 100644 --- a/internal/platform/platform.go +++ b/internal/platform/platform.go @@ -34,6 +34,14 @@ const ( // modeValidator matches seiconfig.ModeValidator without importing sei-config. modeValidator = "validator" + + // modeSeed matches seiconfig.ModeSeed without importing sei-config. + modeSeed = "seed" + + // defaultStorageSizeSeed backs a seed's peer store plus the block and state + // DBs seid opens but never writes. Code-authoritative so a seed does not + // silently claim StorageSizeDefault, which is sized for a chain's full state. + defaultStorageSizeSeed = "20Gi" ) // Config holds infrastructure-level settings that vary per deployment @@ -54,6 +62,16 @@ type Config struct { StorageSizeDefault string StorageSizeArchive string + // NodepoolSeed is required only by clusters that run seeds — absent from + // Validate's startup list so existing app-config files stay valid, and + // enforced instead where a seed pod is rendered. It has no fallback: see + // NodepoolForMode. + // + // StorageSizeSeed is genuinely optional; defaultStorageSizeSeed is a correct + // value, not a placeholder. Read via SeedStorageSize. + NodepoolSeed string + StorageSizeSeed string + // Per-role seid-container resource overrides, keyed by sei.io/role. Each is // optional: an unset field (or unset sub-field) falls back to the // code-authoritative, prod-safe default for that role in internal/noderesource. @@ -62,6 +80,7 @@ type Config struct { NodeResourcesNode ResourceOverride NodeResourcesReplayer ResourceOverride NodeResourcesArchive ResourceOverride + NodeResourcesSeed ResourceOverride SnapshotBucket string SnapshotRegion string @@ -130,8 +149,10 @@ type SchedulingConfig struct { NodepoolName string `json:"nodepoolName"` NodepoolArchive string `json:"nodepoolArchive"` NodepoolValidator string `json:"nodepoolValidator"` - TolerationKey string `json:"tolerationKey"` - ServiceAccount string `json:"serviceAccount"` + // NodepoolSeed is optional; unset routes seeds to NodepoolName. + NodepoolSeed string `json:"nodepoolSeed"` + TolerationKey string `json:"tolerationKey"` + ServiceAccount string `json:"serviceAccount"` } // StorageConfig holds the PVC storage classes and sizes for default and archive nodes. @@ -141,6 +162,8 @@ type StorageConfig struct { ClassArchive string `json:"classArchive"` SizeDefault string `json:"sizeDefault"` SizeArchive string `json:"sizeArchive"` + // SizeSeed is optional; unset falls back to defaultStorageSizeSeed. + SizeSeed string `json:"sizeSeed"` } // ResourcesConfig holds seid-container resource sizing. @@ -155,6 +178,7 @@ type ResourcesConfig struct { Node ResourceOverride `json:"node"` Replayer ResourceOverride `json:"replayer"` Archive ResourceOverride `json:"archive"` + Seed ResourceOverride `json:"seed"` } // ResourceOverride is a per-mode seid-container resource override. Both fields @@ -192,18 +216,37 @@ type ImagesConfig struct { // NodepoolForMode returns the Karpenter NodePool name for the given // sei-config mode string. Archive and validator nodes each use a -// dedicated pool; all other modes share the default pool. +// dedicated pool; seed uses one when configured, since its footprint is far +// smaller than the default pool's instance class. All other modes share the +// default pool. func (c Config) NodepoolForMode(mode string) string { switch mode { case modeArchive: return c.NodepoolArchive case modeValidator: return c.NodepoolValidator + case modeSeed: + // No fallback to NodepoolName: that pool is sized for RPC-class nodes, so + // a 2Gi seed landing there costs an order of magnitude more than a seed is + // worth — and karpenter.sh/do-not-disrupt then pins that node against + // consolidation for the seed's life. Callers rendering a seed pod must + // reject an empty value rather than substitute one; see + // noderesource.GenerateStatefulSet. + return c.NodepoolSeed default: return c.NodepoolName } } +// SeedStorageSize returns the data-volume size for a seed node, falling back to +// the code default when the app-config file omits storage.sizeSeed. +func (c Config) SeedStorageSize() string { + if c.StorageSizeSeed != "" { + return c.StorageSizeSeed + } + return defaultStorageSizeSeed +} + // Validate returns an error if a required field is unset. name is the field's // app-config file key, except the networking/gateway fields (still env-sourced // pending PLT-451) which name their env var. Slice order is the report order @@ -251,6 +294,7 @@ func (c Config) Validate() error { {"resources.node", c.NodeResourcesNode}, {"resources.replayer", c.NodeResourcesReplayer}, {"resources.archive", c.NodeResourcesArchive}, + {"resources.seed", c.NodeResourcesSeed}, } for _, o := range overrides { if err := o.val.validate(o.key); err != nil { diff --git a/internal/platform/platform_test.go b/internal/platform/platform_test.go index 5fccb191..654f1a96 100644 --- a/internal/platform/platform_test.go +++ b/internal/platform/platform_test.go @@ -16,19 +16,21 @@ func TestDataDirIsUnderHomeDir(t *testing.T) { } } -// TestNodepoolForMode pins the mode→pool routing: archive and validator each -// get a dedicated pool; every other mode ("full", and the empty fallback) +// TestNodepoolForMode pins the mode→pool routing: archive, validator and seed +// each get a dedicated pool; every other mode ("full", and the empty fallback) // shares the default pool. func TestNodepoolForMode(t *testing.T) { const ( poolDefault = "sei-node" poolArchive = "sei-archive" poolValidator = "sei-validator" + poolSeed = "sei-seed" ) c := Config{ NodepoolName: poolDefault, NodepoolArchive: poolArchive, NodepoolValidator: poolValidator, + NodepoolSeed: poolSeed, } cases := []struct { mode string @@ -36,6 +38,7 @@ func TestNodepoolForMode(t *testing.T) { }{ {"archive", poolArchive}, {"validator", poolValidator}, + {"seed", poolSeed}, {"full", poolDefault}, {"", poolDefault}, } @@ -45,3 +48,32 @@ func TestNodepoolForMode(t *testing.T) { } } } + +// A seed must never inherit the RPC-class default pool: that costs an order of +// magnitude more than a seed is worth. NodepoolForMode reports the absence so +// the pod render can reject it. +func TestNodepoolForMode_SeedHasNoDefaultPoolFallback(t *testing.T) { + c := Config{NodepoolName: "sei-node"} + + if got := c.NodepoolForMode(modeSeed); got != "" { + t.Errorf("NodepoolForMode(seed) with no seed pool = %q, want empty", got) + } + if got := c.NodepoolForMode("full"); got != "sei-node" { + t.Errorf("NodepoolForMode(full) = %q, want the default pool", got) + } +} + +// sizeSeed is optional: the code default is a correct value, so an app-config +// file predating the key still sizes a seed volume sanely. +func TestSeedStorageSize_FallsBackToCodeDefault(t *testing.T) { + c := Config{} + + if got := c.SeedStorageSize(); got != defaultStorageSizeSeed { + t.Errorf("SeedStorageSize() = %q, want %q", got, defaultStorageSizeSeed) + } + + c.StorageSizeSeed = "40Gi" + if got := c.SeedStorageSize(); got != "40Gi" { + t.Errorf("SeedStorageSize() with override = %q, want 40Gi", got) + } +} diff --git a/internal/platform/platformtest/config.go b/internal/platform/platformtest/config.go index 9145ff2d..f969f68f 100644 --- a/internal/platform/platformtest/config.go +++ b/internal/platform/platformtest/config.go @@ -10,6 +10,7 @@ func Config() platform.Config { NodepoolName: "sei-node", NodepoolArchive: "sei-archive", NodepoolValidator: "sei-validator", + NodepoolSeed: "sei-seed", TolerationKey: "sei.io/workload", ServiceAccount: "seid-node", StorageClassPerf: "gp3-10k-750", diff --git a/internal/task/bootstrap_resources.go b/internal/task/bootstrap_resources.go index 94ac7a86..ce2433c7 100644 --- a/internal/task/bootstrap_resources.go +++ b/internal/task/bootstrap_resources.go @@ -311,6 +311,13 @@ func bootstrapPVCClaimName(node *seiv1alpha1.SeiNode) string { } // bootstrapNodeMode determines the sei-config mode string from the node spec. +// +// Deliberately has no seed arm: NeedsBootstrap requires a snapshot source with a +// bootstrapImage, and SeiNodeSpec.SnapshotSource() is nil for a seed, so a seed +// never reaches the bootstrap-Job path. That invariant is what keeps the default +// arm from rendering a seed as a full node here — with RPC probes and a +// cosmos-exporter — and it is also why buildBootstrapPodSpec does not call +// ValidateSeedProbes. Giving seed a snapshot source would break both silently. func bootstrapNodeMode(node *seiv1alpha1.SeiNode) string { switch { case node.Spec.Archive != nil: diff --git a/manifests/sei.io_seinodes.yaml b/manifests/sei.io_seinodes.yaml index 5646ce81..62ba6dc7 100644 --- a/manifests/sei.io_seinodes.yaml +++ b/manifests/sei.io_seinodes.yaml @@ -52,8 +52,8 @@ spec: spec: description: |- SeiNodeSpec defines the desired state of a standalone Sei node. - Exactly one mode sub-spec (fullNode, archive, replayer, validator) must be set; - the populated field determines the node's operating mode. + Exactly one mode sub-spec (fullNode, archive, replayer, validator, seed) must + be set; the populated field determines the node's operating mode. properties: archive: description: Archive configures an archive node with full history @@ -446,6 +446,60 @@ spec: required: - snapshot type: object + seed: + description: Seed configures a peer-discovery seed node (P2P + PEX + only). + properties: + nodeKey: + description: |- + NodeKey supplies the P2P identity (node_key.json) this seed presents. + + Required, where the validator's NodeKey is optional, because a seed's + NodeID is published: operators dial `NodeID@host:port` and the + secret-connection handshake verifies the pinned value, so a changed NodeID + silently breaks every client carrying the old one. A Secret-sourced key + survives pod recreation and PVC loss; one left to `seid init` regenerates + onto the data volume. Carrying the identity to another cluster means + replicating the Secret there — the controller reads it, it does not move + it. Treat the NodeID as a one-way door. + + The Secret name is immutable, so rotating the identity means deleting and + recreating the SeiNode — and announcing the new NodeID. There is no fast + path for a leaked key: the old NodeID stays dialable until every client + ships a new default, so a thief keeps impersonating a bootstrap anchor. + + Give each seed a distinct Secret. Nothing rejects two seeds sharing one, + and they would present the same NodeID — collapsing the redundant anchors + into a single entry in every dialer's peer store. + properties: + secret: + description: |- + Secret loads the node key from a Kubernetes Secret in the + SeiNode's namespace. + properties: + secretName: + description: |- + SecretName is the name of a Secret in the SeiNode's namespace. + The controller never creates, mutates, or deletes this Secret. + Immutable: re-pointing the node ID on a running validator costs + peer reputation and forces a peer-graph reset; force delete-and-recreate. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + x-kubernetes-validations: + - message: secretName is immutable + rule: self == oldSelf + required: + - secretName + type: object + type: object + x-kubernetes-validations: + - message: exactly one node key source must be set + rule: '(has(self.secret) ? 1 : 0) == 1' + required: + - nodeKey + type: object sidecar: description: Sidecar configures the sei-sidecar container. properties: @@ -827,10 +881,11 @@ spec: - image type: object x-kubernetes-validations: - - message: exactly one of fullNode, archive, replayer, or validator must - be set + - message: exactly one of fullNode, archive, replayer, validator, or seed + must be set rule: '(has(self.fullNode) ? 1 : 0) + (has(self.archive) ? 1 : 0) + - (has(self.replayer) ? 1 : 0) + (has(self.validator) ? 1 : 0) == 1' + (has(self.replayer) ? 1 : 0) + (has(self.validator) ? 1 : 0) + (has(self.seed) + ? 1 : 0) == 1' - message: peers is required when replayer mode is set rule: '!has(self.replayer) || (has(self.peers) && size(self.peers) > 0)' From 848f17d394e13be928462b815c0c2b0a2a57899a Mon Sep 17 00:00:00 2001 From: bdchatham Date: Mon, 27 Jul 2026 12:24:11 -0700 Subject: [PATCH 2/6] chore(seed): bump sei-config to v0.0.24, size seed at the RFC ceiling The bump is what makes the controller's own sei-config constants match the released library; the sidecar carries the seed config defaults. It also lets ServicePorts compare against seiconfig.PortNameGRPC instead of a bare literal, which the constant was added for. Raise the seed memory profile from 2Gi to 4Gi. RFC 006 Appendix A gives "2 GB (4 GB for a primary seed)" and 1000 peers; the profile took the floor of the memory range while sei-config configures the top of the peer range. 1000 connections sets maxInbound, and with it the concurrent pre-auth handshake count, each reading up to a 1 MiB peer-declared size no config key bounds -- a burst that dwarfs live connection state and lands on the full cosmos app seid builds before dispatching on mode. 4Gi is the same t4g.medium class the RFC budgets. GOMEMLIMIT follows automatically: it derives from the resolved limit. ServicePorts keeps rendering from NodePorts() rather than NodePortsForMode(): the latter reports validator as p2p+metrics only, while applyValidatorOverrides enables gRPC for the price feeder and cosmos-exporter, so adopting it would strip a port validators serve. Co-Authored-By: Claude Opus 5 (1M context) --- go.mod | 2 +- go.sum | 2 ++ internal/noderesource/noderesource.go | 25 +++++++++++++++---------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 09dde97b..4751e6be 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 github.com/onsi/gomega v1.39.1 - github.com/sei-protocol/sei-config v0.0.23 + github.com/sei-protocol/sei-config v0.0.24 github.com/sei-protocol/seictl v0.0.68 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0 diff --git a/go.sum b/go.sum index 6e7af962..a52ebc07 100644 --- a/go.sum +++ b/go.sum @@ -172,6 +172,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sei-protocol/sei-config v0.0.23 h1:pGFxRnKoXZLK3Ew/Vd5lgC5RvH9Wo58NnL19CuMgFIk= github.com/sei-protocol/sei-config v0.0.23/go.mod h1:zcEdLzyIH2AyP0/QRBE3s4Y9eGn0C/qAUx1c4o4EROU= +github.com/sei-protocol/sei-config v0.0.24 h1:DqjehEjC24E7/vVQR6EDPjLOdOUOCegxFwZAeB7S23k= +github.com/sei-protocol/sei-config v0.0.24/go.mod h1:zcEdLzyIH2AyP0/QRBE3s4Y9eGn0C/qAUx1c4o4EROU= github.com/sei-protocol/seictl v0.0.68 h1:dCT94Ys4OjiPt5Y6TWqtgJ8GKTiWv7tN87D8HqIQxbk= github.com/sei-protocol/seictl v0.0.68/go.mod h1:kI3HIAIWzuJSme8LqJH1WZiITrSIx5yDtJJea07Xt8Y= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= diff --git a/internal/noderesource/noderesource.go b/internal/noderesource/noderesource.go index 84e4c593..dae01c29 100644 --- a/internal/noderesource/noderesource.go +++ b/internal/noderesource/noderesource.go @@ -71,7 +71,7 @@ const ( memValidator = "128Gi" // full r5.4xlarge envelope; provisioned on the next size up (r5.8xlarge) memRPCClass = "256Gi" // full r7i.8xlarge envelope (fullNode + replayer); provisioned on r7i.12xlarge memArchive = "512Gi" // full r7i.16xlarge envelope (verified live prod); provisioned on r7i.24xlarge - memSeed = "2Gi" // peer buffers + idle DBs only; see defaultNodeResourceProfiles + memSeed = "4Gi" // t4g.medium class; sized for the handshake burst, see defaultNodeResourceProfiles cpuValidator = "16" // r5.4xlarge vCPU; request-only, no CPU limit cpuRPCClass = "32" // r7i.8xlarge vCPU (fullNode + replayer); request-only, no CPU limit cpuArchive = "64" // r7i.16xlarge vCPU; request-only, no CPU limit @@ -408,12 +408,19 @@ var defaultNodeResourceProfiles = map[string]nodeResourceProfile{ // TF's m7i.8xlarge is stale). roleArchive: {cpuRequest: cpuArchive, memory: memArchive}, // Peer discovery only: no block execution, no state commitment, no query - // surface. The working set is P2P connection buffers plus the block/state DBs - // seid opens and never writes. The exception to the "request == the reference - // instance's full envelope" rule above: a seed targets the ~2 vCPU / 2-4 GiB - // t-class shape RFC 006 budgets, and on a node that small the fixed - // kube/system reservation is a large fraction of capacity — so the request - // sits well below advertised RAM rather than equal to it. + // surface. The exception to the "request == the reference instance's full + // envelope" rule above: a seed targets the ~2 vCPU / 2-4 GiB t-class shape + // RFC 006 budgets, and on a node that small the fixed kube/system reservation + // is a large fraction of capacity — so the request sits below advertised RAM + // rather than equal to it. + // + // 4Gi, the top of that range (t4g.medium), because steady state is not what + // sizes a seed. sei-config caps the seed at 1000 connections, which sets + // maxInbound and with it the concurrent pre-auth handshake count; each of + // those reads up to a 1 MiB peer-declared size that no config key bounds. That + // burst dwarfs the ~150 MB of live connection state and lands on top of the + // full cosmos app seid builds before it dispatches on mode. Lowering the + // connection cap or bounding that read upstream would buy 2Gi back. roleSeed: {cpuRequest: cpuSeed, memory: memSeed}, } @@ -669,9 +676,7 @@ func ServicePorts() []corev1.ServicePort { ports := make([]corev1.ServicePort, len(np)) for i, p := range np { ports[i] = corev1.ServicePort{Name: p.Name, Port: p.Port, TargetPort: intstr.FromInt32(p.Port), Protocol: corev1.ProtocolTCP} - // Literal, not seiconfig.PortNameGRPC: that constant ships in an - // unreleased sei-config. Adopt it at the next module bump. - if p.Name == "grpc" { + if p.Name == seiconfig.PortNameGRPC { h2c := "kubernetes.io/h2c" ports[i].AppProtocol = &h2c } From 48506e6e269f8317bc75cc0efa22171aff615e82 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Mon, 27 Jul 2026 12:43:29 -0700 Subject: [PATCH 3/6] fix(seed): surface render failures, correct sizing docs, pin the bootstrap invariant Review findings from the seed-mode cross-review. A StatefulSet render failure returned before the status flush, so a seed created before scheduling.nodepoolSeed exists left no phase and no condition -- the operator saw a stuck SeiNode with nothing explaining it, and the reason only in controller logs. Infra config loads once at startup, so applying a seed before restarting the controller reaches this on ordinary operator sequencing. Emit an Event so the reason reaches kubectl describe. Correct the sizing documentation. Four comments still described the seed as 2Gi after the profile moved to 4Gi, and the profile claimed a t4g.medium class that a 4Gi request==limit cannot schedule onto once kube/system reservation is taken -- the pool provisions the next size up. The rationale also cited connection counts and buffer sizes that live in sei-config and sei-tendermint, where this repo cannot track them. GOMEMLIMIT's doc oversold it: it governs churn, not live memory, and does not see non-heap RSS at all -- so absolute headroom is what absorbs the handshake burst, not the soft limit. The stated reason for scoping it to seed was also wrong; GC slack is proportional to the live set, not absolute, so the honest reason is caution about retuning prod-proven modes. Multiply before dividing, and skip the env entirely rather than emit a degenerate soft limit for a pathologically small override. ValidateSeedProbes claimed a failing readiness probe drops the pod from Service endpoints; PublishNotReadyAddresses means it does not. What it actually costs is the Ready condition and anything gating on it. A seed stays out of the bootstrap-Job path only because its SnapshotSource is nil -- bootstrapNodeMode has no seed arm and would render it as a full node with RPC probes and no ValidateSeedProbes call, with nothing failing to compile. TestSeedNeverBootstraps pins that invariant instead of a comment guarding it. NodeKeySource and SecretNodeKeySource are reused by SeedSpec but their docs described a validator's key as not slashing-relevant and low-stakes. Those land verbatim in kubectl explain seinode.spec.seed.nodeKey, where a published, effectively permanent NodeID is the opposite of low-stakes. Co-Authored-By: Claude Opus 5 (1M context) --- api/v1alpha1/seed_types.go | 4 +- api/v1alpha1/validator_types.go | 20 +++--- internal/controller/node/controller.go | 9 +++ .../node/envtest/seed_validation_test.go | 3 +- .../controller/node/plan_execution_test.go | 32 ++++++++++ internal/controller/node/testhelpers_test.go | 4 +- internal/noderesource/noderesource.go | 63 ++++++++++++------- internal/noderesource/noderesource_test.go | 9 +-- internal/noderesource/probe_guard.go | 11 +++- internal/platform/platform.go | 2 +- .../sidecartransport/sidecartransport_test.go | 1 - internal/task/bootstrap_resources.go | 2 +- 12 files changed, 113 insertions(+), 47 deletions(-) diff --git a/api/v1alpha1/seed_types.go b/api/v1alpha1/seed_types.go index 293f5763..1f487495 100644 --- a/api/v1alpha1/seed_types.go +++ b/api/v1alpha1/seed_types.go @@ -18,8 +18,8 @@ package v1alpha1 // (otherwise seid advertises its in-cluster listen address over PEX and every // node that learns the seed by gossip gets an unroutable peer), a load balancer // and DNS record for TCP 26656 that outlive pod and node churn, and an ingress -// allowance for that port. The readiness probe cannot detect any of them -// missing — see readinessProbeForNode. +// allowance for that port. Nothing detects any of these missing: a seed reads +// Ready when its P2P port is bound, and nothing more. // // A seed serves no RPC, so it has no /status and no `catching_up`: sync // freshness is not a property a seed has. Monitor it on P2P metrics diff --git a/api/v1alpha1/validator_types.go b/api/v1alpha1/validator_types.go index 07ef6819..838742ac 100644 --- a/api/v1alpha1/validator_types.go +++ b/api/v1alpha1/validator_types.go @@ -129,8 +129,8 @@ type SecretSigningKeySource struct { SecretName string `json:"secretName"` } -// NodeKeySource declares where a validator's P2P node key -// (node_key.json) comes from. Exactly one variant must be set. +// NodeKeySource declares where a node's P2P node key (node_key.json) comes +// from. Exactly one variant must be set. Used by validator and seed modes. // // +kubebuilder:validation:XValidation:rule="(has(self.secret) ? 1 : 0) == 1",message="exactly one node key source must be set" type NodeKeySource struct { @@ -140,14 +140,16 @@ type NodeKeySource struct { Secret *SecretNodeKeySource `json:"secret,omitempty"` } -// SecretNodeKeySource references a Kubernetes Secret containing the -// validator's P2P node key. The Secret must contain a data key -// `node_key.json` holding the Tendermint node key, mounted read-only at -// $SEI_HOME/config/node_key.json. +// SecretNodeKeySource references a Kubernetes Secret containing a node's P2P +// node key. The Secret must contain a data key `node_key.json` holding the +// Tendermint node key, mounted read-only at $SEI_HOME/config/node_key.json. // -// node_key.json is identity-bearing but not slashing-relevant — losing it -// only costs the validator's accumulated peer-graph reputation, not stake. -// Treated as Secret-grade for handling consistency with SigningKey. +// How much the key matters depends on the mode that references it. For a +// validator it is identity-bearing but not slashing-relevant — losing it costs +// only accumulated peer-graph reputation, not stake — and is Secret-grade for +// handling consistency with SigningKey. For a seed the derived NodeID is +// published and dialed by strangers, which makes it a one-way door; see +// SeedSpec.NodeKey before treating it as low-stakes. type SecretNodeKeySource struct { // SecretName is the name of a Secret in the SeiNode's namespace. // The controller never creates, mutates, or deletes this Secret. diff --git a/internal/controller/node/controller.go b/internal/controller/node/controller.go index 23d44c35..4a419788 100644 --- a/internal/controller/node/controller.go +++ b/internal/controller/node/controller.go @@ -159,6 +159,15 @@ func (r *SeiNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct holdForWorkflow := node.Status.AdoptedWorkflow != nil && !adoptedWorkflowParkedFailed(node) if !holdInitialSTS && !holdForWorkflow { if err := r.reconcileStatefulSet(ctx, node); err != nil { + // This return precedes the status flush, so a render failure leaves no + // phase and no condition behind — the operator would see a SeiNode + // stuck with nothing explaining it. Emit an Event so the reason reaches + // `kubectl describe` rather than only the controller log. Reachable on + // operator-supplied app-config: infra fields load once at startup, so a + // seed applied before the controller restarts renders against the old + // config. + r.Recorder.Eventf(node, corev1.EventTypeWarning, "StatefulSetRenderFailed", + "Cannot render the StatefulSet: %v", err) return ctrl.Result{}, fmt.Errorf("reconciling statefulset: %w", err) } } diff --git a/internal/controller/node/envtest/seed_validation_test.go b/internal/controller/node/envtest/seed_validation_test.go index 005d945c..9ee26a50 100644 --- a/internal/controller/node/envtest/seed_validation_test.go +++ b/internal/controller/node/envtest/seed_validation_test.go @@ -60,7 +60,8 @@ func TestSeed_WithSecondMode_Rejected(t *testing.T) { g.Expect(err.Error()).To(ContainSubstring("exactly one of")) } -// Widening the exactly-one rule must not reject the four pre-existing modes. +// Widening the exactly-one rule must not reject the pre-existing modes. +// Replayer is omitted: CEL additionally requires peers for it. func TestSeed_OtherModesStillAccepted(t *testing.T) { ns := makeNamespace(t) diff --git a/internal/controller/node/plan_execution_test.go b/internal/controller/node/plan_execution_test.go index 32247f6e..6ce875eb 100644 --- a/internal/controller/node/plan_execution_test.go +++ b/internal/controller/node/plan_execution_test.go @@ -216,6 +216,21 @@ func genesisNode() *seiv1alpha1.SeiNode { } } +func seedNode() *seiv1alpha1.SeiNode { + return &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: "test-seed", Namespace: testNamespace, Generation: 1}, + Spec: seiv1alpha1.SeiNodeSpec{ + ChainID: testChainID, + Image: testImage, + Seed: &seiv1alpha1.SeedSpec{ + NodeKey: seiv1alpha1.NodeKeySource{ + Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: "test-seed-node-key"}, + }, + }, + }, + } +} + func snapshotterNode() *seiv1alpha1.SeiNode { return &seiv1alpha1.SeiNode{ ObjectMeta: metav1.ObjectMeta{Name: "test-node", Namespace: "default", Generation: 1}, @@ -852,6 +867,7 @@ func TestNeedsBootstrap(t *testing.T) { {"replayer without bootstrap image", replayerNode(), false}, {"full node without bootstrap image", snapshotNode(), false}, {"genesis node", genesisNode(), false}, + {"seed", seedNode(), false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -862,6 +878,22 @@ func TestNeedsBootstrap(t *testing.T) { } } +// A seed must never reach the bootstrap-Job path. bootstrapNodeMode has no seed +// arm and would render one as a full node — RPC probes, cosmos-exporter, and no +// ValidateSeedProbes call — and nothing there would fail to compile. The only +// thing preventing it is a nil SnapshotSource, so pin that rather than leave the +// invariant to a comment. +func TestSeedNeverBootstraps(t *testing.T) { + node := seedNode() + + if snap := node.Spec.SnapshotSource(); snap != nil { + t.Fatalf("a seed must have no snapshot source, got %+v", snap) + } + if planner.NeedsBootstrap(node) { + t.Error("a seed must never need a bootstrap Job") + } +} + // --- Bootstrap resource builder tests (task package) --- func TestTaskGenerateBootstrapJob(t *testing.T) { diff --git a/internal/controller/node/testhelpers_test.go b/internal/controller/node/testhelpers_test.go index aac0a273..0b271e5f 100644 --- a/internal/controller/node/testhelpers_test.go +++ b/internal/controller/node/testhelpers_test.go @@ -11,8 +11,8 @@ func newGenesisNode(name, namespace string) *seiv1alpha1.SeiNode { //nolint:unpa return &seiv1alpha1.SeiNode{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: seiv1alpha1.SeiNodeSpec{ - ChainID: "sei-test", - Image: "ghcr.io/sei-protocol/seid:latest", + ChainID: "sei-test", + Image: "ghcr.io/sei-protocol/seid:latest", Validator: &seiv1alpha1.ValidatorSpec{}, Sidecar: &seiv1alpha1.SidecarConfig{Port: 7777}, }, diff --git a/internal/noderesource/noderesource.go b/internal/noderesource/noderesource.go index dae01c29..dac5c3ef 100644 --- a/internal/noderesource/noderesource.go +++ b/internal/noderesource/noderesource.go @@ -71,16 +71,14 @@ const ( memValidator = "128Gi" // full r5.4xlarge envelope; provisioned on the next size up (r5.8xlarge) memRPCClass = "256Gi" // full r7i.8xlarge envelope (fullNode + replayer); provisioned on r7i.12xlarge memArchive = "512Gi" // full r7i.16xlarge envelope (verified live prod); provisioned on r7i.24xlarge - memSeed = "4Gi" // t4g.medium class; sized for the handshake burst, see defaultNodeResourceProfiles + memSeed = "4Gi" // sized for the pre-auth handshake burst, see defaultNodeResourceProfiles cpuValidator = "16" // r5.4xlarge vCPU; request-only, no CPU limit cpuRPCClass = "32" // r7i.8xlarge vCPU (fullNode + replayer); request-only, no CPU limit cpuArchive = "64" // r7i.16xlarge vCPU; request-only, no CPU limit cpuSeed = "1" // PEX gossip is light and spiky; request-only, no CPU limit // goMemLimitPercentOfLimit is the seed's GOMEMLIMIT as a percentage of its - // memory limit. The 13% margin absorbs the non-heap RSS the Go soft limit - // does not govern: thread stacks, mmap'd memIAVL pages, and cgo/kernel - // allocations. See goMemLimitEnv. + // memory limit; the remainder is the non-heap margin. See goMemLimitEnv. goMemLimitPercentOfLimit = 87 dataDir = platform.DataDir @@ -409,18 +407,19 @@ var defaultNodeResourceProfiles = map[string]nodeResourceProfile{ roleArchive: {cpuRequest: cpuArchive, memory: memArchive}, // Peer discovery only: no block execution, no state commitment, no query // surface. The exception to the "request == the reference instance's full - // envelope" rule above: a seed targets the ~2 vCPU / 2-4 GiB t-class shape - // RFC 006 budgets, and on a node that small the fixed kube/system reservation - // is a large fraction of capacity — so the request sits below advertised RAM - // rather than equal to it. + // envelope" rule above — a seed is deliberately small. // - // 4Gi, the top of that range (t4g.medium), because steady state is not what - // sizes a seed. sei-config caps the seed at 1000 connections, which sets - // maxInbound and with it the concurrent pre-auth handshake count; each of - // those reads up to a 1 MiB peer-declared size that no config key bounds. That - // burst dwarfs the ~150 MB of live connection state and lands on top of the - // full cosmos app seid builds before it dispatches on mode. Lowering the - // connection cap or bounding that read upstream would buy 2Gi back. + // Sized for the concurrent pre-auth handshake burst rather than steady state: + // the seed config's connection cap sets how many unauthenticated handshakes + // can be in flight at once, and each reads a peer-declared size bounded well + // above what a legitimate handshake needs. That burst is live memory, so only + // absolute headroom absorbs it, and it lands on top of the full cosmos app + // seid builds before dispatching on mode. RFC 006 Appendix A's primary-seed + // figure. Bounding that read upstream is what would let this shrink; the + // current numbers live in sei-config and sei-tendermint, not here. + // + // Note this exceeds what a 4 GiB node can schedule once kube/system + // reservation is taken, so the seed nodepool provisions the next size up. roleSeed: {cpuRequest: cpuSeed, memory: memSeed}, } @@ -488,7 +487,7 @@ func GenerateStatefulSet(node *seiv1alpha1.SeiNode, p PlatformConfig) (*appsv1.S // Seeds need their own small pool. Required here rather than in // Config.Validate so a cluster that runs no seeds keeps booting on an // app-config file that predates the key — and so the alternative, silently - // scheduling a 2Gi seed onto the RPC-class default pool, is impossible. + // scheduling a small seed onto the RPC-class default pool, is impossible. if !servesSeidRPC(node) && p.NodepoolForMode(NodeMode(node)) == "" { return nil, fmt.Errorf("scheduling.nodepoolSeed is not configured in the app-config file; a seed requires a dedicated small-instance nodepool") } @@ -900,14 +899,27 @@ func buildSidecarMainContainer(node *seiv1alpha1.SeiNode, p PlatformConfig) core // goMemLimitEnv returns the GOMEMLIMIT env var for a seed's seid container. // // The Go runtime targets a heap roughly twice the live set and cannot see the -// cgroup limit, so with memory request==limit an allocation burst is an OOMKill -// rather than a throttle. A soft limit below the hard cap makes the GC spend CPU -// to stay resident instead. Scoped to seed: it is the only mode whose footprint -// (2Gi) is small enough for ordinary GC slack to cross the cap — the r-class -// modes carry absolute headroom in the tens of GiB. +// cgroup limit, so with memory request==limit reclaimable growth becomes an +// OOMKill rather than a throttle. A soft limit below the hard cap makes the GC +// spend CPU to stay resident instead. +// +// What it does and does not cover: it governs churn — buffers allocated and +// freed at rate, which GOGC alone would let balloon toward twice the live set. +// It cannot shrink memory that is genuinely live, and it does not account for +// non-heap RSS at all: thread stacks, cgo allocations, mmap'd memIAVL pages, or +// the kernel socket buffers a thousand connections charge to the cgroup. The +// concurrent pre-auth handshake burst that sizes this mode is live memory, so +// absolute headroom is what defends it — not this limit. The margin below the +// cap exists for the non-heap share. +// +// Scoped to seed as a matter of caution rather than principle: GC slack is +// proportional to the live set, so any mode with request==limit would benefit. +// The other modes are tuned and prod-proven, and changing their runtime +// behaviour is not this change's business. // // Derived from the resolved limit, so an app-config resources.seed override -// keeps it consistent. Reports false when no memory limit is set. +// keeps it consistent. Reports false when no memory limit is set, or when the +// limit is small enough that the computed soft cap would be useless. func goMemLimitEnv(node *seiv1alpha1.SeiNode, res corev1.ResourceRequirements) (corev1.EnvVar, bool) { if servesSeidRPC(node) { return corev1.EnvVar{}, false @@ -916,7 +928,12 @@ func goMemLimitEnv(node *seiv1alpha1.SeiNode, res corev1.ResourceRequirements) ( if !ok { return corev1.EnvVar{}, false } - soft := lim.Value() / 100 * goMemLimitPercentOfLimit + // Multiply first: the truncation from dividing first is harmless but the + // product cannot overflow int64 at any schedulable memory limit. + soft := lim.Value() * goMemLimitPercentOfLimit / 100 + if soft <= 0 { + return corev1.EnvVar{}, false + } return corev1.EnvVar{Name: "GOMEMLIMIT", Value: fmt.Sprintf("%dB", soft)}, true } diff --git a/internal/noderesource/noderesource_test.go b/internal/noderesource/noderesource_test.go index 670b1205..8e6be5e7 100644 --- a/internal/noderesource/noderesource_test.go +++ b/internal/noderesource/noderesource_test.go @@ -2028,7 +2028,7 @@ func TestSeed_GoMemLimitBelowMemoryLimit(t *testing.T) { func TestNonSeed_HasNoGoMemLimit(t *testing.T) { g := NewWithT(t) - sts := mustGenerateStatefulSet(t, newSnapshotNode("rpc-0", "default"), platformtest.Config()) + sts := mustGenerateStatefulSet(t, newSnapshotNode("rpc-0", testNamespace), platformtest.Config()) seid := findContainer(sts.Spec.Template.Spec.Containers, containerNameSeid) g.Expect(findEnvVar(seid.Env, "GOMEMLIMIT")).To(BeNil()) @@ -2043,7 +2043,7 @@ func findEnvVar(env []corev1.EnvVar, name string) *corev1.EnvVar { return nil } -// The alternative to failing closed is silently scheduling a 2Gi seed onto the +// The alternative to failing closed is silently scheduling a small seed onto the // RPC-class default pool, which costs an order of magnitude more than a seed is // worth. Clusters running no seeds are unaffected — the check is per-render, not // at startup. @@ -2056,8 +2056,9 @@ func TestSeed_RenderFailsWithoutSeedNodepool(t *testing.T) { g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring("scheduling.nodepoolSeed")) - // Every other mode still renders on the same config. - _, err = GenerateStatefulSet(newSnapshotNode("rpc-0", "default"), cfg) + // Every other mode still renders on the same config — the requirement is + // per-render, so a cluster running no seeds is unaffected. + _, err = GenerateStatefulSet(newSnapshotNode("rpc-0", testNamespace), cfg) g.Expect(err).NotTo(HaveOccurred()) } diff --git a/internal/noderesource/probe_guard.go b/internal/noderesource/probe_guard.go index 7ed8d4bf..c2477fbd 100644 --- a/internal/noderesource/probe_guard.go +++ b/internal/noderesource/probe_guard.go @@ -35,9 +35,14 @@ func ValidateGatedSeidProbes(c corev1.Container) error { // ValidateSeedProbes is the seed counterpart of the gated-seid contract: a seed's // seid container must carry no probe against seid's RPC or gRPC port, since seed -// mode binds neither for the pod's whole life. A permanently-failing readiness -// probe holds the pod out of Service endpoints and out of any NLB target group -// fronting it — leaving the seed unreachable, which is its only job. +// mode binds neither for the pod's whole life. +// +// A probe that can never pass costs the pod its Ready condition permanently. The +// headless Service sets PublishNotReadyAddresses, so in-cluster DNS still +// resolves — but anything that gates on readiness does not: an externally-facing +// load-balancer target group, a rollout waiting on Ready, an operator reading +// kubectl. A seed exists to be dialed, so silently never becoming Ready is the +// wrong failure to ship. // // Readiness is checked here where ValidateGatedSeidProbes skips it: during a hold // a failing readiness is correct and temporary, on a seed it is permanent. No-op diff --git a/internal/platform/platform.go b/internal/platform/platform.go index ad55437d..f929545e 100644 --- a/internal/platform/platform.go +++ b/internal/platform/platform.go @@ -227,7 +227,7 @@ func (c Config) NodepoolForMode(mode string) string { return c.NodepoolValidator case modeSeed: // No fallback to NodepoolName: that pool is sized for RPC-class nodes, so - // a 2Gi seed landing there costs an order of magnitude more than a seed is + // a seed landing there costs an order of magnitude more than a seed is // worth — and karpenter.sh/do-not-disrupt then pins that node against // consolidation for the seed's life. Callers rendering a seed pod must // reject an empty value rather than substitute one; see diff --git a/internal/sidecartransport/sidecartransport_test.go b/internal/sidecartransport/sidecartransport_test.go index b8218926..5ae205ad 100644 --- a/internal/sidecartransport/sidecartransport_test.go +++ b/internal/sidecartransport/sidecartransport_test.go @@ -92,4 +92,3 @@ func TestNew_DefaultTokenPathConstant(t *testing.T) { t.Errorf("default path drifted from K8s convention: %s", DefaultServiceAccountTokenPath) } } - diff --git a/internal/task/bootstrap_resources.go b/internal/task/bootstrap_resources.go index ce2433c7..ccd8d424 100644 --- a/internal/task/bootstrap_resources.go +++ b/internal/task/bootstrap_resources.go @@ -317,7 +317,7 @@ func bootstrapPVCClaimName(node *seiv1alpha1.SeiNode) string { // never reaches the bootstrap-Job path. That invariant is what keeps the default // arm from rendering a seed as a full node here — with RPC probes and a // cosmos-exporter — and it is also why buildBootstrapPodSpec does not call -// ValidateSeedProbes. Giving seed a snapshot source would break both silently. +// ValidateSeedProbes. TestSeedNeverBootstraps pins the invariant. func bootstrapNodeMode(node *seiv1alpha1.SeiNode) string { switch { case node.Spec.Archive != nil: From c583be5d9ab9cbd247707f1082c8f779f86b3b96 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Mon, 27 Jul 2026 12:53:37 -0700 Subject: [PATCH 4/6] fix(seed): require scheduling.nodepoolSeed at startup Consistency with the pools beside it: nodepoolArchive and nodepoolValidator are already required whether or not a cluster runs those modes. Treating the seed pool as optional was the exception, and it pushed the only enforcement into the render path, where a missing key surfaced as a stuck SeiNode rather than a startup error naming the field. Validate now rejects a config without it, so a cluster states its seed pool the same way it states the others. The render check stays as defence-in-depth alongside the images.kubeRBACProxy check it mirrors -- NodepoolForMode gives a seed no fallback, so an empty value there would silently schedule it onto the RPC-class default pool. Deploy ordering: an app-config file predating this key now fails the controller at startup, so platform adds scheduling.nodepoolSeed to each cell before the image rolls. Co-Authored-By: Claude Opus 5 (1M context) --- internal/noderesource/noderesource.go | 12 ++++++------ internal/platform/load_test.go | 20 ++++++++++++++++++++ internal/platform/platform.go | 19 +++++++++---------- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/internal/noderesource/noderesource.go b/internal/noderesource/noderesource.go index dac5c3ef..bad9f15c 100644 --- a/internal/noderesource/noderesource.go +++ b/internal/noderesource/noderesource.go @@ -484,12 +484,12 @@ func GenerateStatefulSet(node *seiv1alpha1.SeiNode, p PlatformConfig) (*appsv1.S if p.KubeRBACProxyImage == "" { return nil, fmt.Errorf("images.kubeRBACProxy is not configured in the app-config file") } - // Seeds need their own small pool. Required here rather than in - // Config.Validate so a cluster that runs no seeds keeps booting on an - // app-config file that predates the key — and so the alternative, silently - // scheduling a small seed onto the RPC-class default pool, is impossible. - if !servesSeidRPC(node) && p.NodepoolForMode(NodeMode(node)) == "" { - return nil, fmt.Errorf("scheduling.nodepoolSeed is not configured in the app-config file; a seed requires a dedicated small-instance nodepool") + // Unreachable in a running controller — Config.Validate requires + // scheduling.nodepoolSeed at startup. Kept for the same reason the check + // above is: NodepoolForMode gives a seed no fallback, so an empty value here + // would silently schedule it onto the RPC-class default pool. + if !servesSeidRPC(node) && p.NodepoolSeed == "" { + return nil, fmt.Errorf("scheduling.nodepoolSeed is not configured in the app-config file") } one := int32(1) labels := ResourceLabels(node) diff --git a/internal/platform/load_test.go b/internal/platform/load_test.go index 0afaaf1d..88eaa6d8 100644 --- a/internal/platform/load_test.go +++ b/internal/platform/load_test.go @@ -13,6 +13,7 @@ scheduling: nodepoolName: file-nodepool nodepoolArchive: file-nodepool-archive nodepoolValidator: file-nodepool-validator + nodepoolSeed: file-nodepool-seed tolerationKey: file-toleration serviceAccount: file-sa storage: @@ -142,6 +143,25 @@ func TestLoad_MissingNodepoolValidator_FailsValidate(t *testing.T) { } } +// scheduling.nodepoolSeed is required on the same terms as the archive and +// validator pools — whether or not the cluster runs seeds. NodepoolForMode gives +// a seed no fallback, so an unset key would otherwise put a small seed on the +// RPC-class default pool. +func TestLoad_MissingNodepoolSeed_FailsValidate(t *testing.T) { + setGatewayEnv(t) + body := strings.Replace(fullConfig, " nodepoolSeed: file-nodepool-seed\n", "", 1) + path := writeConfig(t, body) + t.Setenv(envControllerConfig, path) + + cfg, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "scheduling.nodepoolSeed") { + t.Fatalf("want Validate error naming scheduling.nodepoolSeed, got %v", err) + } +} + // A missing gateway env var fails Validate, naming the env var (gateway is still // env-sourced pending PLT-451). func TestLoad_MissingGateway_FailsValidate(t *testing.T) { diff --git a/internal/platform/platform.go b/internal/platform/platform.go index f929545e..ab6592cb 100644 --- a/internal/platform/platform.go +++ b/internal/platform/platform.go @@ -62,13 +62,12 @@ type Config struct { StorageSizeDefault string StorageSizeArchive string - // NodepoolSeed is required only by clusters that run seeds — absent from - // Validate's startup list so existing app-config files stay valid, and - // enforced instead where a seed pod is rendered. It has no fallback: see - // NodepoolForMode. + // NodepoolSeed is required, like the archive and validator pools, whether or + // not the cluster runs seeds. A seed must not inherit the RPC-class default + // pool, so NodepoolForMode gives it no fallback. // - // StorageSizeSeed is genuinely optional; defaultStorageSizeSeed is a correct - // value, not a placeholder. Read via SeedStorageSize. + // StorageSizeSeed is optional; defaultStorageSizeSeed is a correct value, not + // a placeholder. Read via SeedStorageSize. NodepoolSeed string StorageSizeSeed string @@ -149,10 +148,9 @@ type SchedulingConfig struct { NodepoolName string `json:"nodepoolName"` NodepoolArchive string `json:"nodepoolArchive"` NodepoolValidator string `json:"nodepoolValidator"` - // NodepoolSeed is optional; unset routes seeds to NodepoolName. - NodepoolSeed string `json:"nodepoolSeed"` - TolerationKey string `json:"tolerationKey"` - ServiceAccount string `json:"serviceAccount"` + NodepoolSeed string `json:"nodepoolSeed"` + TolerationKey string `json:"tolerationKey"` + ServiceAccount string `json:"serviceAccount"` } // StorageConfig holds the PVC storage classes and sizes for default and archive nodes. @@ -259,6 +257,7 @@ func (c Config) Validate() error { {"scheduling.nodepoolName", c.NodepoolName}, {"scheduling.nodepoolArchive", c.NodepoolArchive}, {"scheduling.nodepoolValidator", c.NodepoolValidator}, + {"scheduling.nodepoolSeed", c.NodepoolSeed}, {"scheduling.tolerationKey", c.TolerationKey}, {"scheduling.serviceAccount", c.ServiceAccount}, {"storage.classPerf", c.StorageClassPerf}, From c641723873ccd83f7b1921903d8723301d191742 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Mon, 27 Jul 2026 13:08:12 -0700 Subject: [PATCH 5/6] fix(seed): size the seed to RFC 006 Appendix A Back to 2Gi. 4Gi was sizing around seid's unbounded pre-auth handshake read, and buying that off here cost two of the RFC's own numbers: a 4Gi request==limit cannot schedule on a 4 GiB node once kube/system reservation is taken, so the pool provisions 8 GiB and three seeds land near $190/month against a $50-150 line. The exposure is real and now tracked as PLT-850 -- bound the handshake read to roughly the largest legitimate message. The burst is live memory, so no soft limit reclaims it and only absolute headroom would absorb it, which no figure in the RFC provides. Accepting the risk against a high-priority upstream fix keeps all four Appendix A figures consistent; when PLT-850 lands this footprint carries real headroom. Co-Authored-By: Claude Opus 5 (1M context) --- internal/noderesource/noderesource.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/internal/noderesource/noderesource.go b/internal/noderesource/noderesource.go index bad9f15c..6d4fcecd 100644 --- a/internal/noderesource/noderesource.go +++ b/internal/noderesource/noderesource.go @@ -71,7 +71,7 @@ const ( memValidator = "128Gi" // full r5.4xlarge envelope; provisioned on the next size up (r5.8xlarge) memRPCClass = "256Gi" // full r7i.8xlarge envelope (fullNode + replayer); provisioned on r7i.12xlarge memArchive = "512Gi" // full r7i.16xlarge envelope (verified live prod); provisioned on r7i.24xlarge - memSeed = "4Gi" // sized for the pre-auth handshake burst, see defaultNodeResourceProfiles + memSeed = "2Gi" // RFC 006 Appendix A; see defaultNodeResourceProfiles cpuValidator = "16" // r5.4xlarge vCPU; request-only, no CPU limit cpuRPCClass = "32" // r7i.8xlarge vCPU (fullNode + replayer); request-only, no CPU limit cpuArchive = "64" // r7i.16xlarge vCPU; request-only, no CPU limit @@ -407,19 +407,18 @@ var defaultNodeResourceProfiles = map[string]nodeResourceProfile{ roleArchive: {cpuRequest: cpuArchive, memory: memArchive}, // Peer discovery only: no block execution, no state commitment, no query // surface. The exception to the "request == the reference instance's full - // envelope" rule above — a seed is deliberately small. + // envelope" rule above — a seed is deliberately small, and this is RFC 006 + // Appendix A's figure, which also keeps it on the t-class instances and + // inside the cost line that RFC budgets. // - // Sized for the concurrent pre-auth handshake burst rather than steady state: - // the seed config's connection cap sets how many unauthenticated handshakes - // can be in flight at once, and each reads a peer-declared size bounded well - // above what a legitimate handshake needs. That burst is live memory, so only - // absolute headroom absorbs it, and it lands on top of the full cosmos app - // seid builds before dispatching on mode. RFC 006 Appendix A's primary-seed - // figure. Bounding that read upstream is what would let this shrink; the - // current numbers live in sei-config and sei-tendermint, not here. - // - // Note this exceeds what a 4 GiB node can schedule once kube/system - // reservation is taken, so the seed nodepool provisions the next size up. + // Accepted risk while PLT-850 is open: seid's pre-auth handshake read is + // bounded far above what a legitimate handshake needs, and the seed's + // connection cap scales it, so a remote unauthenticated flood can allocate + // well past this footprint. That memory is live, so GOMEMLIMIT cannot reclaim + // it and only absolute headroom would absorb it — which no figure in the RFC + // provides. Sizing around it would break the instance class and the budget, + // so the exposure is tracked upstream rather than bought off here. When + // PLT-850 lands this figure carries real headroom. roleSeed: {cpuRequest: cpuSeed, memory: memSeed}, } From c37f7cf804b63ed4f515069649d733ce415ce39b Mon Sep 17 00:00:00 2001 From: bdchatham Date: Mon, 27 Jul 2026 13:14:32 -0700 Subject: [PATCH 6/6] test: name the shared chainID and image in the node fixtures gofmt realigned two fields in testhelpers_test.go, which was enough for --new-from-patch to attribute the pre-existing goconst findings on those lines to this branch. Use the constants instead of silencing it: the const block already documented that defaultTestChainID has to match what these fixtures hardcode and that nothing enforced it, so this closes that gap and adds the image alongside. Co-Authored-By: Claude Opus 5 (1M context) --- internal/controller/node/reconciler_test.go | 11 ++++++----- internal/controller/node/testhelpers_test.go | 8 ++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/internal/controller/node/reconciler_test.go b/internal/controller/node/reconciler_test.go index 62a82b55..091c81ae 100644 --- a/internal/controller/node/reconciler_test.go +++ b/internal/controller/node/reconciler_test.go @@ -110,11 +110,12 @@ func getSeiNode(t *testing.T, ctx context.Context, c client.Client, name, namesp const ( testImageV2 = "ghcr.io/sei-protocol/seid:v2.0.0" testRevision = "rev-2" - // defaultTestChainID must match the chainID the snapshot/genesis fixtures - // hardcode in testhelpers_test.go (not enforced by the compiler). - defaultTestChainID = "sei-test" - atlantic2ChainID = "atlantic-2" - pacific1ChainID = "pacific-1" + // defaultTestChainID and defaultTestNodeImage are what the snapshot/genesis + // fixtures in testhelpers_test.go build with. + defaultTestChainID = "sei-test" + defaultTestNodeImage = "ghcr.io/sei-protocol/seid:latest" + atlantic2ChainID = "atlantic-2" + pacific1ChainID = "pacific-1" ) func TestNodeReconcile_NotFound(t *testing.T) { diff --git a/internal/controller/node/testhelpers_test.go b/internal/controller/node/testhelpers_test.go index 0b271e5f..f120e0fa 100644 --- a/internal/controller/node/testhelpers_test.go +++ b/internal/controller/node/testhelpers_test.go @@ -11,8 +11,8 @@ func newGenesisNode(name, namespace string) *seiv1alpha1.SeiNode { //nolint:unpa return &seiv1alpha1.SeiNode{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: seiv1alpha1.SeiNodeSpec{ - ChainID: "sei-test", - Image: "ghcr.io/sei-protocol/seid:latest", + ChainID: defaultTestChainID, + Image: defaultTestNodeImage, Validator: &seiv1alpha1.ValidatorSpec{}, Sidecar: &seiv1alpha1.SidecarConfig{Port: 7777}, }, @@ -23,8 +23,8 @@ func newSnapshotNode(name, namespace string) *seiv1alpha1.SeiNode { //nolint:unp return &seiv1alpha1.SeiNode{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: seiv1alpha1.SeiNodeSpec{ - ChainID: "sei-test", - Image: "ghcr.io/sei-protocol/seid:latest", + ChainID: defaultTestChainID, + Image: defaultTestNodeImage, FullNode: &seiv1alpha1.FullNodeSpec{ Snapshot: &seiv1alpha1.SnapshotSource{ S3: &seiv1alpha1.S3SnapshotSource{