diff --git a/api/v1alpha1/seed_types.go b/api/v1alpha1/seed_types.go new file mode 100644 index 00000000..1f487495 --- /dev/null +++ b/api/v1alpha1/seed_types.go @@ -0,0 +1,48 @@ +package v1alpha1 + +// SeedSpec configures a CometBFT seed node: a peer-discovery server running the +// P2P transport and PEX reactor only. It hands a dialing node a spread of peer +// addresses, then drops the connection. +// +// A seed joins no consensus, syncs no blocks, and serves no queries — seid binds +// no RPC, gRPC, REST or EVM listener in this mode. Hence no snapshot source +// (nothing to restore), no snapshotGeneration (no history to snapshot), and no +// signing key or operator keyring (it signs nothing). Its data volume holds the +// peer store plus the block and state DBs seid opens and never writes. +// +// Peers come from SeiNodeSpec.Peers as in every mode; a seed with none is a +// valid bootstrap root that others dial without dialing out itself. +// +// A seed reachable from outside the cluster needs three things this spec does +// not provide: SeiNodeSpec.ExternalAddress set to the published `host:port` +// (otherwise seid advertises its in-cluster listen address over PEX and every +// node that learns the seed by gossip gets an unroutable peer), a load balancer +// and DNS record for TCP 26656 that outlive pod and node churn, and an ingress +// allowance for that port. Nothing detects any of these missing: a seed reads +// Ready when its P2P port is bound, and nothing more. +// +// A seed serves no RPC, so it has no /status and no `catching_up`: sync +// freshness is not a property a seed has. Monitor it on P2P metrics +// (tendermint_p2p_*) at the metrics port instead. +type SeedSpec struct { + // NodeKey supplies the P2P identity (node_key.json) this seed presents. + // + // Required, where the validator's NodeKey is optional, because a seed's + // NodeID is published: operators dial `NodeID@host:port` and the + // secret-connection handshake verifies the pinned value, so a changed NodeID + // silently breaks every client carrying the old one. A Secret-sourced key + // survives pod recreation and PVC loss; one left to `seid init` regenerates + // onto the data volume. Carrying the identity to another cluster means + // replicating the Secret there — the controller reads it, it does not move + // it. Treat the NodeID as a one-way door. + // + // The Secret name is immutable, so rotating the identity means deleting and + // recreating the SeiNode — and announcing the new NodeID. There is no fast + // path for a leaked key: the old NodeID stays dialable until every client + // ships a new default, so a thief keeps impersonating a bootstrap anchor. + // + // Give each seed a distinct Secret. Nothing rejects two seeds sharing one, + // and they would present the same NodeID — collapsing the redundant anchors + // into a single entry in every dialer's peer store. + NodeKey NodeKeySource `json:"nodeKey"` +} diff --git a/api/v1alpha1/seinode_types.go b/api/v1alpha1/seinode_types.go index e8fe069f..44584c1d 100644 --- a/api/v1alpha1/seinode_types.go +++ b/api/v1alpha1/seinode_types.go @@ -7,9 +7,9 @@ import ( ) // SeiNodeSpec defines the desired state of a standalone Sei node. -// Exactly one mode sub-spec (fullNode, archive, replayer, validator) must be set; -// the populated field determines the node's operating mode. -// +kubebuilder:validation:XValidation:rule="(has(self.fullNode) ? 1 : 0) + (has(self.archive) ? 1 : 0) + (has(self.replayer) ? 1 : 0) + (has(self.validator) ? 1 : 0) == 1",message="exactly one of fullNode, archive, replayer, or validator must be set" +// Exactly one mode sub-spec (fullNode, archive, replayer, validator, seed) must +// be set; the populated field determines the node's operating mode. +// +kubebuilder:validation:XValidation:rule="(has(self.fullNode) ? 1 : 0) + (has(self.archive) ? 1 : 0) + (has(self.replayer) ? 1 : 0) + (has(self.validator) ? 1 : 0) + (has(self.seed) ? 1 : 0) == 1",message="exactly one of fullNode, archive, replayer, validator, or seed must be set" // +kubebuilder:validation:XValidation:rule="!has(self.replayer) || (has(self.peers) && size(self.peers) > 0)",message="peers is required when replayer mode is set" type SeiNodeSpec struct { // ChainID of the chain this node belongs to. @@ -73,6 +73,10 @@ type SeiNodeSpec struct { // +optional Validator *ValidatorSpec `json:"validator,omitempty"` + // Seed configures a peer-discovery seed node (P2P + PEX only). + // +optional + Seed *SeedSpec `json:"seed,omitempty"` + // ExternalAddress is the routable P2P host:port written into seid's // `p2p.external_address`. SeiNetwork-managed nodes get this stamped by the // SeiNetwork reconciler when TCP networking is enabled. Standalone SeiNodes @@ -138,6 +142,27 @@ func (s *SeiNodeSpec) SnapshotSource() *SnapshotSource { } } +// NodeKeySecret returns the Secret supplying this node's P2P identity +// (node_key.json), or nil when the node has none and `seid init` generates one +// onto the data volume. +// +// Two modes carry a node key on differing terms — a validator's is optional and +// paired with a signing key, a seed's is required and standalone — so each +// sub-spec declares its own field rather than hoisting one to SeiNodeSpec, out of +// reach of the validator's key-pairing CEL rules. This accessor resolves that +// split in one place, so callers needing only "which Secret holds the node key" +// stay mode-blind. Mirrors SnapshotSource. +func (s *SeiNodeSpec) NodeKeySecret() *SecretNodeKeySource { + switch { + case s.Validator != nil && s.Validator.NodeKey != nil: + return s.Validator.NodeKey.Secret + case s.Seed != nil: + return s.Seed.NodeKey.Secret + default: + return nil + } +} + // --------------------------------------------------------------------------- // Status // --------------------------------------------------------------------------- @@ -294,9 +319,9 @@ const ( // SeiNodes with spec.validator.signingKey. ConditionSigningKeyReady = "SigningKeyReady" - // ConditionNodeKeyReady indicates whether a referenced validator - // node-key Secret passes all validation requirements. Only set on - // SeiNodes with spec.validator.nodeKey. + // ConditionNodeKeyReady indicates whether a referenced node-key Secret + // passes all validation requirements. Only set on SeiNodes that source a + // node key from a Secret — spec.validator.nodeKey or spec.seed.nodeKey. ConditionNodeKeyReady = "NodeKeyReady" // ConditionOperatorKeyringReady indicates whether a referenced diff --git a/api/v1alpha1/validator_types.go b/api/v1alpha1/validator_types.go index 07ef6819..838742ac 100644 --- a/api/v1alpha1/validator_types.go +++ b/api/v1alpha1/validator_types.go @@ -129,8 +129,8 @@ type SecretSigningKeySource struct { SecretName string `json:"secretName"` } -// NodeKeySource declares where a validator's P2P node key -// (node_key.json) comes from. Exactly one variant must be set. +// NodeKeySource declares where a node's P2P node key (node_key.json) comes +// from. Exactly one variant must be set. Used by validator and seed modes. // // +kubebuilder:validation:XValidation:rule="(has(self.secret) ? 1 : 0) == 1",message="exactly one node key source must be set" type NodeKeySource struct { @@ -140,14 +140,16 @@ type NodeKeySource struct { Secret *SecretNodeKeySource `json:"secret,omitempty"` } -// SecretNodeKeySource references a Kubernetes Secret containing the -// validator's P2P node key. The Secret must contain a data key -// `node_key.json` holding the Tendermint node key, mounted read-only at -// $SEI_HOME/config/node_key.json. +// SecretNodeKeySource references a Kubernetes Secret containing a node's P2P +// node key. The Secret must contain a data key `node_key.json` holding the +// Tendermint node key, mounted read-only at $SEI_HOME/config/node_key.json. // -// node_key.json is identity-bearing but not slashing-relevant — losing it -// only costs the validator's accumulated peer-graph reputation, not stake. -// Treated as Secret-grade for handling consistency with SigningKey. +// How much the key matters depends on the mode that references it. For a +// validator it is identity-bearing but not slashing-relevant — losing it costs +// only accumulated peer-graph reputation, not stake — and is Secret-grade for +// handling consistency with SigningKey. For a seed the derived NodeID is +// published and dialed by strangers, which makes it a one-way door; see +// SeedSpec.NodeKey before treating it as low-stakes. type SecretNodeKeySource struct { // SecretName is the name of a Secret in the SeiNode's namespace. // The controller never creates, mutates, or deletes this Secret. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 1a926cd9..93f90a4c 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -846,6 +846,22 @@ func (in *SecretSigningKeySource) DeepCopy() *SecretSigningKeySource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SeedSpec) DeepCopyInto(out *SeedSpec) { + *out = *in + in.NodeKey.DeepCopyInto(&out.NodeKey) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SeedSpec. +func (in *SeedSpec) DeepCopy() *SeedSpec { + if in == nil { + return nil + } + out := new(SeedSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SeiNetwork) DeepCopyInto(out *SeiNetwork) { *out = *in @@ -1110,6 +1126,11 @@ func (in *SeiNodeSpec) DeepCopyInto(out *SeiNodeSpec) { *out = new(ValidatorSpec) (*in).DeepCopyInto(*out) } + if in.Seed != nil { + in, out := &in.Seed, &out.Seed + *out = new(SeedSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SeiNodeSpec. diff --git a/config/crd/sei.io_seinodes.yaml b/config/crd/sei.io_seinodes.yaml index 5646ce81..62ba6dc7 100644 --- a/config/crd/sei.io_seinodes.yaml +++ b/config/crd/sei.io_seinodes.yaml @@ -52,8 +52,8 @@ spec: spec: description: |- SeiNodeSpec defines the desired state of a standalone Sei node. - Exactly one mode sub-spec (fullNode, archive, replayer, validator) must be set; - the populated field determines the node's operating mode. + Exactly one mode sub-spec (fullNode, archive, replayer, validator, seed) must + be set; the populated field determines the node's operating mode. properties: archive: description: Archive configures an archive node with full history @@ -446,6 +446,60 @@ spec: required: - snapshot type: object + seed: + description: Seed configures a peer-discovery seed node (P2P + PEX + only). + properties: + nodeKey: + description: |- + NodeKey supplies the P2P identity (node_key.json) this seed presents. + + Required, where the validator's NodeKey is optional, because a seed's + NodeID is published: operators dial `NodeID@host:port` and the + secret-connection handshake verifies the pinned value, so a changed NodeID + silently breaks every client carrying the old one. A Secret-sourced key + survives pod recreation and PVC loss; one left to `seid init` regenerates + onto the data volume. Carrying the identity to another cluster means + replicating the Secret there — the controller reads it, it does not move + it. Treat the NodeID as a one-way door. + + The Secret name is immutable, so rotating the identity means deleting and + recreating the SeiNode — and announcing the new NodeID. There is no fast + path for a leaked key: the old NodeID stays dialable until every client + ships a new default, so a thief keeps impersonating a bootstrap anchor. + + Give each seed a distinct Secret. Nothing rejects two seeds sharing one, + and they would present the same NodeID — collapsing the redundant anchors + into a single entry in every dialer's peer store. + properties: + secret: + description: |- + Secret loads the node key from a Kubernetes Secret in the + SeiNode's namespace. + properties: + secretName: + description: |- + SecretName is the name of a Secret in the SeiNode's namespace. + The controller never creates, mutates, or deletes this Secret. + Immutable: re-pointing the node ID on a running validator costs + peer reputation and forces a peer-graph reset; force delete-and-recreate. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + x-kubernetes-validations: + - message: secretName is immutable + rule: self == oldSelf + required: + - secretName + type: object + type: object + x-kubernetes-validations: + - message: exactly one node key source must be set + rule: '(has(self.secret) ? 1 : 0) == 1' + required: + - nodeKey + type: object sidecar: description: Sidecar configures the sei-sidecar container. properties: @@ -827,10 +881,11 @@ spec: - image type: object x-kubernetes-validations: - - message: exactly one of fullNode, archive, replayer, or validator must - be set + - message: exactly one of fullNode, archive, replayer, validator, or seed + must be set rule: '(has(self.fullNode) ? 1 : 0) + (has(self.archive) ? 1 : 0) + - (has(self.replayer) ? 1 : 0) + (has(self.validator) ? 1 : 0) == 1' + (has(self.replayer) ? 1 : 0) + (has(self.validator) ? 1 : 0) + (has(self.seed) + ? 1 : 0) == 1' - message: peers is required when replayer mode is set rule: '!has(self.replayer) || (has(self.peers) && size(self.peers) > 0)' diff --git a/go.mod b/go.mod index 09dde97b..4751e6be 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 github.com/onsi/gomega v1.39.1 - github.com/sei-protocol/sei-config v0.0.23 + github.com/sei-protocol/sei-config v0.0.24 github.com/sei-protocol/seictl v0.0.68 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0 diff --git a/go.sum b/go.sum index 6e7af962..a52ebc07 100644 --- a/go.sum +++ b/go.sum @@ -172,6 +172,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sei-protocol/sei-config v0.0.23 h1:pGFxRnKoXZLK3Ew/Vd5lgC5RvH9Wo58NnL19CuMgFIk= github.com/sei-protocol/sei-config v0.0.23/go.mod h1:zcEdLzyIH2AyP0/QRBE3s4Y9eGn0C/qAUx1c4o4EROU= +github.com/sei-protocol/sei-config v0.0.24 h1:DqjehEjC24E7/vVQR6EDPjLOdOUOCegxFwZAeB7S23k= +github.com/sei-protocol/sei-config v0.0.24/go.mod h1:zcEdLzyIH2AyP0/QRBE3s4Y9eGn0C/qAUx1c4o4EROU= github.com/sei-protocol/seictl v0.0.68 h1:dCT94Ys4OjiPt5Y6TWqtgJ8GKTiWv7tN87D8HqIQxbk= github.com/sei-protocol/seictl v0.0.68/go.mod h1:kI3HIAIWzuJSme8LqJH1WZiITrSIx5yDtJJea07Xt8Y= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= diff --git a/internal/controller/node/controller.go b/internal/controller/node/controller.go index 23d44c35..4a419788 100644 --- a/internal/controller/node/controller.go +++ b/internal/controller/node/controller.go @@ -159,6 +159,15 @@ func (r *SeiNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct holdForWorkflow := node.Status.AdoptedWorkflow != nil && !adoptedWorkflowParkedFailed(node) if !holdInitialSTS && !holdForWorkflow { if err := r.reconcileStatefulSet(ctx, node); err != nil { + // This return precedes the status flush, so a render failure leaves no + // phase and no condition behind — the operator would see a SeiNode + // stuck with nothing explaining it. Emit an Event so the reason reaches + // `kubectl describe` rather than only the controller log. Reachable on + // operator-supplied app-config: infra fields load once at startup, so a + // seed applied before the controller restarts renders against the old + // config. + r.Recorder.Eventf(node, corev1.EventTypeWarning, "StatefulSetRenderFailed", + "Cannot render the StatefulSet: %v", err) return ctrl.Result{}, fmt.Errorf("reconciling statefulset: %w", err) } } diff --git a/internal/controller/node/envtest/seed_validation_test.go b/internal/controller/node/envtest/seed_validation_test.go new file mode 100644 index 00000000..9ee26a50 --- /dev/null +++ b/internal/controller/node/envtest/seed_validation_test.go @@ -0,0 +1,100 @@ +//go:build envtest + +package envtest_test + +import ( + "testing" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1" +) + +// seedNode returns a seed whose identity comes from the named Secret. An empty +// secretName yields a seed with no pinned identity, which admission must reject. +func seedNode(ns, name, secretName string) *seiv1alpha1.SeiNode { + spec := seiv1alpha1.SeiNodeSpec{ + ChainID: "envtest-1", + Image: "sei:latest", + Seed: &seiv1alpha1.SeedSpec{}, + } + if secretName != "" { + spec.Seed.NodeKey = seiv1alpha1.NodeKeySource{ + Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: secretName}, + } + } + return &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Spec: spec, + } +} + +func TestSeed_WithNodeKey_Accepted(t *testing.T) { + g := NewWithT(t) + ns := makeNamespace(t) + g.Expect(testCli.Create(testCtx, seedNode(ns, "seed-ok", "seed-0-node-key"))).To(Succeed()) +} + +// A seed's NodeID is published, so an unpinned identity is rejected at admission +// rather than left to regenerate onto the data volume at boot. +func TestSeed_WithoutNodeKey_Rejected(t *testing.T) { + g := NewWithT(t) + ns := makeNamespace(t) + + err := testCli.Create(testCtx, seedNode(ns, "seed-no-key", "")) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("nodeKey")) +} + +// The mode sub-specs stay mutually exclusive with seed added to the set. +func TestSeed_WithSecondMode_Rejected(t *testing.T) { + g := NewWithT(t) + ns := makeNamespace(t) + + node := seedNode(ns, "seed-plus-full", "seed-0-node-key") + node.Spec.FullNode = &seiv1alpha1.FullNodeSpec{} + + err := testCli.Create(testCtx, node) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("exactly one of")) +} + +// Widening the exactly-one rule must not reject the pre-existing modes. +// Replayer is omitted: CEL additionally requires peers for it. +func TestSeed_OtherModesStillAccepted(t *testing.T) { + ns := makeNamespace(t) + + modes := map[string]func(*seiv1alpha1.SeiNodeSpec){ + "fullnode": func(s *seiv1alpha1.SeiNodeSpec) { s.FullNode = &seiv1alpha1.FullNodeSpec{} }, + "archive": func(s *seiv1alpha1.SeiNodeSpec) { s.Archive = &seiv1alpha1.ArchiveSpec{} }, + "validator": func(s *seiv1alpha1.SeiNodeSpec) { s.Validator = &seiv1alpha1.ValidatorSpec{} }, + } + for name, set := range modes { + t.Run(name, func(t *testing.T) { + g := NewWithT(t) + spec := seiv1alpha1.SeiNodeSpec{ChainID: "envtest-1", Image: "sei:latest"} + set(&spec) + node := &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: "mode-" + name, Namespace: ns}, + Spec: spec, + } + g.Expect(testCli.Create(testCtx, node)).To(Succeed()) + }) + } +} + +// The Secret backs a published NodeID, so re-pointing it in place is refused; +// rotating the identity means delete-and-recreate. +func TestSeed_NodeKeySecretName_Immutable(t *testing.T) { + g := NewWithT(t) + ns := makeNamespace(t) + + node := seedNode(ns, "seed-immutable", "seed-0-node-key") + g.Expect(testCli.Create(testCtx, node)).To(Succeed()) + + node.Spec.Seed.NodeKey.Secret.SecretName = "seed-0-node-key-v2" + err := testCli.Update(testCtx, node) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("immutable")) +} diff --git a/internal/controller/node/plan_execution_test.go b/internal/controller/node/plan_execution_test.go index 32247f6e..6ce875eb 100644 --- a/internal/controller/node/plan_execution_test.go +++ b/internal/controller/node/plan_execution_test.go @@ -216,6 +216,21 @@ func genesisNode() *seiv1alpha1.SeiNode { } } +func seedNode() *seiv1alpha1.SeiNode { + return &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: "test-seed", Namespace: testNamespace, Generation: 1}, + Spec: seiv1alpha1.SeiNodeSpec{ + ChainID: testChainID, + Image: testImage, + Seed: &seiv1alpha1.SeedSpec{ + NodeKey: seiv1alpha1.NodeKeySource{ + Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: "test-seed-node-key"}, + }, + }, + }, + } +} + func snapshotterNode() *seiv1alpha1.SeiNode { return &seiv1alpha1.SeiNode{ ObjectMeta: metav1.ObjectMeta{Name: "test-node", Namespace: "default", Generation: 1}, @@ -852,6 +867,7 @@ func TestNeedsBootstrap(t *testing.T) { {"replayer without bootstrap image", replayerNode(), false}, {"full node without bootstrap image", snapshotNode(), false}, {"genesis node", genesisNode(), false}, + {"seed", seedNode(), false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -862,6 +878,22 @@ func TestNeedsBootstrap(t *testing.T) { } } +// A seed must never reach the bootstrap-Job path. bootstrapNodeMode has no seed +// arm and would render one as a full node — RPC probes, cosmos-exporter, and no +// ValidateSeedProbes call — and nothing there would fail to compile. The only +// thing preventing it is a nil SnapshotSource, so pin that rather than leave the +// invariant to a comment. +func TestSeedNeverBootstraps(t *testing.T) { + node := seedNode() + + if snap := node.Spec.SnapshotSource(); snap != nil { + t.Fatalf("a seed must have no snapshot source, got %+v", snap) + } + if planner.NeedsBootstrap(node) { + t.Error("a seed must never need a bootstrap Job") + } +} + // --- Bootstrap resource builder tests (task package) --- func TestTaskGenerateBootstrapJob(t *testing.T) { diff --git a/internal/controller/node/reconciler_test.go b/internal/controller/node/reconciler_test.go index 62a82b55..091c81ae 100644 --- a/internal/controller/node/reconciler_test.go +++ b/internal/controller/node/reconciler_test.go @@ -110,11 +110,12 @@ func getSeiNode(t *testing.T, ctx context.Context, c client.Client, name, namesp const ( testImageV2 = "ghcr.io/sei-protocol/seid:v2.0.0" testRevision = "rev-2" - // defaultTestChainID must match the chainID the snapshot/genesis fixtures - // hardcode in testhelpers_test.go (not enforced by the compiler). - defaultTestChainID = "sei-test" - atlantic2ChainID = "atlantic-2" - pacific1ChainID = "pacific-1" + // defaultTestChainID and defaultTestNodeImage are what the snapshot/genesis + // fixtures in testhelpers_test.go build with. + defaultTestChainID = "sei-test" + defaultTestNodeImage = "ghcr.io/sei-protocol/seid:latest" + atlantic2ChainID = "atlantic-2" + pacific1ChainID = "pacific-1" ) func TestNodeReconcile_NotFound(t *testing.T) { diff --git a/internal/controller/node/testhelpers_test.go b/internal/controller/node/testhelpers_test.go index aac0a273..f120e0fa 100644 --- a/internal/controller/node/testhelpers_test.go +++ b/internal/controller/node/testhelpers_test.go @@ -11,8 +11,8 @@ func newGenesisNode(name, namespace string) *seiv1alpha1.SeiNode { //nolint:unpa return &seiv1alpha1.SeiNode{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: seiv1alpha1.SeiNodeSpec{ - ChainID: "sei-test", - Image: "ghcr.io/sei-protocol/seid:latest", + ChainID: defaultTestChainID, + Image: defaultTestNodeImage, Validator: &seiv1alpha1.ValidatorSpec{}, Sidecar: &seiv1alpha1.SidecarConfig{Port: 7777}, }, @@ -23,8 +23,8 @@ func newSnapshotNode(name, namespace string) *seiv1alpha1.SeiNode { //nolint:unp return &seiv1alpha1.SeiNode{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: seiv1alpha1.SeiNodeSpec{ - ChainID: "sei-test", - Image: "ghcr.io/sei-protocol/seid:latest", + ChainID: defaultTestChainID, + Image: defaultTestNodeImage, FullNode: &seiv1alpha1.FullNodeSpec{ Snapshot: &seiv1alpha1.SnapshotSource{ S3: &seiv1alpha1.S3SnapshotSource{ diff --git a/internal/controller/node/workflow.go b/internal/controller/node/workflow.go index 6844d4f4..684357a0 100644 --- a/internal/controller/node/workflow.go +++ b/internal/controller/node/workflow.go @@ -622,6 +622,8 @@ func ineligibleWorkflowRole(node *seiv1alpha1.SeiNode) string { return "archive" case node.Spec.Replayer != nil: return "replayer" + case node.Spec.Seed != nil: + return "seed" // stores no chain state to re-bootstrap default: return "non-full" // unset/unknown mode — refused by the allowlist } diff --git a/internal/noderesource/noderesource.go b/internal/noderesource/noderesource.go index 467e1b76..6d4fcecd 100644 --- a/internal/noderesource/noderesource.go +++ b/internal/noderesource/noderesource.go @@ -62,6 +62,7 @@ const ( roleArchive = "archive" roleReplayer = "replayer" roleFullNode = "node" + roleSeed = "seed" // Per-mode seid-container footprint: memory and CPU request are both set to // the full sei-infra / verified-prod instance envelope for the role (memory @@ -70,9 +71,15 @@ const ( memValidator = "128Gi" // full r5.4xlarge envelope; provisioned on the next size up (r5.8xlarge) memRPCClass = "256Gi" // full r7i.8xlarge envelope (fullNode + replayer); provisioned on r7i.12xlarge memArchive = "512Gi" // full r7i.16xlarge envelope (verified live prod); provisioned on r7i.24xlarge + memSeed = "2Gi" // RFC 006 Appendix A; see defaultNodeResourceProfiles cpuValidator = "16" // r5.4xlarge vCPU; request-only, no CPU limit cpuRPCClass = "32" // r7i.8xlarge vCPU (fullNode + replayer); request-only, no CPU limit cpuArchive = "64" // r7i.16xlarge vCPU; request-only, no CPU limit + cpuSeed = "1" // PEX gossip is light and spiky; request-only, no CPU limit + + // goMemLimitPercentOfLimit is the seed's GOMEMLIMIT as a percentage of its + // memory limit; the remainder is the non-heap margin. See goMemLimitEnv. + goMemLimitPercentOfLimit = 87 dataDir = platform.DataDir @@ -273,6 +280,8 @@ func deriveRole(node *seiv1alpha1.SeiNode) string { return roleArchive case node.Spec.Replayer != nil: return roleReplayer + case node.Spec.Seed != nil: + return roleSeed default: return roleFullNode } @@ -286,6 +295,8 @@ func NodeMode(node *seiv1alpha1.SeiNode) string { return string(seiconfig.ModeArchive) case node.Spec.Validator != nil: return string(seiconfig.ModeValidator) + case node.Spec.Seed != nil: + return string(seiconfig.ModeSeed) case node.Spec.Replayer != nil: return string(seiconfig.ModeFull) default: @@ -293,6 +304,18 @@ func NodeMode(node *seiv1alpha1.SeiNode) string { } } +// servesSeidRPC reports whether seid binds its RPC and gRPC listeners. True for +// every mode but seed, which runs the P2P transport and PEX reactor only. +// +// It gates the two pod-spec decisions that poll those ports — the readiness probe +// target, and whether cosmos-exporter is attached. On a seed both wait forever: +// readiness never passes, and cosmos-exporter's wait loop never reaches its own +// listener, so its liveness probe kills it. Named for the property rather than +// the mode so the call sites read as intent. +func servesSeidRPC(node *seiv1alpha1.SeiNode) bool { + return node.Spec.Seed == nil +} + // NeedsLongStartup returns true when the node's bootstrap strategy involves // replaying blocks, requiring extended startup probe thresholds. func NeedsLongStartup(node *seiv1alpha1.SeiNode) bool { @@ -318,6 +341,11 @@ func DefaultStorageForMode(mode string, p PlatformConfig) (storageClass string, return p.StorageClassArchive, p.StorageSizeArchive case string(seiconfig.ModeFull), string(seiconfig.ModeValidator): return p.StorageClassPerf, p.StorageSizeDefault + case string(seiconfig.ModeSeed): + // A seed stores a peer store and two DBs it never writes, so it needs + // orders of magnitude less than StorageSizeDefault (sized for a chain's + // full state) and no performance class. + return p.StorageClassDefault, p.SeedStorageSize() default: return p.StorageClassDefault, p.StorageSizeDefault } @@ -377,6 +405,21 @@ var defaultNodeResourceProfiles = map[string]nodeResourceProfile{ // Full r7i.16xlarge envelope — the live-prod snapshotter shape (the sei-infra // TF's m7i.8xlarge is stale). roleArchive: {cpuRequest: cpuArchive, memory: memArchive}, + // Peer discovery only: no block execution, no state commitment, no query + // surface. The exception to the "request == the reference instance's full + // envelope" rule above — a seed is deliberately small, and this is RFC 006 + // Appendix A's figure, which also keeps it on the t-class instances and + // inside the cost line that RFC budgets. + // + // Accepted risk while PLT-850 is open: seid's pre-auth handshake read is + // bounded far above what a legitimate handshake needs, and the seed's + // connection cap scales it, so a remote unauthenticated flood can allocate + // well past this footprint. That memory is live, so GOMEMLIMIT cannot reclaim + // it and only absolute headroom would absorb it — which no figure in the RFC + // provides. Sizing around it would break the instance class and the budget, + // so the exposure is tracked upstream rather than bought off here. When + // PLT-850 lands this figure carries real headroom. + roleSeed: {cpuRequest: cpuSeed, memory: memSeed}, } // overrideForRole returns the app-config resource override for a node's mode. @@ -388,6 +431,8 @@ func overrideForRole(role string, p PlatformConfig) platform.ResourceOverride { return p.NodeResourcesArchive case roleReplayer: return p.NodeResourcesReplayer + case roleSeed: + return p.NodeResourcesSeed default: return p.NodeResourcesNode } @@ -438,6 +483,13 @@ func GenerateStatefulSet(node *seiv1alpha1.SeiNode, p PlatformConfig) (*appsv1.S if p.KubeRBACProxyImage == "" { return nil, fmt.Errorf("images.kubeRBACProxy is not configured in the app-config file") } + // Unreachable in a running controller — Config.Validate requires + // scheduling.nodepoolSeed at startup. Kept for the same reason the check + // above is: NodepoolForMode gives a seed no fallback, so an empty value here + // would silently schedule it onto the RPC-class default pool. + if !servesSeidRPC(node) && p.NodepoolSeed == "" { + return nil, fmt.Errorf("scheduling.nodepoolSeed is not configured in the app-config file") + } one := int32(1) labels := ResourceLabels(node) podSpec, err := buildNodePodSpec(node, p) @@ -622,7 +674,7 @@ func ServicePorts() []corev1.ServicePort { ports := make([]corev1.ServicePort, len(np)) for i, p := range np { ports[i] = corev1.ServicePort{Name: p.Name, Port: p.Port, TargetPort: intstr.FromInt32(p.Port), Protocol: corev1.ProtocolTCP} - if p.Name == "grpc" { + if p.Name == seiconfig.PortNameGRPC { h2c := "kubernetes.io/h2c" ports[i].AppProtocol = &h2c } @@ -723,10 +775,6 @@ func buildNodePodSpec(node *seiv1alpha1.SeiNode, p PlatformConfig) (corev1.PodSp buildSidecarContainer(node, p), buildRBACProxyContainer(node, p), } - ceContainer, err := buildCosmosExporterContainer(p) - if err != nil { - return corev1.PodSpec{}, err - } seidContainer := buildSidecarMainContainer(node, p) // Refuse to render a gated seid container that carries an RPC-based // liveness/startup probe: a held seid answers neither port, so such a probe @@ -734,9 +782,21 @@ func buildNodePodSpec(node *seiv1alpha1.SeiNode, p PlatformConfig) (corev1.PodSp if err := ValidateGatedSeidProbes(seidContainer); err != nil { return corev1.PodSpec{}, err } - spec.Containers = []corev1.Container{ - seidContainer, - ceContainer, + // A seed's readiness must not depend on seid's RPC/gRPC, which it never binds. + if err := ValidateSeedProbes(node, seidContainer); err != nil { + return corev1.PodSpec{}, err + } + spec.Containers = []corev1.Container{seidContainer} + + // cosmos-exporter scrapes staking and validator state over seid's gRPC. A + // seed serves neither, so attaching it there would leave it blocked in its + // wait loop until its own liveness probe killed it. + if servesSeidRPC(node) { + ceContainer, err := buildCosmosExporterContainer(p) + if err != nil { + return corev1.PodSpec{}, err + } + spec.Containers = append(spec.Containers, ceContainer) } return spec, nil @@ -817,6 +877,9 @@ func buildSidecarMainContainer(node *seiv1alpha1.SeiNode, p PlatformConfig) core container := buildNodeMainContainer(node) container.Command, container.Args = sidecarWaitCommand(node) container.Resources = ResourcesForNode(node, p) + if env, ok := goMemLimitEnv(node, container.Resources); ok { + container.Env = append(container.Env, env) + } // Sidecar binds loopback; gate startup on the proxy's /v0/healthz // (a bypass path that forwards through to the sidecar). container.StartupProbe = &corev1.Probe{ @@ -828,7 +891,80 @@ func buildSidecarMainContainer(node *seiv1alpha1.SeiNode, p PlatformConfig) core PeriodSeconds: 5, FailureThreshold: 86400, } - container.ReadinessProbe = &corev1.Probe{ + container.ReadinessProbe = readinessProbeForNode(node) + return container +} + +// goMemLimitEnv returns the GOMEMLIMIT env var for a seed's seid container. +// +// The Go runtime targets a heap roughly twice the live set and cannot see the +// cgroup limit, so with memory request==limit reclaimable growth becomes an +// OOMKill rather than a throttle. A soft limit below the hard cap makes the GC +// spend CPU to stay resident instead. +// +// What it does and does not cover: it governs churn — buffers allocated and +// freed at rate, which GOGC alone would let balloon toward twice the live set. +// It cannot shrink memory that is genuinely live, and it does not account for +// non-heap RSS at all: thread stacks, cgo allocations, mmap'd memIAVL pages, or +// the kernel socket buffers a thousand connections charge to the cgroup. The +// concurrent pre-auth handshake burst that sizes this mode is live memory, so +// absolute headroom is what defends it — not this limit. The margin below the +// cap exists for the non-heap share. +// +// Scoped to seed as a matter of caution rather than principle: GC slack is +// proportional to the live set, so any mode with request==limit would benefit. +// The other modes are tuned and prod-proven, and changing their runtime +// behaviour is not this change's business. +// +// Derived from the resolved limit, so an app-config resources.seed override +// keeps it consistent. Reports false when no memory limit is set, or when the +// limit is small enough that the computed soft cap would be useless. +func goMemLimitEnv(node *seiv1alpha1.SeiNode, res corev1.ResourceRequirements) (corev1.EnvVar, bool) { + if servesSeidRPC(node) { + return corev1.EnvVar{}, false + } + lim, ok := res.Limits[corev1.ResourceMemory] + if !ok { + return corev1.EnvVar{}, false + } + // Multiply first: the truncation from dividing first is harmless but the + // product cannot overflow int64 at any schedulable memory limit. + soft := lim.Value() * goMemLimitPercentOfLimit / 100 + if soft <= 0 { + return corev1.EnvVar{}, false + } + return corev1.EnvVar{Name: "GOMEMLIMIT", Value: fmt.Sprintf("%dB", soft)}, true +} + +// readinessProbeForNode returns the seid readiness probe for the node's mode. +// +// Chain-following modes gate on seid's /lag_status, which reports sync distance +// — the meaningful "ready to serve" signal. A seed serves no RPC, so its only +// in-band signal is that the P2P transport is bound: it distinguishes a crashed +// seid or a failed bind from a live one, and nothing more. +// +// Two limits an operator must know. It proves nothing about EXTERNAL +// reachability — a seed with no DNS record, no load balancer, or a closed +// security group reads Ready while no stranger can dial it. And each probe +// completes a TCP connect without the secret handshake, so it increments +// tendermint_p2p_new_connections{direction="in"} and books a handshake failure +// roughly 6/min; alerts on those series need that baseline subtracted, and +// inbound connections alone do not prove a seed is publicly reachable. +func readinessProbeForNode(node *seiv1alpha1.SeiNode) *corev1.Probe { + if !servesSeidRPC(node) { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt32(seiconfig.PortP2P), + }, + }, + InitialDelaySeconds: 30, + PeriodSeconds: 10, + FailureThreshold: 3, + TimeoutSeconds: 5, + } + } + return &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/lag_status", @@ -840,7 +976,6 @@ func buildSidecarMainContainer(node *seiv1alpha1.SeiNode, p PlatformConfig) core FailureThreshold: 3, TimeoutSeconds: 5, } - return container } // defaultCosmosExporterResources: no CPU limit — cosmos-exporter calls @@ -1189,11 +1324,11 @@ func nodeKeyMounts(node *seiv1alpha1.SeiNode) []corev1.VolumeMount { }} } +// nodeKeySecretSource returns the Secret holding this node's P2P identity, from +// whichever mode sub-spec carries one (validator or seed). Mode-blind by +// delegation, so the node-key volume and mount below serve both. func nodeKeySecretSource(node *seiv1alpha1.SeiNode) *seiv1alpha1.SecretNodeKeySource { - if node.Spec.Validator == nil || node.Spec.Validator.NodeKey == nil { - return nil - } - return node.Spec.Validator.NodeKey.Secret + return node.Spec.NodeKeySecret() } // operatorKeyringVolumes projects the operator-keyring Secret as a directory diff --git a/internal/noderesource/noderesource_test.go b/internal/noderesource/noderesource_test.go index 8c8977d6..8e6be5e7 100644 --- a/internal/noderesource/noderesource_test.go +++ b/internal/noderesource/noderesource_test.go @@ -841,6 +841,12 @@ func nodeForRole(role string) *seiv1alpha1.SeiNode { spec.Archive = &seiv1alpha1.ArchiveSpec{} case roleReplayer: spec.Replayer = &seiv1alpha1.ReplayerSpec{} + case roleSeed: + spec.Seed = &seiv1alpha1.SeedSpec{ + NodeKey: seiv1alpha1.NodeKeySource{ + Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: "seed-node-key"}, + }, + } default: spec.FullNode = &seiv1alpha1.FullNodeSpec{} } @@ -861,6 +867,7 @@ func TestResourcesForNode_DefaultsPerMode(t *testing.T) { {"", cpuRPCClass, memRPCClass}, // fullNode / rpc {roleReplayer, cpuRPCClass, memRPCClass}, {roleArchive, cpuArchive, memArchive}, + {roleSeed, cpuSeed, memSeed}, } for _, tc := range cases { t.Run("mode="+tc.role, func(t *testing.T) { @@ -1645,7 +1652,7 @@ func TestGenerateStatefulSet_SeidMount_PassesGuard(t *testing.T) { // --- Cosmos exporter --- -func TestCosmosExporter_AlwaysPresent(t *testing.T) { +func TestCosmosExporter_PresentOnModesServingGRPC(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("ce-0", "default") @@ -1656,6 +1663,32 @@ func TestCosmosExporter_AlwaysPresent(t *testing.T) { g.Expect(sts.Spec.Template.Spec.Containers).To(HaveLen(2)) } +// cosmos-exporter's wait loop blocks until seid's gRPC accepts, and a seed never +// binds it — so attaching the container would leave it waiting until its own +// liveness probe killed it, in CrashLoopBackOff for the pod's life. +func TestCosmosExporter_AbsentOnSeed(t *testing.T) { + g := NewWithT(t) + + sts := mustGenerateStatefulSet(t, nodeForRole(roleSeed), platformtest.Config()) + + containers := sts.Spec.Template.Spec.Containers + g.Expect(findContainer(containers, containerNameCosmosExporter)).To(BeNil()) + g.Expect(containers).To(HaveLen(1)) + g.Expect(containers[0].Name).To(Equal(containerNameSeid)) +} + +// A seed needs the peer store and two DBs it never writes — not the full-state +// volume StorageSizeDefault sizes for. +func TestDefaultStorageForMode_Seed(t *testing.T) { + g := NewWithT(t) + cfg := platformtest.Config() + + sc, size := DefaultStorageForMode(string(seiconfig.ModeSeed), cfg) + g.Expect(sc).To(Equal(cfg.StorageClassDefault)) + g.Expect(size).To(Equal(cfg.SeedStorageSize())) + g.Expect(size).NotTo(Equal(cfg.StorageSizeDefault)) +} + func TestCosmosExporter_DefaultImage(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("ce-0", "default") @@ -1927,3 +1960,117 @@ func TestCosmosExporter_NonRootSecurityContext(t *testing.T) { g.Expect(*ce.SecurityContext.RunAsNonRoot).To(BeTrue()) g.Expect(*ce.SecurityContext.RunAsUser).To(Equal(int64(65532))) } + +// --- Seed identity --- + +// A seed's NodeID is published, so it must come from the Secret rather than be +// regenerated onto the data volume. The mount uses subPath, so kubelet never +// hot-swaps the identity under a running seid. +func TestSeed_NodeKeySecretMountedOnPodTemplate(t *testing.T) { + g := NewWithT(t) + + sts := mustGenerateStatefulSet(t, nodeForRole(roleSeed), platformtest.Config()) + + vol := findVolume(sts.Spec.Template.Spec.Volumes, nodeKeyVolumeName) + g.Expect(vol).NotTo(BeNil(), "seed node-key volume must be present") + g.Expect(vol.Secret).NotTo(BeNil()) + g.Expect(vol.Secret.SecretName).To(Equal("seed-node-key")) + g.Expect(vol.Secret.Items[0].Key).To(Equal(nodeKeyDataKey)) + + seid := findContainer(sts.Spec.Template.Spec.Containers, containerNameSeid) + g.Expect(seid).NotTo(BeNil()) + mount := findVolumeMount(seid.VolumeMounts, nodeKeyVolumeName) + g.Expect(mount).NotTo(BeNil(), "seed seid container must mount its node key") + g.Expect(mount.SubPath).To(Equal(nodeKeyDataKey)) + g.Expect(mount.ReadOnly).To(BeTrue()) +} + +// deriveRole feeds the sei.io/role pod label the platform PodMonitor lifts into +// sei_role, and NodeMode drives storage and nodepool selection. Both default to +// full-node values, so a missing seed arm is silently wrong rather than a +// compile error. +func TestSeed_RoleAndModeAreNotFullNodeDefaults(t *testing.T) { + g := NewWithT(t) + node := nodeForRole(roleSeed) + + g.Expect(deriveRole(node)).To(Equal(roleSeed)) + g.Expect(NodeMode(node)).To(Equal(string(seiconfig.ModeSeed))) + g.Expect(ResourceLabels(node)).To(HaveKeyWithValue(roleLabel, roleSeed)) + // A seed produces no snapshots, so the publish selector must not match it. + g.Expect(ResourceLabels(node)).NotTo(HaveKey(snapshotPublishLabel)) + // Nothing to replay: startup thresholds stay at the short default. + g.Expect(NeedsLongStartup(node)).To(BeFalse()) +} + +// Memory request==limit makes an allocation burst an OOMKill, and the Go GC +// cannot see the cgroup limit. A soft limit below the cap trades CPU to stay +// resident. Seed-only: the r-class modes carry tens of GiB of absolute headroom. +func TestSeed_GoMemLimitBelowMemoryLimit(t *testing.T) { + g := NewWithT(t) + cfg := platformtest.Config() + + sts := mustGenerateStatefulSet(t, nodeForRole(roleSeed), cfg) + seid := findContainer(sts.Spec.Template.Spec.Containers, containerNameSeid) + g.Expect(seid).NotTo(BeNil()) + + env := findEnvVar(seid.Env, "GOMEMLIMIT") + g.Expect(env).NotTo(BeNil(), "a seed must carry GOMEMLIMIT") + + limit := seid.Resources.Limits[corev1.ResourceMemory] + soft, err := resource.ParseQuantity(strings.TrimSuffix(env.Value, "B")) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(soft.Value()).To(BeNumerically("<", limit.Value())) + g.Expect(soft.Value()).To(BeNumerically(">", limit.Value()/2), "soft limit should not waste the footprint") +} + +// Chain-following modes keep the runtime's own heap sizing; their headroom is +// large enough that GC slack cannot cross the cap. +func TestNonSeed_HasNoGoMemLimit(t *testing.T) { + g := NewWithT(t) + + sts := mustGenerateStatefulSet(t, newSnapshotNode("rpc-0", testNamespace), platformtest.Config()) + seid := findContainer(sts.Spec.Template.Spec.Containers, containerNameSeid) + + g.Expect(findEnvVar(seid.Env, "GOMEMLIMIT")).To(BeNil()) +} + +func findEnvVar(env []corev1.EnvVar, name string) *corev1.EnvVar { + for i := range env { + if env[i].Name == name { + return &env[i] + } + } + return nil +} + +// The alternative to failing closed is silently scheduling a small seed onto the +// RPC-class default pool, which costs an order of magnitude more than a seed is +// worth. Clusters running no seeds are unaffected — the check is per-render, not +// at startup. +func TestSeed_RenderFailsWithoutSeedNodepool(t *testing.T) { + g := NewWithT(t) + cfg := platformtest.Config() + cfg.NodepoolSeed = "" + + _, err := GenerateStatefulSet(nodeForRole(roleSeed), cfg) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("scheduling.nodepoolSeed")) + + // Every other mode still renders on the same config — the requirement is + // per-render, so a cluster running no seeds is unaffected. + _, err = GenerateStatefulSet(newSnapshotNode("rpc-0", testNamespace), cfg) + g.Expect(err).NotTo(HaveOccurred()) +} + +// A configured seed pool lands on the pod, not the default pool. +func TestSeed_SchedulesOnSeedNodepool(t *testing.T) { + g := NewWithT(t) + cfg := platformtest.Config() + + sts := mustGenerateStatefulSet(t, nodeForRole(roleSeed), cfg) + spec := sts.Spec.Template.Spec + + terms := spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + g.Expect(terms[0].MatchExpressions[0].Values).To(ConsistOf(cfg.NodepoolSeed)) + g.Expect(spec.Tolerations[0].Value).To(Equal(cfg.NodepoolSeed)) +} diff --git a/internal/noderesource/probe_guard.go b/internal/noderesource/probe_guard.go index 5a93bc98..c2477fbd 100644 --- a/internal/noderesource/probe_guard.go +++ b/internal/noderesource/probe_guard.go @@ -5,6 +5,8 @@ import ( seiconfig "github.com/sei-protocol/sei-config" corev1 "k8s.io/api/core/v1" + + seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1" ) // ValidateGatedSeidProbes enforces the probe contract the workflow hold leans @@ -31,6 +33,40 @@ func ValidateGatedSeidProbes(c corev1.Container) error { return nil } +// ValidateSeedProbes is the seed counterpart of the gated-seid contract: a seed's +// seid container must carry no probe against seid's RPC or gRPC port, since seed +// mode binds neither for the pod's whole life. +// +// A probe that can never pass costs the pod its Ready condition permanently. The +// headless Service sets PublishNotReadyAddresses, so in-cluster DNS still +// resolves — but anything that gates on readiness does not: an externally-facing +// load-balancer target group, a rollout waiting on Ready, an operator reading +// kubectl. A seed exists to be dialed, so silently never becoming Ready is the +// wrong failure to ship. +// +// Readiness is checked here where ValidateGatedSeidProbes skips it: during a hold +// a failing readiness is correct and temporary, on a seed it is permanent. No-op +// for every other mode. +func ValidateSeedProbes(node *seiv1alpha1.SeiNode, c corev1.Container) error { + if servesSeidRPC(node) { + return nil + } + probes := []struct { + name string + probe *corev1.Probe + }{ + {"readiness", c.ReadinessProbe}, + {"liveness", c.LivenessProbe}, + {"startup", c.StartupProbe}, + } + for _, p := range probes { + if probeTargetsSeidRPC(p.probe) { + return fmt.Errorf("seed seid container %q %s probe must not target seid's RPC/gRPC port; seed mode binds neither", c.Name, p.name) + } + } + return nil +} + // probeTargetsSeidRPC reports whether a probe polls seid's RPC or gRPC port // (via HTTPGet or TCPSocket). A nil probe or one targeting any other port // (e.g. the RBAC-proxy healthz port) returns false. diff --git a/internal/noderesource/probe_guard_test.go b/internal/noderesource/probe_guard_test.go index f8189bbd..641a5521 100644 --- a/internal/noderesource/probe_guard_test.go +++ b/internal/noderesource/probe_guard_test.go @@ -54,3 +54,59 @@ func TestValidateGatedSeidProbes_RejectsRPCProbes(t *testing.T) { }} g.Expect(ValidateGatedSeidProbes(withGRPCStartup)).To(HaveOccurred()) } + +// A seed serves only P2P, so its readiness gates on the transport being bound — +// the failure most worth catching, since a loopback-bound seed boots clean and +// accepts nothing. +func TestSeedContainer_ReadinessProbesP2PNotRPC(t *testing.T) { + g := NewWithT(t) + c := buildSidecarMainContainer(nodeForRole(roleSeed), platformtest.Config()) + + g.Expect(c.ReadinessProbe).NotTo(BeNil()) + g.Expect(c.ReadinessProbe.HTTPGet).To(BeNil(), "a seed serves no RPC to probe over HTTP") + g.Expect(c.ReadinessProbe.TCPSocket).NotTo(BeNil()) + g.Expect(c.ReadinessProbe.TCPSocket.Port.IntVal).To(Equal(seiconfig.PortP2P)) + g.Expect(ValidateSeedProbes(nodeForRole(roleSeed), c)).To(Succeed()) +} + +// Chain-following modes keep the /lag_status readiness gate, which reports sync +// distance rather than mere liveness. +func TestNonSeedContainer_ReadinessProbesLagStatus(t *testing.T) { + g := NewWithT(t) + node := newGenesisNode("mynet-0", "default") + c := buildSidecarMainContainer(node, platformtest.Config()) + + g.Expect(c.ReadinessProbe.HTTPGet).NotTo(BeNil()) + g.Expect(c.ReadinessProbe.HTTPGet.Path).To(Equal("/lag_status")) + g.Expect(c.ReadinessProbe.HTTPGet.Port.IntVal).To(Equal(seiconfig.PortRPC)) + // The seed guard is a no-op for modes that do bind RPC. + g.Expect(ValidateSeedProbes(node, c)).To(Succeed()) +} + +// A permanently-failing readiness probe would hold a seed out of Service +// endpoints and any NLB target group fronting it, so the render fails closed. +func TestValidateSeedProbes_RejectsRPCProbes(t *testing.T) { + seed := nodeForRole(roleSeed) + base := buildSidecarMainContainer(seed, platformtest.Config()) + + rpcProbe := func(port int32) *corev1.Probe { + return &corev1.Probe{ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt32(port)}, + }} + } + + cases := map[string]func(*corev1.Container){ + "readiness on RPC": func(c *corev1.Container) { c.ReadinessProbe = rpcProbe(seiconfig.PortRPC) }, + "liveness on RPC": func(c *corev1.Container) { c.LivenessProbe = rpcProbe(seiconfig.PortRPC) }, + "startup on gRPC": func(c *corev1.Container) { c.StartupProbe = rpcProbe(seiconfig.PortGRPC) }, + "readiness on gRPC": func(c *corev1.Container) { c.ReadinessProbe = rpcProbe(seiconfig.PortGRPC) }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + g := NewWithT(t) + c := base + mutate(&c) + g.Expect(ValidateSeedProbes(seed, c)).NotTo(Succeed()) + }) + } +} diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 376c3efc..fa78c3b0 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -310,6 +310,8 @@ func (r *NodeResolver) plannerForMode(node *seiv1alpha1.SeiNode) (NodePlanner, e return &replayerPlanner{platform: r.Platform}, nil case node.Spec.Validator != nil: return &validatorPlanner{platform: r.Platform}, nil + case node.Spec.Seed != nil: + return &seedPlanner{platform: r.Platform}, nil default: return nil, fmt.Errorf("no mode sub-spec set on SeiNode %s/%s", node.Namespace, node.Name) } @@ -379,12 +381,13 @@ func validateSigningKeyParams(node *seiv1alpha1.SeiNode) any { } } +// needsValidateNodeKey reports whether the node sources its P2P identity from a +// Secret and so needs the pre-flight validation task. Mode-blind: it asks the +// spec which Secret holds the node key, not which mode is set, so validator and +// seed are covered by the same gate. func needsValidateNodeKey(node *seiv1alpha1.SeiNode) bool { - if node.Spec.Validator == nil || node.Spec.Validator.NodeKey == nil { - return false - } - return node.Spec.Validator.NodeKey.Secret != nil && - node.Spec.Validator.NodeKey.Secret.SecretName != "" + s := node.Spec.NodeKeySecret() + return s != nil && s.SecretName != "" } func validateNodeKeyParams(node *seiv1alpha1.SeiNode) any { @@ -392,7 +395,7 @@ func validateNodeKeyParams(node *seiv1alpha1.SeiNode) any { return nil } return &task.ValidateNodeKeyParams{ - SecretName: node.Spec.Validator.NodeKey.Secret.SecretName, + SecretName: node.Spec.NodeKeySecret().SecretName, Namespace: node.Namespace, } } diff --git a/internal/planner/seed.go b/internal/planner/seed.go new file mode 100644 index 00000000..8f67ce6e --- /dev/null +++ b/internal/planner/seed.go @@ -0,0 +1,78 @@ +package planner + +import ( + "fmt" + + seiconfig "github.com/sei-protocol/sei-config" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1" + "github.com/sei-protocol/sei-k8s-controller/internal/platform" + "github.com/sei-protocol/sei-k8s-controller/internal/task" +) + +type seedPlanner struct { + platform platform.Config +} + +func (p *seedPlanner) Mode() string { return string(seiconfig.ModeSeed) } + +func (p *seedPlanner) Validate(node *seiv1alpha1.SeiNode) error { + if node.Spec.Seed == nil { + return fmt.Errorf("seed sub-spec is nil") + } + // CEL already requires the field and a Secret variant, but an in-memory + // spec that never went through admission can still reach here. A seed with + // no pinned identity would silently generate a fresh NodeID on the data + // volume, so fail loudly rather than serve an unstable address. + if s := node.Spec.NodeKeySecret(); s == nil || s.SecretName == "" { + return fmt.Errorf("seed: nodeKey.secret.secretName is required — a seed's NodeID is published and must not be regenerated") + } + return nil +} + +// BuildPlan drives a seed through the plain genesis progression. A seed never +// bootstraps from a snapshot — it stores no chain state to restore — so the nil +// SnapshotSource selects the genesis task sequence and, via +// needsStateSyncWitnesses, omits ConfigureStateSync entirely. +func (p *seedPlanner) BuildPlan(node *seiv1alpha1.SeiNode) (*seiv1alpha1.TaskPlan, error) { + if node.Status.Phase == seiv1alpha1.PhaseRunning { + return p.buildRunningPlan(node) + } + // No per-mode controller overrides: every seed-specific config key comes + // from sei-config's applySeedOverrides, resolved sidecar-side off + // ConfigIntent.Mode. commonOverrides still supplies the shared keys. + intent := &seiconfig.ConfigIntent{ + Mode: seiconfig.ModeSeed, + Overrides: mergeOverrides(commonOverrides(node), node.Spec.Overrides), + } + return buildBasePlan(node, nil, intent) +} + +// buildRunningPlan returns the update plan for a Running seed. Same shape as +// the other modes, with the node-key gate ahead of any StatefulSet mutation so +// a missing or malformed identity Secret fails controller-side rather than as a +// kubelet volume-mount error on the recreated pod. +func (p *seedPlanner) buildRunningPlan(node *seiv1alpha1.SeiNode) (*seiv1alpha1.TaskPlan, error) { + if imageDrifted(node) || sidecarImageDrifted(node, p.platform) { + setNodeUpdateCondition(node, metav1.ConditionTrue, "UpdateStarted", imageDriftMessage(node, p.platform)) + prog := make([]string, 0, 8) + if needsValidateNodeKey(node) { + prog = append(prog, task.TaskTypeValidateNodeKey) + } + prog = append(prog, + task.TaskTypeApplyStatefulSet, + task.TaskTypeApplyService, + TaskConfigPatch, + TaskConfigValidate, + task.TaskTypeReplacePod, + task.TaskTypeObserveImage, + TaskMarkReady, + ) + return assembleUpdatePlan(node, prog, p2pConfigPatch(node)) + } + if sidecarNeedsReapproval(node) { + return buildMarkReadyPlan(node) + } + return nil, nil +} diff --git a/internal/planner/seed_test.go b/internal/planner/seed_test.go new file mode 100644 index 00000000..8e64444e --- /dev/null +++ b/internal/planner/seed_test.go @@ -0,0 +1,155 @@ +package planner + +import ( + "encoding/json" + "slices" + "testing" + + seiconfig "github.com/sei-protocol/sei-config" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1" + "github.com/sei-protocol/sei-k8s-controller/internal/task" +) + +// seedImage and its predecessor exercise the image-drift update path. +const ( + seedImage = "seid:v6.4.1" + seedPrevImage = "seid:v6.4.0" +) + +func seedNode() *seiv1alpha1.SeiNode { + return &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: "seed-0", Namespace: sourceChainID}, + Spec: seiv1alpha1.SeiNodeSpec{ + ChainID: sourceChainID, + Image: seedImage, + Seed: &seiv1alpha1.SeedSpec{ + NodeKey: seiv1alpha1.NodeKeySource{ + Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: "seed-0-node-key"}, + }, + }, + }, + } +} + +// A seed takes the genesis progression and validates its pinned identity before +// the StatefulSet is applied. The absences are the point: a seed stores no chain +// state, so restoring or state-syncing one is meaningless, and it signs nothing. +func TestSeedPlanner_GenesisProgression(t *testing.T) { + p := &seedPlanner{} + plan, err := p.BuildPlan(seedNode()) + if err != nil { + t.Fatalf("BuildPlan: %v", err) + } + + got := planTaskTypes(plan) + want := []string{ + task.TaskTypeEnsureDataPVC, + task.TaskTypeValidateNodeKey, + task.TaskTypeApplyRBACProxyConfig, + task.TaskTypeApplyStatefulSet, + task.TaskTypeApplyService, + TaskConfigureGenesis, + TaskConfigApply, + TaskConfigValidate, + TaskMarkReady, + } + if !slices.Equal(got, want) { + t.Errorf("seed init progression:\n got %v\nwant %v", got, want) + } + + for _, absent := range []string{ + TaskSnapshotRestore, + TaskConfigureStateSync, + task.TaskTypeValidateSigningKey, + task.TaskTypeValidateOperatorKeyring, + } { + if slices.Contains(got, absent) { + t.Errorf("seed plan must not contain %s, got %v", absent, got) + } + } +} + +// The config-apply payload carries the mode; sei-config's applySeedOverrides +// resolves every seed-specific key from it sidecar-side. +func TestSeedPlanner_ConfigIntentCarriesSeedMode(t *testing.T) { + p := &seedPlanner{} + plan, err := p.BuildPlan(seedNode()) + if err != nil { + t.Fatalf("BuildPlan: %v", err) + } + + idx := slices.IndexFunc(plan.Tasks, func(pt seiv1alpha1.PlannedTask) bool { + return pt.Type == TaskConfigApply + }) + if idx < 0 { + t.Fatalf("plan has no %s task: %v", TaskConfigApply, planTaskTypes(plan)) + } + + var intent seiconfig.ConfigIntent + if err := json.Unmarshal(plan.Tasks[idx].Params.Raw, &intent); err != nil { + t.Fatalf("unmarshaling config-apply params: %v", err) + } + if intent.Mode != seiconfig.ModeSeed { + t.Errorf("config-apply mode = %q, want %q", intent.Mode, seiconfig.ModeSeed) + } +} + +// A seed's NodeID is published, so an unpinned identity must fail the plan +// rather than boot a node that regenerates its key onto the data volume. +func TestSeedPlanner_ValidateRequiresNodeKey(t *testing.T) { + node := seedNode() + node.Spec.Seed.NodeKey = seiv1alpha1.NodeKeySource{} + + p := &seedPlanner{} + if err := p.Validate(node); err == nil { + t.Error("Validate should reject a seed with no nodeKey Secret") + } + + if err := p.Validate(seedNode()); err != nil { + t.Errorf("Validate on a well-formed seed: %v", err) + } +} + +// An image roll re-validates the identity Secret before touching the +// StatefulSet, so a broken Secret surfaces controller-side rather than as a +// kubelet mount failure on the replacement pod. +func TestSeedPlanner_UpdatePlanValidatesNodeKeyFirst(t *testing.T) { + node := seedNode() + node.Status.Phase = seiv1alpha1.PhaseRunning + node.Status.CurrentImage = seedPrevImage + + p := &seedPlanner{} + plan, err := p.BuildPlan(node) + if err != nil { + t.Fatalf("BuildPlan: %v", err) + } + if plan == nil { + t.Fatal("expected an update plan for a drifted image") + } + + got := planTaskTypes(plan) + if got[0] != task.TaskTypeValidateNodeKey { + t.Errorf("update plan should validate the node key first, got %v", got) + } + if slices.Contains(got, TaskConfigureStateSync) { + t.Errorf("update plan must not contain configure-state-sync, got %v", got) + } +} + +// Steady state builds nothing: no drift, no plan. +func TestSeedPlanner_NoPlanWithoutDrift(t *testing.T) { + node := seedNode() + node.Status.Phase = seiv1alpha1.PhaseRunning + node.Status.CurrentImage = node.Spec.Image + + p := &seedPlanner{} + plan, err := p.BuildPlan(node) + if err != nil { + t.Fatalf("BuildPlan: %v", err) + } + if plan != nil { + t.Errorf("expected no plan in steady state, got %v", planTaskTypes(plan)) + } +} diff --git a/internal/platform/load.go b/internal/platform/load.go index b64eb068..a95d000e 100644 --- a/internal/platform/load.go +++ b/internal/platform/load.go @@ -41,6 +41,7 @@ func Load() (Config, error) { NodepoolName: file.Scheduling.NodepoolName, NodepoolArchive: file.Scheduling.NodepoolArchive, NodepoolValidator: file.Scheduling.NodepoolValidator, + NodepoolSeed: file.Scheduling.NodepoolSeed, TolerationKey: file.Scheduling.TolerationKey, ServiceAccount: file.Scheduling.ServiceAccount, @@ -49,11 +50,13 @@ func Load() (Config, error) { StorageClassArchive: file.Storage.ClassArchive, StorageSizeDefault: file.Storage.SizeDefault, StorageSizeArchive: file.Storage.SizeArchive, + StorageSizeSeed: file.Storage.SizeSeed, NodeResourcesValidator: file.Resources.Validator, NodeResourcesNode: file.Resources.Node, NodeResourcesReplayer: file.Resources.Replayer, NodeResourcesArchive: file.Resources.Archive, + NodeResourcesSeed: file.Resources.Seed, SnapshotBucket: file.Snapshot.Bucket, SnapshotRegion: file.Snapshot.Region, diff --git a/internal/platform/load_test.go b/internal/platform/load_test.go index 0afaaf1d..88eaa6d8 100644 --- a/internal/platform/load_test.go +++ b/internal/platform/load_test.go @@ -13,6 +13,7 @@ scheduling: nodepoolName: file-nodepool nodepoolArchive: file-nodepool-archive nodepoolValidator: file-nodepool-validator + nodepoolSeed: file-nodepool-seed tolerationKey: file-toleration serviceAccount: file-sa storage: @@ -142,6 +143,25 @@ func TestLoad_MissingNodepoolValidator_FailsValidate(t *testing.T) { } } +// scheduling.nodepoolSeed is required on the same terms as the archive and +// validator pools — whether or not the cluster runs seeds. NodepoolForMode gives +// a seed no fallback, so an unset key would otherwise put a small seed on the +// RPC-class default pool. +func TestLoad_MissingNodepoolSeed_FailsValidate(t *testing.T) { + setGatewayEnv(t) + body := strings.Replace(fullConfig, " nodepoolSeed: file-nodepool-seed\n", "", 1) + path := writeConfig(t, body) + t.Setenv(envControllerConfig, path) + + cfg, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "scheduling.nodepoolSeed") { + t.Fatalf("want Validate error naming scheduling.nodepoolSeed, got %v", err) + } +} + // A missing gateway env var fails Validate, naming the env var (gateway is still // env-sourced pending PLT-451). func TestLoad_MissingGateway_FailsValidate(t *testing.T) { diff --git a/internal/platform/platform.go b/internal/platform/platform.go index 6d81b129..ab6592cb 100644 --- a/internal/platform/platform.go +++ b/internal/platform/platform.go @@ -34,6 +34,14 @@ const ( // modeValidator matches seiconfig.ModeValidator without importing sei-config. modeValidator = "validator" + + // modeSeed matches seiconfig.ModeSeed without importing sei-config. + modeSeed = "seed" + + // defaultStorageSizeSeed backs a seed's peer store plus the block and state + // DBs seid opens but never writes. Code-authoritative so a seed does not + // silently claim StorageSizeDefault, which is sized for a chain's full state. + defaultStorageSizeSeed = "20Gi" ) // Config holds infrastructure-level settings that vary per deployment @@ -54,6 +62,15 @@ type Config struct { StorageSizeDefault string StorageSizeArchive string + // NodepoolSeed is required, like the archive and validator pools, whether or + // not the cluster runs seeds. A seed must not inherit the RPC-class default + // pool, so NodepoolForMode gives it no fallback. + // + // StorageSizeSeed is optional; defaultStorageSizeSeed is a correct value, not + // a placeholder. Read via SeedStorageSize. + NodepoolSeed string + StorageSizeSeed string + // Per-role seid-container resource overrides, keyed by sei.io/role. Each is // optional: an unset field (or unset sub-field) falls back to the // code-authoritative, prod-safe default for that role in internal/noderesource. @@ -62,6 +79,7 @@ type Config struct { NodeResourcesNode ResourceOverride NodeResourcesReplayer ResourceOverride NodeResourcesArchive ResourceOverride + NodeResourcesSeed ResourceOverride SnapshotBucket string SnapshotRegion string @@ -130,6 +148,7 @@ type SchedulingConfig struct { NodepoolName string `json:"nodepoolName"` NodepoolArchive string `json:"nodepoolArchive"` NodepoolValidator string `json:"nodepoolValidator"` + NodepoolSeed string `json:"nodepoolSeed"` TolerationKey string `json:"tolerationKey"` ServiceAccount string `json:"serviceAccount"` } @@ -141,6 +160,8 @@ type StorageConfig struct { ClassArchive string `json:"classArchive"` SizeDefault string `json:"sizeDefault"` SizeArchive string `json:"sizeArchive"` + // SizeSeed is optional; unset falls back to defaultStorageSizeSeed. + SizeSeed string `json:"sizeSeed"` } // ResourcesConfig holds seid-container resource sizing. @@ -155,6 +176,7 @@ type ResourcesConfig struct { Node ResourceOverride `json:"node"` Replayer ResourceOverride `json:"replayer"` Archive ResourceOverride `json:"archive"` + Seed ResourceOverride `json:"seed"` } // ResourceOverride is a per-mode seid-container resource override. Both fields @@ -192,18 +214,37 @@ type ImagesConfig struct { // NodepoolForMode returns the Karpenter NodePool name for the given // sei-config mode string. Archive and validator nodes each use a -// dedicated pool; all other modes share the default pool. +// dedicated pool; seed uses one when configured, since its footprint is far +// smaller than the default pool's instance class. All other modes share the +// default pool. func (c Config) NodepoolForMode(mode string) string { switch mode { case modeArchive: return c.NodepoolArchive case modeValidator: return c.NodepoolValidator + case modeSeed: + // No fallback to NodepoolName: that pool is sized for RPC-class nodes, so + // a seed landing there costs an order of magnitude more than a seed is + // worth — and karpenter.sh/do-not-disrupt then pins that node against + // consolidation for the seed's life. Callers rendering a seed pod must + // reject an empty value rather than substitute one; see + // noderesource.GenerateStatefulSet. + return c.NodepoolSeed default: return c.NodepoolName } } +// SeedStorageSize returns the data-volume size for a seed node, falling back to +// the code default when the app-config file omits storage.sizeSeed. +func (c Config) SeedStorageSize() string { + if c.StorageSizeSeed != "" { + return c.StorageSizeSeed + } + return defaultStorageSizeSeed +} + // Validate returns an error if a required field is unset. name is the field's // app-config file key, except the networking/gateway fields (still env-sourced // pending PLT-451) which name their env var. Slice order is the report order @@ -216,6 +257,7 @@ func (c Config) Validate() error { {"scheduling.nodepoolName", c.NodepoolName}, {"scheduling.nodepoolArchive", c.NodepoolArchive}, {"scheduling.nodepoolValidator", c.NodepoolValidator}, + {"scheduling.nodepoolSeed", c.NodepoolSeed}, {"scheduling.tolerationKey", c.TolerationKey}, {"scheduling.serviceAccount", c.ServiceAccount}, {"storage.classPerf", c.StorageClassPerf}, @@ -251,6 +293,7 @@ func (c Config) Validate() error { {"resources.node", c.NodeResourcesNode}, {"resources.replayer", c.NodeResourcesReplayer}, {"resources.archive", c.NodeResourcesArchive}, + {"resources.seed", c.NodeResourcesSeed}, } for _, o := range overrides { if err := o.val.validate(o.key); err != nil { diff --git a/internal/platform/platform_test.go b/internal/platform/platform_test.go index 5fccb191..654f1a96 100644 --- a/internal/platform/platform_test.go +++ b/internal/platform/platform_test.go @@ -16,19 +16,21 @@ func TestDataDirIsUnderHomeDir(t *testing.T) { } } -// TestNodepoolForMode pins the mode→pool routing: archive and validator each -// get a dedicated pool; every other mode ("full", and the empty fallback) +// TestNodepoolForMode pins the mode→pool routing: archive, validator and seed +// each get a dedicated pool; every other mode ("full", and the empty fallback) // shares the default pool. func TestNodepoolForMode(t *testing.T) { const ( poolDefault = "sei-node" poolArchive = "sei-archive" poolValidator = "sei-validator" + poolSeed = "sei-seed" ) c := Config{ NodepoolName: poolDefault, NodepoolArchive: poolArchive, NodepoolValidator: poolValidator, + NodepoolSeed: poolSeed, } cases := []struct { mode string @@ -36,6 +38,7 @@ func TestNodepoolForMode(t *testing.T) { }{ {"archive", poolArchive}, {"validator", poolValidator}, + {"seed", poolSeed}, {"full", poolDefault}, {"", poolDefault}, } @@ -45,3 +48,32 @@ func TestNodepoolForMode(t *testing.T) { } } } + +// A seed must never inherit the RPC-class default pool: that costs an order of +// magnitude more than a seed is worth. NodepoolForMode reports the absence so +// the pod render can reject it. +func TestNodepoolForMode_SeedHasNoDefaultPoolFallback(t *testing.T) { + c := Config{NodepoolName: "sei-node"} + + if got := c.NodepoolForMode(modeSeed); got != "" { + t.Errorf("NodepoolForMode(seed) with no seed pool = %q, want empty", got) + } + if got := c.NodepoolForMode("full"); got != "sei-node" { + t.Errorf("NodepoolForMode(full) = %q, want the default pool", got) + } +} + +// sizeSeed is optional: the code default is a correct value, so an app-config +// file predating the key still sizes a seed volume sanely. +func TestSeedStorageSize_FallsBackToCodeDefault(t *testing.T) { + c := Config{} + + if got := c.SeedStorageSize(); got != defaultStorageSizeSeed { + t.Errorf("SeedStorageSize() = %q, want %q", got, defaultStorageSizeSeed) + } + + c.StorageSizeSeed = "40Gi" + if got := c.SeedStorageSize(); got != "40Gi" { + t.Errorf("SeedStorageSize() with override = %q, want 40Gi", got) + } +} diff --git a/internal/platform/platformtest/config.go b/internal/platform/platformtest/config.go index 9145ff2d..f969f68f 100644 --- a/internal/platform/platformtest/config.go +++ b/internal/platform/platformtest/config.go @@ -10,6 +10,7 @@ func Config() platform.Config { NodepoolName: "sei-node", NodepoolArchive: "sei-archive", NodepoolValidator: "sei-validator", + NodepoolSeed: "sei-seed", TolerationKey: "sei.io/workload", ServiceAccount: "seid-node", StorageClassPerf: "gp3-10k-750", diff --git a/internal/sidecartransport/sidecartransport_test.go b/internal/sidecartransport/sidecartransport_test.go index b8218926..5ae205ad 100644 --- a/internal/sidecartransport/sidecartransport_test.go +++ b/internal/sidecartransport/sidecartransport_test.go @@ -92,4 +92,3 @@ func TestNew_DefaultTokenPathConstant(t *testing.T) { t.Errorf("default path drifted from K8s convention: %s", DefaultServiceAccountTokenPath) } } - diff --git a/internal/task/bootstrap_resources.go b/internal/task/bootstrap_resources.go index 94ac7a86..ccd8d424 100644 --- a/internal/task/bootstrap_resources.go +++ b/internal/task/bootstrap_resources.go @@ -311,6 +311,13 @@ func bootstrapPVCClaimName(node *seiv1alpha1.SeiNode) string { } // bootstrapNodeMode determines the sei-config mode string from the node spec. +// +// Deliberately has no seed arm: NeedsBootstrap requires a snapshot source with a +// bootstrapImage, and SeiNodeSpec.SnapshotSource() is nil for a seed, so a seed +// never reaches the bootstrap-Job path. That invariant is what keeps the default +// arm from rendering a seed as a full node here — with RPC probes and a +// cosmos-exporter — and it is also why buildBootstrapPodSpec does not call +// ValidateSeedProbes. TestSeedNeverBootstraps pins the invariant. func bootstrapNodeMode(node *seiv1alpha1.SeiNode) string { switch { case node.Spec.Archive != nil: diff --git a/manifests/sei.io_seinodes.yaml b/manifests/sei.io_seinodes.yaml index 5646ce81..62ba6dc7 100644 --- a/manifests/sei.io_seinodes.yaml +++ b/manifests/sei.io_seinodes.yaml @@ -52,8 +52,8 @@ spec: spec: description: |- SeiNodeSpec defines the desired state of a standalone Sei node. - Exactly one mode sub-spec (fullNode, archive, replayer, validator) must be set; - the populated field determines the node's operating mode. + Exactly one mode sub-spec (fullNode, archive, replayer, validator, seed) must + be set; the populated field determines the node's operating mode. properties: archive: description: Archive configures an archive node with full history @@ -446,6 +446,60 @@ spec: required: - snapshot type: object + seed: + description: Seed configures a peer-discovery seed node (P2P + PEX + only). + properties: + nodeKey: + description: |- + NodeKey supplies the P2P identity (node_key.json) this seed presents. + + Required, where the validator's NodeKey is optional, because a seed's + NodeID is published: operators dial `NodeID@host:port` and the + secret-connection handshake verifies the pinned value, so a changed NodeID + silently breaks every client carrying the old one. A Secret-sourced key + survives pod recreation and PVC loss; one left to `seid init` regenerates + onto the data volume. Carrying the identity to another cluster means + replicating the Secret there — the controller reads it, it does not move + it. Treat the NodeID as a one-way door. + + The Secret name is immutable, so rotating the identity means deleting and + recreating the SeiNode — and announcing the new NodeID. There is no fast + path for a leaked key: the old NodeID stays dialable until every client + ships a new default, so a thief keeps impersonating a bootstrap anchor. + + Give each seed a distinct Secret. Nothing rejects two seeds sharing one, + and they would present the same NodeID — collapsing the redundant anchors + into a single entry in every dialer's peer store. + properties: + secret: + description: |- + Secret loads the node key from a Kubernetes Secret in the + SeiNode's namespace. + properties: + secretName: + description: |- + SecretName is the name of a Secret in the SeiNode's namespace. + The controller never creates, mutates, or deletes this Secret. + Immutable: re-pointing the node ID on a running validator costs + peer reputation and forces a peer-graph reset; force delete-and-recreate. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + x-kubernetes-validations: + - message: secretName is immutable + rule: self == oldSelf + required: + - secretName + type: object + type: object + x-kubernetes-validations: + - message: exactly one node key source must be set + rule: '(has(self.secret) ? 1 : 0) == 1' + required: + - nodeKey + type: object sidecar: description: Sidecar configures the sei-sidecar container. properties: @@ -827,10 +881,11 @@ spec: - image type: object x-kubernetes-validations: - - message: exactly one of fullNode, archive, replayer, or validator must - be set + - message: exactly one of fullNode, archive, replayer, validator, or seed + must be set rule: '(has(self.fullNode) ? 1 : 0) + (has(self.archive) ? 1 : 0) + - (has(self.replayer) ? 1 : 0) + (has(self.validator) ? 1 : 0) == 1' + (has(self.replayer) ? 1 : 0) + (has(self.validator) ? 1 : 0) + (has(self.seed) + ? 1 : 0) == 1' - message: peers is required when replayer mode is set rule: '!has(self.replayer) || (has(self.peers) && size(self.peers) > 0)'