From 6e4b468f514e9ba0da5d17ea69849ab8c01a3294 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Mon, 11 May 2026 17:28:02 -0700 Subject: [PATCH 1/5] feat: SeiNode validator.operatorKeyring CRD surface (closes #219) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements Phase 1 step 1 of the in-pod governance signing workstream (design: sei-protocol/seictl#166). Adds validator.operatorKeyring to the SeiNode CRD — a discriminated union (Secret variant; future TMKMS/Vault/KMS reserved) declaring the operator-account keyring used by the sidecar for governance, MsgEditValidator, withdraw-rewards, and other operator-account txs. The keyring data Secret (mounted as a volume at $SEI_HOME/keyring-file/) and a separate passphrase Secret (projected as an env var) are mounted exclusively on the sidecar container — never on the seid main container, init container, or bootstrap pod. Four CEL invariants on ValidatorSpec enforce the trust boundary at admission: - operatorKeyring.secret.secretName != signingKey.secret.secretName - operatorKeyring.secret.secretName != nodeKey.secret.secretName - operatorKeyring.secret.secretName != passphraseSecretRef.secretName - secretName / passphraseSecretRef.secretName immutable post-create Pre-flight task validate-operator-keyring mirrors validate-signing-key: checks both Secrets exist + non-empty, validates the file-keyring's *.info and *.address index files, and confirms the named key entry is present if KeyName is specified. Does NOT decrypt — that sits with the sidecar's startup smoke test, not the controller's TCB. Sidecar container security context tightened in the same workstream: RunAsNonRoot, RunAsUser/Group 65532, ReadOnlyRootFilesystem, AllowPrivilegeEscalation false, Capabilities.Drop ALL, SeccompProfile RuntimeDefault, pod-level FSGroup 65532 (required for the non-root UID to read the 0o400 Secret mount). Bootstrap-pod isolation guard generalized — assertNoSigningKeyOnBootstrapPod becomes assertNoValidatorSecretsOnBootstrapPod, walking volumes for any of {signing-key, operator-keyring, passphrase} Secrets and env vars for the passphrase Secret. NodeKey deliberately excluded (no signing authority). Co-Authored-By: Claude Opus 4.7 (1M context) --- api/v1alpha1/seinode_types.go | 13 + api/v1alpha1/validator_types.go | 94 +++++ api/v1alpha1/zz_generated.deepcopy.go | 56 +++ config/crd/sei.io_seinodedeployments.yaml | 90 +++++ config/crd/sei.io_seinodes.yaml | 88 +++++ internal/noderesource/noderesource.go | 141 +++++++- internal/noderesource/noderesource_test.go | 148 ++++++++ internal/planner/bootstrap.go | 6 + internal/planner/planner.go | 37 ++ internal/planner/validator_test.go | 129 +++++++ internal/task/bootstrap_resources.go | 84 ++++- internal/task/bootstrap_resources_test.go | 194 ++++++++++ internal/task/task.go | 15 +- internal/task/validate_operator_keyring.go | 183 ++++++++++ .../task/validate_operator_keyring_test.go | 333 ++++++++++++++++++ manifests/sei.io_seinodedeployments.yaml | 90 +++++ manifests/sei.io_seinodes.yaml | 88 +++++ 17 files changed, 1753 insertions(+), 36 deletions(-) create mode 100644 internal/task/bootstrap_resources_test.go create mode 100644 internal/task/validate_operator_keyring.go create mode 100644 internal/task/validate_operator_keyring_test.go diff --git a/api/v1alpha1/seinode_types.go b/api/v1alpha1/seinode_types.go index 75bffada..2b981a95 100644 --- a/api/v1alpha1/seinode_types.go +++ b/api/v1alpha1/seinode_types.go @@ -261,6 +261,12 @@ const ( // node-key Secret passes all validation requirements. Only set on // SeiNodes with spec.validator.nodeKey. ConditionNodeKeyReady = "NodeKeyReady" + + // ConditionOperatorKeyringReady indicates whether a referenced + // operator-keyring Secret pair (keyring data + passphrase) passes + // pre-flight validation. Only set on SeiNodes with + // spec.validator.operatorKeyring. + ConditionOperatorKeyringReady = "OperatorKeyringReady" ) // Reasons for the ImportPVCReady condition. @@ -284,6 +290,13 @@ const ( ReasonNodeKeyInvalid = "NodeKeyInvalid" // terminal: fail the plan ) +// Reasons for the OperatorKeyringReady condition. +const ( + ReasonOperatorKeyringValidated = "OperatorKeyringValidated" // validation succeeded + ReasonOperatorKeyringNotReady = "OperatorKeyringNotReady" // transient: retry + ReasonOperatorKeyringInvalid = "OperatorKeyringInvalid" // terminal: fail the plan +) + // SeiNodeStatus defines the observed state of a SeiNode. type SeiNodeStatus struct { // Phase is the high-level lifecycle state. diff --git a/api/v1alpha1/validator_types.go b/api/v1alpha1/validator_types.go index 9d58ff88..1d36f07f 100644 --- a/api/v1alpha1/validator_types.go +++ b/api/v1alpha1/validator_types.go @@ -5,6 +5,9 @@ package v1alpha1 // // +kubebuilder:validation:XValidation:rule="has(self.signingKey) == has(self.nodeKey)",message="signingKey and nodeKey must be set together (validators get both or neither)" // +kubebuilder:validation:XValidation:rule="!has(self.signingKey) || !has(self.nodeKey) || self.signingKey.secret.secretName != self.nodeKey.secret.secretName",message="signingKey and nodeKey must reference distinct Secrets — packing both keys in one Secret collapses the bootstrap-pod trust boundary" +// +kubebuilder:validation:XValidation:rule="!has(self.operatorKeyring) || !has(self.signingKey) || self.operatorKeyring.secret.secretName != self.signingKey.secret.secretName",message="operatorKeyring and signingKey must reference distinct Secrets — collapsing them into one Secret would force the sidecar/seid trust boundary to evaporate" +// +kubebuilder:validation:XValidation:rule="!has(self.operatorKeyring) || !has(self.nodeKey) || self.operatorKeyring.secret.secretName != self.nodeKey.secret.secretName",message="operatorKeyring and nodeKey must reference distinct Secrets" +// +kubebuilder:validation:XValidation:rule="!has(self.operatorKeyring) || self.operatorKeyring.secret.secretName != self.operatorKeyring.secret.passphraseSecretRef.secretName",message="operatorKeyring data Secret and passphrase Secret must be distinct" type ValidatorSpec struct { // Snapshot configures how the node obtains its initial chain state. // When absent the node block-syncs from genesis. @@ -35,6 +38,21 @@ type ValidatorSpec struct { // first appears on the network when the production pod starts. // +optional NodeKey *NodeKeySource `json:"nodeKey,omitempty"` + + // OperatorKeyring declares the source of this validator's operator-account + // keyring used by the sidecar to sign and broadcast governance, + // MsgEditValidator, withdraw-rewards, and other operator-account + // transactions. + // + // Independently optional from signingKey/nodeKey: a validator may run as a + // non-signing observer with operatorKeyring set (governance-only + // operations), or as a consensus-signing validator without operatorKeyring + // (governance performed out-of-band). + // + // Mounted exclusively on the sidecar container; the seid main container + // and bootstrap pods never carry this material. + // +optional + OperatorKeyring *OperatorKeyringSource `json:"operatorKeyring,omitempty"` } // SigningKeySource declares where a validator's consensus signing key @@ -109,6 +127,82 @@ type SecretNodeKeySource struct { SecretName string `json:"secretName"` } +// OperatorKeyringSource declares where a validator's operator-account +// keyring (used by the sidecar to sign governance, MsgEditValidator, +// withdraw-rewards, and other operator-account transactions) comes from. +// Exactly one variant must be set; variants are mutually exclusive. +// +// The CEL rule is shaped as a sum across all variants so adding a future +// sibling (TMKMS, Vault, KMS) is purely additive — bump the sum by one term. +// +// +kubebuilder:validation:XValidation:rule="(has(self.secret) ? 1 : 0) == 1",message="exactly one operator keyring source must be set" +type OperatorKeyringSource struct { + // Secret loads a Cosmos SDK file-backend keyring from a Kubernetes Secret + // in the SeiNode's namespace. + // +optional + Secret *SecretOperatorKeyringSource `json:"secret,omitempty"` + + // Future siblings: TMKMS, Vault, KMS. When added, bump the + // XValidation sum-of-variants rule. +} + +// SecretOperatorKeyringSource references the Kubernetes Secrets that supply +// the operator-account keyring directory and its unlock passphrase. The +// controller never creates, mutates, or deletes either Secret — their +// lifecycles are fully external (kubectl + SOPS, ESO, CSI Secrets Store). +// +// The keyring data and passphrase live in deliberately separate Secrets: +// the data Secret is projected as a directory-shaped volume mount, so +// co-locating the passphrase as a data key would project it as a file +// under the keyring directory and the file-backend would treat it as +// keyring contents. +type SecretOperatorKeyringSource struct { + // SecretName names a Secret in the SeiNode's namespace whose data keys + // are the on-disk Cosmos SDK file-keyring layout. Minimum required: + // .info (armored encrypted key blob) + // .address (name→address index) + // + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="secretName is immutable" + SecretName string `json:"secretName"` + + // KeyName is the name of the keyring entry to use when signing + // (the name passed to `seid keys add `). Defaults to + // "node_admin" to preserve continuity with the seienv convention. + // Mutable — rotating to a different entry within the same Secret + // is a routine operator-account change, not a slashing risk. + // + // +optional + // +kubebuilder:default="node_admin" + // +kubebuilder:validation:MaxLength=64 + // +kubebuilder:validation:Pattern=`^[a-zA-Z0-9_-]+$` + KeyName string `json:"keyName,omitempty"` + + // PassphraseSecretRef names a separate Secret containing the keyring + // unlock passphrase. Required for the file backend. + PassphraseSecretRef PassphraseSecretRef `json:"passphraseSecretRef"` +} + +// PassphraseSecretRef points at a single data key inside a Secret. +type PassphraseSecretRef struct { + // SecretName names the passphrase Secret in the SeiNode's namespace. + // + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="passphrase secretName is immutable" + SecretName string `json:"secretName"` + + // Key is the data key inside the Secret. Defaults to "passphrase". + // + // +optional + // +kubebuilder:default="passphrase" + // +kubebuilder:validation:MaxLength=253 + Key string `json:"key,omitempty"` +} + // GenesisCeremonyNodeConfig holds per-node genesis ceremony parameters. // Populated by the SeiNodeDeployment controller when genesis is configured. type GenesisCeremonyNodeConfig struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index fe9d7c6f..50bf633f 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -408,6 +408,41 @@ func (in *NodeKeySource) DeepCopy() *NodeKeySource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OperatorKeyringSource) DeepCopyInto(out *OperatorKeyringSource) { + *out = *in + if in.Secret != nil { + in, out := &in.Secret, &out.Secret + *out = new(SecretOperatorKeyringSource) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperatorKeyringSource. +func (in *OperatorKeyringSource) DeepCopy() *OperatorKeyringSource { + if in == nil { + return nil + } + out := new(OperatorKeyringSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PassphraseSecretRef) DeepCopyInto(out *PassphraseSecretRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PassphraseSecretRef. +func (in *PassphraseSecretRef) DeepCopy() *PassphraseSecretRef { + if in == nil { + return nil + } + out := new(PassphraseSecretRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PeerSource) DeepCopyInto(out *PeerSource) { *out = *in @@ -605,6 +640,22 @@ func (in *SecretNodeKeySource) DeepCopy() *SecretNodeKeySource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretOperatorKeyringSource) DeepCopyInto(out *SecretOperatorKeyringSource) { + *out = *in + out.PassphraseSecretRef = in.PassphraseSecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretOperatorKeyringSource. +func (in *SecretOperatorKeyringSource) DeepCopy() *SecretOperatorKeyringSource { + if in == nil { + return nil + } + out := new(SecretOperatorKeyringSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecretSigningKeySource) DeepCopyInto(out *SecretSigningKeySource) { *out = *in @@ -1256,6 +1307,11 @@ func (in *ValidatorSpec) DeepCopyInto(out *ValidatorSpec) { *out = new(NodeKeySource) (*in).DeepCopyInto(*out) } + if in.OperatorKeyring != nil { + in, out := &in.OperatorKeyring, &out.OperatorKeyring + *out = new(OperatorKeyringSource) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ValidatorSpec. diff --git a/config/crd/sei.io_seinodedeployments.yaml b/config/crd/sei.io_seinodedeployments.yaml index f6fa6cf9..86a91f5f 100644 --- a/config/crd/sei.io_seinodedeployments.yaml +++ b/config/crd/sei.io_seinodedeployments.yaml @@ -683,6 +683,83 @@ spec: x-kubernetes-validations: - message: exactly one node key source must be set rule: '(has(self.secret) ? 1 : 0) == 1' + operatorKeyring: + description: |- + OperatorKeyring declares the source of this validator's operator-account + keyring used by the sidecar to sign and broadcast governance, + MsgEditValidator, withdraw-rewards, and other operator-account + transactions. + + Independently optional from signingKey/nodeKey: a validator may run as a + non-signing observer with operatorKeyring set (governance-only + operations), or as a consensus-signing validator without operatorKeyring + (governance performed out-of-band). + + Mounted exclusively on the sidecar container; the seid main container + and bootstrap pods never carry this material. + properties: + secret: + description: |- + Secret loads a Cosmos SDK file-backend keyring from a Kubernetes Secret + in the SeiNode's namespace. + properties: + keyName: + default: node_admin + description: |- + KeyName is the name of the keyring entry to use when signing + (the name passed to `seid keys add `). Defaults to + "node_admin" to preserve continuity with the seienv convention. + Mutable — rotating to a different entry within the same Secret + is a routine operator-account change, not a slashing risk. + maxLength: 64 + pattern: ^[a-zA-Z0-9_-]+$ + type: string + passphraseSecretRef: + description: |- + PassphraseSecretRef names a separate Secret containing the keyring + unlock passphrase. Required for the file backend. + properties: + key: + default: passphrase + description: Key is the data key inside the + Secret. Defaults to "passphrase". + maxLength: 253 + type: string + secretName: + description: SecretName names the passphrase + Secret in the SeiNode's namespace. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + x-kubernetes-validations: + - message: passphrase secretName is immutable + rule: self == oldSelf + required: + - secretName + type: object + secretName: + description: |- + SecretName names a Secret in the SeiNode's namespace whose data keys + are the on-disk Cosmos SDK file-keyring layout. Minimum required: + .info (armored encrypted key blob) + .address (name→address index) + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + x-kubernetes-validations: + - message: secretName is immutable + rule: self == oldSelf + required: + - passphraseSecretRef + - secretName + type: object + type: object + x-kubernetes-validations: + - message: exactly one operator keyring source must be + set + rule: '(has(self.secret) ? 1 : 0) == 1' signingKey: description: |- SigningKey declares the source of this validator's consensus signing @@ -774,6 +851,19 @@ spec: bootstrap-pod trust boundary rule: '!has(self.signingKey) || !has(self.nodeKey) || self.signingKey.secret.secretName != self.nodeKey.secret.secretName' + - message: operatorKeyring and signingKey must reference distinct + Secrets — collapsing them into one Secret would force + the sidecar/seid trust boundary to evaporate + rule: '!has(self.operatorKeyring) || !has(self.signingKey) + || self.operatorKeyring.secret.secretName != self.signingKey.secret.secretName' + - message: operatorKeyring and nodeKey must reference distinct + Secrets + rule: '!has(self.operatorKeyring) || !has(self.nodeKey) + || self.operatorKeyring.secret.secretName != self.nodeKey.secret.secretName' + - message: operatorKeyring data Secret and passphrase Secret + must be distinct + rule: '!has(self.operatorKeyring) || self.operatorKeyring.secret.secretName + != self.operatorKeyring.secret.passphraseSecretRef.secretName' required: - chainId - image diff --git a/config/crd/sei.io_seinodes.yaml b/config/crd/sei.io_seinodes.yaml index 5dda9b46..5286662c 100644 --- a/config/crd/sei.io_seinodes.yaml +++ b/config/crd/sei.io_seinodes.yaml @@ -538,6 +538,82 @@ spec: x-kubernetes-validations: - message: exactly one node key source must be set rule: '(has(self.secret) ? 1 : 0) == 1' + operatorKeyring: + description: |- + OperatorKeyring declares the source of this validator's operator-account + keyring used by the sidecar to sign and broadcast governance, + MsgEditValidator, withdraw-rewards, and other operator-account + transactions. + + Independently optional from signingKey/nodeKey: a validator may run as a + non-signing observer with operatorKeyring set (governance-only + operations), or as a consensus-signing validator without operatorKeyring + (governance performed out-of-band). + + Mounted exclusively on the sidecar container; the seid main container + and bootstrap pods never carry this material. + properties: + secret: + description: |- + Secret loads a Cosmos SDK file-backend keyring from a Kubernetes Secret + in the SeiNode's namespace. + properties: + keyName: + default: node_admin + description: |- + KeyName is the name of the keyring entry to use when signing + (the name passed to `seid keys add `). Defaults to + "node_admin" to preserve continuity with the seienv convention. + Mutable — rotating to a different entry within the same Secret + is a routine operator-account change, not a slashing risk. + maxLength: 64 + pattern: ^[a-zA-Z0-9_-]+$ + type: string + passphraseSecretRef: + description: |- + PassphraseSecretRef names a separate Secret containing the keyring + unlock passphrase. Required for the file backend. + properties: + key: + default: passphrase + description: Key is the data key inside the Secret. + Defaults to "passphrase". + maxLength: 253 + type: string + secretName: + description: SecretName names the passphrase Secret + in the SeiNode's namespace. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + x-kubernetes-validations: + - message: passphrase secretName is immutable + rule: self == oldSelf + required: + - secretName + type: object + secretName: + description: |- + SecretName names a Secret in the SeiNode's namespace whose data keys + are the on-disk Cosmos SDK file-keyring layout. Minimum required: + .info (armored encrypted key blob) + .address (name→address index) + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + x-kubernetes-validations: + - message: secretName is immutable + rule: self == oldSelf + required: + - passphraseSecretRef + - secretName + type: object + type: object + x-kubernetes-validations: + - message: exactly one operator keyring source must be set + rule: '(has(self.secret) ? 1 : 0) == 1' signingKey: description: |- SigningKey declares the source of this validator's consensus signing @@ -628,6 +704,18 @@ spec: trust boundary rule: '!has(self.signingKey) || !has(self.nodeKey) || self.signingKey.secret.secretName != self.nodeKey.secret.secretName' + - message: operatorKeyring and signingKey must reference distinct + Secrets — collapsing them into one Secret would force the sidecar/seid + trust boundary to evaporate + rule: '!has(self.operatorKeyring) || !has(self.signingKey) || self.operatorKeyring.secret.secretName + != self.signingKey.secret.secretName' + - message: operatorKeyring and nodeKey must reference distinct Secrets + rule: '!has(self.operatorKeyring) || !has(self.nodeKey) || self.operatorKeyring.secret.secretName + != self.nodeKey.secret.secretName' + - message: operatorKeyring data Secret and passphrase Secret must + be distinct + rule: '!has(self.operatorKeyring) || self.operatorKeyring.secret.secretName + != self.operatorKeyring.secret.passphraseSecretRef.secretName' required: - chainId - image diff --git a/internal/noderesource/noderesource.go b/internal/noderesource/noderesource.go index 879d7528..3bae3068 100644 --- a/internal/noderesource/noderesource.go +++ b/internal/noderesource/noderesource.go @@ -33,6 +33,18 @@ const ( nodeKeyVolumeName = "node-key" nodeKeyDataKey = "node_key.json" + + operatorKeyringVolumeName = "operator-keyring" + // operatorKeyringDirName matches the on-disk directory the Cosmos SDK + // file-backend keyring appends to $SEI_HOME — keyring.New(name, BackendFile, + // homeDir, ...) implicitly opens homeDir/keyring-file/. + operatorKeyringDirName = "keyring-file" + keyringPassphraseEnvVar = "SEI_KEYRING_PASSPHRASE" + + // sidecarNonRootUID is the nonroot UID/GID baked into distroless and + // chainguard static-debian12 base images. Pod-level fsGroup matches so + // the non-root sidecar can read kubelet-projected 0o400 Secret files. + sidecarNonRootUID int64 = 65532 ) // PlatformConfig is an alias for platform.Config. @@ -251,14 +263,19 @@ func buildNodePodSpec(node *seiv1alpha1.SeiNode, p PlatformConfig) corev1.PodSpe signingVolumes := signingKeyVolumes(node) nodeVolumes := nodeKeyVolumes(node) - volumes := make([]corev1.Volume, 0, 1+len(signingVolumes)+len(nodeVolumes)) + keyringVolumes := operatorKeyringVolumes(node) + volumes := make([]corev1.Volume, 0, 1+len(signingVolumes)+len(nodeVolumes)+len(keyringVolumes)) volumes = append(volumes, dataVolume) volumes = append(volumes, signingVolumes...) volumes = append(volumes, nodeVolumes...) + volumes = append(volumes, keyringVolumes...) pool := p.NodepoolForMode(NodeMode(node)) spec := corev1.PodSpec{ + // automountServiceAccountToken is left at the kubelet default (true) + // — the projected token is a hard dependency for Phase 4 TokenReview + // authentication on sidecar HTTP endpoints (see #165). ServiceAccountName: p.ServiceAccount, Tolerations: []corev1.Toleration{ {Key: p.TolerationKey, Value: pool, Effect: corev1.TaintEffectNoSchedule}, @@ -280,6 +297,12 @@ func buildNodePodSpec(node *seiv1alpha1.SeiNode, p PlatformConfig) corev1.PodSpe } spec.ShareProcessNamespace = ptr.To(true) + // fsGroup is required so the non-root sidecar (UID 65532) can read + // 0o400 Secret-projected files (operator keyring) kubelet owns root:root. + fsGroup := sidecarNonRootUID + spec.SecurityContext = &corev1.PodSecurityContext{ + FSGroup: &fsGroup, + } spec.InitContainers = []corev1.Container{ buildSeidInitContainer(node), buildSidecarContainer(node, p), @@ -306,26 +329,35 @@ func SidecarPort(node *seiv1alpha1.SeiNode) int32 { func buildSidecarContainer(node *seiv1alpha1.SeiNode, p PlatformConfig) corev1.Container { port := SidecarPort(node) + keyringEnv := operatorKeyringEnvVars(node) + env := make([]corev1.EnvVar, 0, 7+len(keyringEnv)) + env = append(env, + corev1.EnvVar{Name: "SEI_CHAIN_ID", Value: node.Spec.ChainID}, + corev1.EnvVar{Name: "SEI_SIDECAR_PORT", Value: fmt.Sprintf("%d", port)}, + corev1.EnvVar{Name: "SEI_HOME", Value: dataDir}, + corev1.EnvVar{Name: "SEI_GENESIS_BUCKET", Value: p.GenesisBucket}, + corev1.EnvVar{Name: "SEI_GENESIS_REGION", Value: p.GenesisRegion}, + corev1.EnvVar{Name: "SEI_SNAPSHOT_BUCKET", Value: p.SnapshotBucket}, + corev1.EnvVar{Name: "SEI_SNAPSHOT_REGION", Value: p.SnapshotRegion}, + ) + env = append(env, keyringEnv...) + + keyringMounts := operatorKeyringMounts(node) + mounts := make([]corev1.VolumeMount, 0, 1+len(keyringMounts)) + mounts = append(mounts, corev1.VolumeMount{Name: "data", MountPath: dataDir}) + mounts = append(mounts, keyringMounts...) + c := corev1.Container{ Name: "sei-sidecar", Image: sidecarImage(node), Command: []string{"seictl", "serve"}, RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), - Env: []corev1.EnvVar{ - {Name: "SEI_CHAIN_ID", Value: node.Spec.ChainID}, - {Name: "SEI_SIDECAR_PORT", Value: fmt.Sprintf("%d", port)}, - {Name: "SEI_HOME", Value: dataDir}, - {Name: "SEI_GENESIS_BUCKET", Value: p.GenesisBucket}, - {Name: "SEI_GENESIS_REGION", Value: p.GenesisRegion}, - {Name: "SEI_SNAPSHOT_BUCKET", Value: p.SnapshotBucket}, - {Name: "SEI_SNAPSHOT_REGION", Value: p.SnapshotRegion}, - }, + Env: env, Ports: []corev1.ContainerPort{ {Name: "sidecar", ContainerPort: port, Protocol: corev1.ProtocolTCP}, }, - VolumeMounts: []corev1.VolumeMount{ - {Name: "data", MountPath: dataDir}, - }, + VolumeMounts: mounts, + SecurityContext: sidecarSecurityContext(), LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ @@ -567,3 +599,86 @@ func nodeKeySecretSource(node *seiv1alpha1.SeiNode) *seiv1alpha1.SecretNodeKeySo } return node.Spec.Validator.NodeKey.Secret } + +// operatorKeyringVolumes projects the operator-keyring Secret as a directory +// under $SEI_HOME/keyring-file/ — the Cosmos SDK file-backend layout. +// Mounted on the sidecar container only; the seid main and bootstrap pods +// never see this material. +func operatorKeyringVolumes(node *seiv1alpha1.SeiNode) []corev1.Volume { + src := operatorKeyringSecretSource(node) + if src == nil { + return nil + } + return []corev1.Volume{{ + Name: operatorKeyringVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: src.SecretName, + DefaultMode: ptr.To[int32](0o400), + }, + }, + }} +} + +func operatorKeyringMounts(node *seiv1alpha1.SeiNode) []corev1.VolumeMount { + if operatorKeyringSecretSource(node) == nil { + return nil + } + return []corev1.VolumeMount{{ + Name: operatorKeyringVolumeName, + MountPath: dataDir + "/" + operatorKeyringDirName, + ReadOnly: true, + }} +} + +// operatorKeyringEnvVars injects the keyring unlock passphrase into the +// sidecar process via a separate Secret reference. The passphrase lives in +// its own Secret because the keyring data Secret is projected as a +// directory — co-locating the passphrase as a data key would land it as a +// file inside the keyring directory. +func operatorKeyringEnvVars(node *seiv1alpha1.SeiNode) []corev1.EnvVar { + src := operatorKeyringSecretSource(node) + if src == nil { + return nil + } + key := src.PassphraseSecretRef.Key + if key == "" { + key = "passphrase" + } + return []corev1.EnvVar{{ + Name: keyringPassphraseEnvVar, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: src.PassphraseSecretRef.SecretName}, + Key: key, + }, + }, + }} +} + +func operatorKeyringSecretSource(node *seiv1alpha1.SeiNode) *seiv1alpha1.SecretOperatorKeyringSource { + if node.Spec.Validator == nil || node.Spec.Validator.OperatorKeyring == nil { + return nil + } + return node.Spec.Validator.OperatorKeyring.Secret +} + +// sidecarSecurityContext locks the sidecar to non-root, read-only rootfs, +// no privilege escalation, all caps dropped, and the runtime's default +// seccomp profile. Scope is deliberately the sidecar only — applying the +// same to the seid main container is a larger blast-radius change owned +// by a different workstream. +func sidecarSecurityContext() *corev1.SecurityContext { + yes, no := true, false + uid := sidecarNonRootUID + gid := sidecarNonRootUID + return &corev1.SecurityContext{ + RunAsNonRoot: &yes, + RunAsUser: &uid, + RunAsGroup: &gid, + AllowPrivilegeEscalation: &no, + ReadOnlyRootFilesystem: &yes, + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + } +} diff --git a/internal/noderesource/noderesource_test.go b/internal/noderesource/noderesource_test.go index bbeca08e..0f02501d 100644 --- a/internal/noderesource/noderesource_test.go +++ b/internal/noderesource/noderesource_test.go @@ -904,3 +904,151 @@ func TestNodeKey_BothMountsCoexist(t *testing.T) { g.Expect(signingMount.MountPath).NotTo(Equal(nodeMount.MountPath), "signing-key and node-key mounts must target distinct paths under /sei/config/") } + +// --- Operator keyring (validator) --- + +func newValidatorNodeWithOperatorKeyring(name, namespace string) *seiv1alpha1.SeiNode { + return &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: seiv1alpha1.SeiNodeSpec{ + ChainID: "atlantic-2", + Image: "ghcr.io/sei-protocol/seid:latest", + Validator: &seiv1alpha1.ValidatorSpec{ + OperatorKeyring: &seiv1alpha1.OperatorKeyringSource{ + Secret: &seiv1alpha1.SecretOperatorKeyringSource{ + SecretName: "validator-0-opk", + KeyName: "node_admin", + PassphraseSecretRef: seiv1alpha1.PassphraseSecretRef{ + SecretName: "validator-0-opk-passphrase", + Key: "passphrase", + }, + }, + }, + }, + }, + } +} + +func TestOperatorKeyring_SecretVolumePresentOnPodTemplate(t *testing.T) { + g := NewWithT(t) + node := newValidatorNodeWithOperatorKeyring("validator-0", "default") + + sts := GenerateStatefulSet(node, platformtest.Config()) + + vol := findVolume(sts.Spec.Template.Spec.Volumes, operatorKeyringVolumeName) + g.Expect(vol).NotTo(BeNil(), "operator-keyring volume must be present on the StatefulSet pod template") + g.Expect(vol.Secret).NotTo(BeNil()) + g.Expect(vol.Secret.SecretName).To(Equal("validator-0-opk")) + g.Expect(*vol.Secret.DefaultMode).To(Equal(int32(0o400))) + g.Expect(vol.Secret.Items).To(BeNil(), + "operator-keyring projects the whole Secret as a directory under keyring-file/") +} + +func TestOperatorKeyring_SidecarContainerHasMountAndEnv(t *testing.T) { + g := NewWithT(t) + node := newValidatorNodeWithOperatorKeyring("validator-0", "default") + + sts := GenerateStatefulSet(node, platformtest.Config()) + sidecar := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") + g.Expect(sidecar).NotTo(BeNil(), "sei-sidecar init container must exist") + + mount := findVolumeMount(sidecar.VolumeMounts, operatorKeyringVolumeName) + g.Expect(mount).NotTo(BeNil(), "sidecar must mount operator-keyring volume") + g.Expect(mount.MountPath).To(Equal(dataDir + "/" + operatorKeyringDirName)) + g.Expect(mount.ReadOnly).To(BeTrue()) + g.Expect(mount.SubPath).To(BeEmpty(), + "operator-keyring is a directory mount, not subPath — sidecar needs the whole dir") + + var passphraseEnv *corev1.EnvVar + for i := range sidecar.Env { + if sidecar.Env[i].Name == keyringPassphraseEnvVar { + passphraseEnv = &sidecar.Env[i] + } + } + g.Expect(passphraseEnv).NotTo(BeNil(), "sidecar must have %s env injected", keyringPassphraseEnvVar) + g.Expect(passphraseEnv.ValueFrom).NotTo(BeNil()) + g.Expect(passphraseEnv.ValueFrom.SecretKeyRef).NotTo(BeNil()) + g.Expect(passphraseEnv.ValueFrom.SecretKeyRef.Name).To(Equal("validator-0-opk-passphrase")) + g.Expect(passphraseEnv.ValueFrom.SecretKeyRef.Key).To(Equal("passphrase")) +} + +func TestOperatorKeyring_SeidMainContainerHasNoMountOrEnv(t *testing.T) { + g := NewWithT(t) + node := newValidatorNodeWithOperatorKeyring("validator-0", "default") + + sts := GenerateStatefulSet(node, platformtest.Config()) + seid := findContainer(sts.Spec.Template.Spec.Containers, "seid") + g.Expect(seid).NotTo(BeNil(), "seid main container must exist") + + g.Expect(findVolumeMount(seid.VolumeMounts, operatorKeyringVolumeName)).To(BeNil(), + "seid main container must NOT mount operator-keyring — has no business reading operator-account material") + g.Expect(envValue(seid.Env, keyringPassphraseEnvVar)).To(BeEmpty(), + "seid main container must not carry the keyring passphrase env") + + seidInit := findInitContainer(sts.Spec.Template.Spec.InitContainers, "seid-init") + g.Expect(seidInit).NotTo(BeNil(), "seid-init container must exist") + g.Expect(findVolumeMount(seidInit.VolumeMounts, operatorKeyringVolumeName)).To(BeNil(), + "seid-init must NOT mount operator-keyring") +} + +func TestOperatorKeyring_Unset_NoVolumeOrMount(t *testing.T) { + g := NewWithT(t) + node := newSnapshotNode("snap-0", "default") + + sts := GenerateStatefulSet(node, platformtest.Config()) + + g.Expect(findVolume(sts.Spec.Template.Spec.Volumes, operatorKeyringVolumeName)).To(BeNil()) + + sidecar := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") + g.Expect(sidecar).NotTo(BeNil()) + g.Expect(findVolumeMount(sidecar.VolumeMounts, operatorKeyringVolumeName)).To(BeNil(), + "sidecar must not mount operator-keyring when OperatorKeyring is unset") + g.Expect(envValue(sidecar.Env, keyringPassphraseEnvVar)).To(BeEmpty()) +} + +func TestSidecarContainer_SecurityContext(t *testing.T) { + g := NewWithT(t) + node := newSnapshotNode("snap-0", "default") + + sts := GenerateStatefulSet(node, platformtest.Config()) + sidecar := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") + g.Expect(sidecar).NotTo(BeNil()) + g.Expect(sidecar.SecurityContext).NotTo(BeNil()) + + sc := sidecar.SecurityContext + g.Expect(*sc.RunAsNonRoot).To(BeTrue()) + g.Expect(*sc.RunAsUser).To(Equal(int64(65532))) + g.Expect(*sc.RunAsGroup).To(Equal(int64(65532))) + g.Expect(*sc.ReadOnlyRootFilesystem).To(BeTrue()) + g.Expect(*sc.AllowPrivilegeEscalation).To(BeFalse()) + g.Expect(sc.Capabilities).NotTo(BeNil()) + g.Expect(sc.Capabilities.Drop).To(ContainElement(corev1.Capability("ALL"))) + g.Expect(sc.SeccompProfile).NotTo(BeNil()) + g.Expect(sc.SeccompProfile.Type).To(Equal(corev1.SeccompProfileTypeRuntimeDefault)) +} + +func TestSeidMainContainer_NoSecurityContextChange(t *testing.T) { + g := NewWithT(t) + node := newSnapshotNode("snap-0", "default") + + sts := GenerateStatefulSet(node, platformtest.Config()) + seid := findContainer(sts.Spec.Template.Spec.Containers, "seid") + g.Expect(seid).NotTo(BeNil()) + // seid main container hardening is an out-of-scope, larger blast-radius + // change owned by a different workstream. + g.Expect(seid.SecurityContext).To(BeNil()) + + seidInit := findInitContainer(sts.Spec.Template.Spec.InitContainers, "seid-init") + g.Expect(seidInit).NotTo(BeNil()) + g.Expect(seidInit.SecurityContext).To(BeNil()) +} + +func TestPodSpec_FSGroup(t *testing.T) { + g := NewWithT(t) + node := newSnapshotNode("snap-0", "default") + + sts := GenerateStatefulSet(node, platformtest.Config()) + g.Expect(sts.Spec.Template.Spec.SecurityContext).NotTo(BeNil()) + g.Expect(*sts.Spec.Template.Spec.SecurityContext.FSGroup).To(Equal(int64(65532)), + "pod-level fsGroup must match sidecar UID so non-root sidecar can read 0o400 Secret mounts") +} diff --git a/internal/planner/bootstrap.go b/internal/planner/bootstrap.go index e8c1b80e..f38bd9a3 100644 --- a/internal/planner/bootstrap.go +++ b/internal/planner/bootstrap.go @@ -62,6 +62,12 @@ func buildBootstrapPlan( return nil, err } } + if needsValidateOperatorKeyring(node) { + if err := appendTask(task.TaskTypeValidateOperatorKeyring, + validateOperatorKeyringParams(node)); err != nil { + return nil, err + } + } // Phase 1: Deploy bootstrap infrastructure if err := appendTask(task.TaskTypeDeployBootstrapSvc, diff --git a/internal/planner/planner.go b/internal/planner/planner.go index f6b80461..ef6c0583 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -351,6 +351,38 @@ func validateNodeKeyParams(node *seiv1alpha1.SeiNode) any { } } +func needsValidateOperatorKeyring(node *seiv1alpha1.SeiNode) bool { + if node.Spec.Validator == nil || node.Spec.Validator.OperatorKeyring == nil { + return false + } + s := node.Spec.Validator.OperatorKeyring.Secret + return s != nil && s.SecretName != "" +} + +func validateOperatorKeyringParams(node *seiv1alpha1.SeiNode) any { + if !needsValidateOperatorKeyring(node) { + return nil + } + s := node.Spec.Validator.OperatorKeyring.Secret + // Defaults mirror the CRD kubebuilder:default markers so plans built + // from a partially-defaulted in-memory spec still get the right values. + keyName := s.KeyName + if keyName == "" { + keyName = "node_admin" + } + passphraseKey := s.PassphraseSecretRef.Key + if passphraseKey == "" { + passphraseKey = "passphrase" + } + return &task.ValidateOperatorKeyringParams{ + SecretName: s.SecretName, + KeyName: keyName, + PassphraseSecretName: s.PassphraseSecretRef.SecretName, + PassphraseSecretKey: passphraseKey, + Namespace: node.Namespace, + } +} + // isGenesisCeremonyNode returns true when the node participates in a group genesis ceremony. func isGenesisCeremonyNode(node *seiv1alpha1.SeiNode) bool { return node.Spec.Validator != nil && node.Spec.Validator.GenesisCeremony != nil @@ -504,6 +536,9 @@ func buildBasePlan( if needsValidateNodeKey(node) { prog = append(prog, task.TaskTypeValidateNodeKey) } + if needsValidateOperatorKeyring(node) { + prog = append(prog, task.TaskTypeValidateOperatorKeyring) + } prog = append(prog, task.TaskTypeApplyStatefulSet, task.TaskTypeApplyService) prog = append(prog, sidecarProg...) @@ -551,6 +586,8 @@ func paramsForTaskType( return validateSigningKeyParams(node) case task.TaskTypeValidateNodeKey: return validateNodeKeyParams(node) + case task.TaskTypeValidateOperatorKeyring: + return validateOperatorKeyringParams(node) // Sidecar tasks case TaskSnapshotRestore: diff --git a/internal/planner/validator_test.go b/internal/planner/validator_test.go index 27a270d0..ca744c77 100644 --- a/internal/planner/validator_test.go +++ b/internal/planner/validator_test.go @@ -244,6 +244,135 @@ func TestValidatorPlanner_BuildPlan_IdentityInsertsValidateTasks_Base(t *testing } } +func TestValidatorPlanner_BuildPlan_OperatorKeyringInsertsValidateTask_Base(t *testing.T) { + node := &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: "validator-0", Namespace: "pacific-1"}, + Spec: seiv1alpha1.SeiNodeSpec{ + ChainID: "pacific-1", + Image: "seid:v6.4.1", + Validator: &seiv1alpha1.ValidatorSpec{ + SigningKey: &seiv1alpha1.SigningKeySource{ + Secret: &seiv1alpha1.SecretSigningKeySource{SecretName: "validator-0-key"}, + }, + NodeKey: &seiv1alpha1.NodeKeySource{ + Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: "validator-0-nodekey"}, + }, + OperatorKeyring: &seiv1alpha1.OperatorKeyringSource{ + Secret: &seiv1alpha1.SecretOperatorKeyringSource{ + SecretName: "validator-0-opk", + KeyName: "node_admin", + PassphraseSecretRef: seiv1alpha1.PassphraseSecretRef{ + SecretName: "validator-0-opk-passphrase", + Key: "passphrase", + }, + }, + }, + }, + }, + } + plan, err := (&validatorPlanner{}).BuildPlan(node) + if err != nil { + t.Fatalf("BuildPlan: %v", err) + } + + nodeKeyIdx := indexOfTaskType(plan, task.TaskTypeValidateNodeKey) + keyringIdx := indexOfTaskType(plan, task.TaskTypeValidateOperatorKeyring) + stsIdx := indexOfTaskType(plan, task.TaskTypeApplyStatefulSet) + if keyringIdx < 0 { + t.Fatalf("plan must contain %s; got %v", task.TaskTypeValidateOperatorKeyring, taskTypes(plan)) + } + if nodeKeyIdx >= keyringIdx || keyringIdx >= stsIdx { + t.Fatalf("expected ordering nodeKey(%d) < operatorKeyring(%d) < apply-statefulset(%d); got %v", + nodeKeyIdx, keyringIdx, stsIdx, taskTypes(plan)) + } + + for _, pt := range plan.Tasks { + if pt.Type != task.TaskTypeValidateOperatorKeyring { + continue + } + raw := string(pt.Params.Raw) + if !strings.Contains(raw, "validator-0-opk") { + t.Fatalf("validate-operator-keyring params must reference data secretName; got %q", raw) + } + if !strings.Contains(raw, "validator-0-opk-passphrase") { + t.Fatalf("validate-operator-keyring params must reference passphrase secretName; got %q", raw) + } + if !strings.Contains(raw, "node_admin") { + t.Fatalf("validate-operator-keyring params must reference keyName; got %q", raw) + } + } +} + +func TestValidatorPlanner_BuildPlan_OperatorKeyringInsertsValidateTask_Bootstrap(t *testing.T) { + node := &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: "validator-0", Namespace: "pacific-1"}, + Spec: seiv1alpha1.SeiNodeSpec{ + ChainID: "pacific-1", + Image: "seid:v6.4.1", + Validator: &seiv1alpha1.ValidatorSpec{ + Snapshot: &seiv1alpha1.SnapshotSource{ + BootstrapImage: "ghcr.io/sei/bootstrap:v1", + S3: &seiv1alpha1.S3SnapshotSource{TargetHeight: 12345678}, + }, + SigningKey: &seiv1alpha1.SigningKeySource{ + Secret: &seiv1alpha1.SecretSigningKeySource{SecretName: "validator-0-key"}, + }, + NodeKey: &seiv1alpha1.NodeKeySource{ + Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: "validator-0-nodekey"}, + }, + OperatorKeyring: &seiv1alpha1.OperatorKeyringSource{ + Secret: &seiv1alpha1.SecretOperatorKeyringSource{ + SecretName: "validator-0-opk", + PassphraseSecretRef: seiv1alpha1.PassphraseSecretRef{ + SecretName: "validator-0-opk-passphrase", + }, + }, + }, + }, + }, + } + plan, err := (&validatorPlanner{}).BuildPlan(node) + if err != nil { + t.Fatalf("BuildPlan: %v", err) + } + nodeKeyIdx := indexOfTaskType(plan, task.TaskTypeValidateNodeKey) + keyringIdx := indexOfTaskType(plan, task.TaskTypeValidateOperatorKeyring) + deployJobIdx := indexOfTaskType(plan, task.TaskTypeDeployBootstrapJob) + if keyringIdx < 0 { + t.Fatalf("plan must contain %s; got %v", task.TaskTypeValidateOperatorKeyring, taskTypes(plan)) + } + if nodeKeyIdx >= keyringIdx || keyringIdx >= deployJobIdx { + t.Fatalf("expected ordering nodeKey(%d) < operatorKeyring(%d) < deploy-bootstrap-job(%d); got %v", + nodeKeyIdx, keyringIdx, deployJobIdx, taskTypes(plan)) + } +} + +func TestValidatorPlanner_BuildPlan_NoOperatorKeyringOmitsValidateTask(t *testing.T) { + node := &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: "validator-0", Namespace: "pacific-1"}, + Spec: seiv1alpha1.SeiNodeSpec{ + ChainID: "pacific-1", + Image: "seid:v6.4.1", + Validator: &seiv1alpha1.ValidatorSpec{ + SigningKey: &seiv1alpha1.SigningKeySource{ + Secret: &seiv1alpha1.SecretSigningKeySource{SecretName: "validator-0-key"}, + }, + NodeKey: &seiv1alpha1.NodeKeySource{ + Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: "validator-0-nodekey"}, + }, + }, + }, + } + plan, err := (&validatorPlanner{}).BuildPlan(node) + if err != nil { + t.Fatalf("BuildPlan: %v", err) + } + if idx := indexOfTaskType(plan, task.TaskTypeValidateOperatorKeyring); idx >= 0 { + t.Fatalf("plan must not contain %s when OperatorKeyring is unset; got %v at index %d", + task.TaskTypeValidateOperatorKeyring, taskTypes(plan), idx) + } +} + func TestValidatorPlanner_BuildPlan_NoSigningKeyOmitsValidateTask(t *testing.T) { node := &seiv1alpha1.SeiNode{ ObjectMeta: metav1.ObjectMeta{Name: "validator-0", Namespace: "pacific-1"}, diff --git a/internal/task/bootstrap_resources.go b/internal/task/bootstrap_resources.go index cf8cf700..09ca3cf6 100644 --- a/internal/task/bootstrap_resources.go +++ b/internal/task/bootstrap_resources.go @@ -39,9 +39,10 @@ func BootstrapLabels(node *seiv1alpha1.SeiNode) map[string]string { // GenerateBootstrapJob creates the batch Job that runs seid with --halt-height // to populate a PVC before the StatefulSet takes over. // -// The pod-spec must never carry consensus signing material — bootstrap pods -// are physically incapable of signing because no validator key file is on -// their filesystem. assertNoSigningKeyOnBootstrapPod is the runtime guard. +// The pod-spec must never carry validator-owned credentials — bootstrap pods +// are physically incapable of signing because no signing-key, operator +// keyring, or keyring passphrase is on their filesystem or in their env. +// assertNoValidatorSecretsOnBootstrapPod is the runtime guard. func GenerateBootstrapJob( node *seiv1alpha1.SeiNode, snap *seiv1alpha1.SnapshotSource, @@ -53,7 +54,7 @@ func GenerateBootstrapJob( labels := BootstrapLabels(node) podSpec := buildBootstrapPodSpec(node, snap, platformCfg) - if err := assertNoSigningKeyOnBootstrapPod(node, &podSpec); err != nil { + if err := assertNoValidatorSecretsOnBootstrapPod(node, &podSpec); err != nil { return nil, err } @@ -327,22 +328,73 @@ func JobFailureReason(job *batchv1.Job) string { return "bootstrap job failed" } -// assertNoSigningKeyOnBootstrapPod fails closed if a future refactor -// accidentally lands the validator's signing-key Secret on the bootstrap -// pod-spec. The bootstrap path must never carry consensus signing material. -func assertNoSigningKeyOnBootstrapPod(node *seiv1alpha1.SeiNode, spec *corev1.PodSpec) error { - if node.Spec.Validator == nil || - node.Spec.Validator.SigningKey == nil || - node.Spec.Validator.SigningKey.Secret == nil { +// assertNoValidatorSecretsOnBootstrapPod fails closed if a future refactor +// accidentally lands ANY validator-owned Secret (signing key, operator +// keyring, or keyring passphrase) on the bootstrap pod-spec. Bootstrap +// pods run `seid start --halt-height` and must be physically incapable of +// signing: no signing-related material on their filesystem, no operator +// keyring material in their env. +// +// node-key is deliberately excluded — it carries no signing authority; +// bootstrap mounting it would be a design bug elsewhere, not a slashing +// risk. +func assertNoValidatorSecretsOnBootstrapPod(node *seiv1alpha1.SeiNode, spec *corev1.PodSpec) error { + if node.Spec.Validator == nil { + return nil + } + forbiddens := forbiddenBootstrapSecrets(node) + if len(forbiddens) == 0 { return nil } - secretName := node.Spec.Validator.SigningKey.Secret.SecretName + for _, v := range spec.Volumes { - if v.Secret != nil && v.Secret.SecretName == secretName { - return fmt.Errorf("bootstrap pod-spec for %s/%s references signing-key Secret %q on volume %q; "+ - "bootstrap pods must never carry consensus signing material", - node.Namespace, node.Name, secretName, v.Name) + if v.Secret == nil { + continue + } + for _, f := range forbiddens { + if v.Secret.SecretName == f.name { + return fmt.Errorf("bootstrap pod-spec for %s/%s references %s Secret %q on volume %q; "+ + "bootstrap pods must never carry validator-owned credentials", + node.Namespace, node.Name, f.kind, f.name, v.Name) + } + } + } + + // Env injection is a separate leakage path — kubelet resolves + // valueFrom.secretKeyRef regardless of volume mounts. + allContainers := make([]corev1.Container, 0, len(spec.Containers)+len(spec.InitContainers)) + allContainers = append(allContainers, spec.Containers...) + allContainers = append(allContainers, spec.InitContainers...) + for _, c := range allContainers { + for _, ev := range c.Env { + if ev.ValueFrom == nil || ev.ValueFrom.SecretKeyRef == nil { + continue + } + for _, f := range forbiddens { + if ev.ValueFrom.SecretKeyRef.Name == f.name { + return fmt.Errorf("bootstrap pod-spec for %s/%s references %s Secret %q in container %q env; "+ + "bootstrap pods must never carry validator-owned credentials", + node.Namespace, node.Name, f.kind, f.name, c.Name) + } + } } } + return nil } + +type forbiddenSecret struct{ name, kind string } + +func forbiddenBootstrapSecrets(node *seiv1alpha1.SeiNode) []forbiddenSecret { + var out []forbiddenSecret + if sk := node.Spec.Validator.SigningKey; sk != nil && sk.Secret != nil { + out = append(out, forbiddenSecret{sk.Secret.SecretName, "signing-key"}) + } + if ok := node.Spec.Validator.OperatorKeyring; ok != nil && ok.Secret != nil { + out = append(out, + forbiddenSecret{ok.Secret.SecretName, "operator-keyring"}, + forbiddenSecret{ok.Secret.PassphraseSecretRef.SecretName, "operator-keyring-passphrase"}, + ) + } + return out +} diff --git a/internal/task/bootstrap_resources_test.go b/internal/task/bootstrap_resources_test.go new file mode 100644 index 00000000..624e176a --- /dev/null +++ b/internal/task/bootstrap_resources_test.go @@ -0,0 +1,194 @@ +package task + +import ( + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1" +) + +const ( + bootstrapTestPassphraseSecret = "validator-0-passphrase" + bootstrapTestNodeKeySecret = "validator-0-nodekey" + bootstrapTestSidecarContainer = "sei-sidecar" + bootstrapTestDataVolumeName = "data" +) + +func validatorNodeWithSecrets(signingKeySecret, operatorKeyringSecret, passphraseSecret string) *seiv1alpha1.SeiNode { + node := &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: opkValidatorName, Namespace: opkNs}, + Spec: seiv1alpha1.SeiNodeSpec{ + Validator: &seiv1alpha1.ValidatorSpec{}, + }, + } + if signingKeySecret != "" { + node.Spec.Validator.SigningKey = &seiv1alpha1.SigningKeySource{ + Secret: &seiv1alpha1.SecretSigningKeySource{SecretName: signingKeySecret}, + } + } + if operatorKeyringSecret != "" { + node.Spec.Validator.OperatorKeyring = &seiv1alpha1.OperatorKeyringSource{ + Secret: &seiv1alpha1.SecretOperatorKeyringSource{ + SecretName: operatorKeyringSecret, + PassphraseSecretRef: seiv1alpha1.PassphraseSecretRef{ + SecretName: passphraseSecret, + Key: opkPassphraseKey, + }, + }, + } + } + return node +} + +func TestAssertNoValidatorSecretsOnBootstrapPod_NoValidator(t *testing.T) { + node := &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: "fn-0", Namespace: opkNs}, + } + if err := assertNoValidatorSecretsOnBootstrapPod(node, &corev1.PodSpec{}); err != nil { + t.Fatalf("expected nil error for non-validator node, got %v", err) + } +} + +func TestAssertNoValidatorSecretsOnBootstrapPod_CleanPodSpec(t *testing.T) { + node := validatorNodeWithSecrets("sk", "opk", "opk-pass") + spec := &corev1.PodSpec{ + Volumes: []corev1.Volume{ + {Name: bootstrapTestDataVolumeName, VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data-validator-0"}, + }}, + }, + } + if err := assertNoValidatorSecretsOnBootstrapPod(node, spec); err != nil { + t.Fatalf("expected nil error for clean bootstrap pod-spec, got %v", err) + } +} + +func TestAssertNoValidatorSecretsOnBootstrapPod_SigningKeyVolume_Rejects(t *testing.T) { + node := validatorNodeWithSecrets("validator-0-signing", "opk", "opk-pass") + spec := &corev1.PodSpec{ + Volumes: []corev1.Volume{ + {Name: "signing-key", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: "validator-0-signing"}, + }}, + }, + } + err := assertNoValidatorSecretsOnBootstrapPod(node, spec) + if err == nil { + t.Fatal("expected error for signing-key volume on bootstrap pod, got nil") + } + if !strings.Contains(err.Error(), "signing-key") { + t.Fatalf("expected error to mention signing-key, got: %v", err) + } +} + +func TestAssertNoValidatorSecretsOnBootstrapPod_OperatorKeyringVolume_Rejects(t *testing.T) { + node := validatorNodeWithSecrets("sk", "validator-0-opk", "opk-pass") + spec := &corev1.PodSpec{ + Volumes: []corev1.Volume{ + {Name: "operator-keyring", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: "validator-0-opk"}, + }}, + }, + } + err := assertNoValidatorSecretsOnBootstrapPod(node, spec) + if err == nil { + t.Fatal("expected error for operator-keyring volume on bootstrap pod, got nil") + } + if !strings.Contains(err.Error(), "operator-keyring") { + t.Fatalf("expected error to mention operator-keyring, got: %v", err) + } +} + +func TestAssertNoValidatorSecretsOnBootstrapPod_PassphraseVolume_Rejects(t *testing.T) { + node := validatorNodeWithSecrets("sk", "opk", bootstrapTestPassphraseSecret) + spec := &corev1.PodSpec{ + Volumes: []corev1.Volume{ + {Name: "passphrase-vol", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: bootstrapTestPassphraseSecret}, + }}, + }, + } + err := assertNoValidatorSecretsOnBootstrapPod(node, spec) + if err == nil { + t.Fatal("expected error for passphrase volume on bootstrap pod, got nil") + } + if !strings.Contains(err.Error(), "operator-keyring-passphrase") { + t.Fatalf("expected error to mention operator-keyring-passphrase, got: %v", err) + } +} + +func TestAssertNoValidatorSecretsOnBootstrapPod_PassphraseEnv_Rejects(t *testing.T) { + node := validatorNodeWithSecrets("sk", "opk", bootstrapTestPassphraseSecret) + spec := &corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: bootstrapTestSidecarContainer, + Env: []corev1.EnvVar{ + {Name: "SEI_KEYRING_PASSPHRASE", ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: bootstrapTestPassphraseSecret}, + Key: opkPassphraseKey, + }, + }}, + }, + }, + }, + } + err := assertNoValidatorSecretsOnBootstrapPod(node, spec) + if err == nil { + t.Fatal("expected error for passphrase env on bootstrap pod, got nil") + } + if !strings.Contains(err.Error(), "operator-keyring-passphrase") || !strings.Contains(err.Error(), bootstrapTestSidecarContainer) { + t.Fatalf("expected error to mention passphrase kind and container name, got: %v", err) + } +} + +func TestAssertNoValidatorSecretsOnBootstrapPod_PassphraseEnvInInitContainer_Rejects(t *testing.T) { + node := validatorNodeWithSecrets("sk", "opk", bootstrapTestPassphraseSecret) + spec := &corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: bootstrapTestSidecarContainer, + Env: []corev1.EnvVar{ + {Name: "SEI_KEYRING_PASSPHRASE", ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: bootstrapTestPassphraseSecret}, + Key: opkPassphraseKey, + }, + }}, + }, + }, + }, + } + if err := assertNoValidatorSecretsOnBootstrapPod(node, spec); err == nil { + t.Fatal("expected error for passphrase env on init container, got nil") + } +} + +func TestAssertNoValidatorSecretsOnBootstrapPod_NodeKeyExcluded(t *testing.T) { + node := &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: opkValidatorName, Namespace: opkNs}, + Spec: seiv1alpha1.SeiNodeSpec{ + Validator: &seiv1alpha1.ValidatorSpec{ + NodeKey: &seiv1alpha1.NodeKeySource{ + Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: bootstrapTestNodeKeySecret}, + }, + }, + }, + } + // node-key Secret on bootstrap is a design bug elsewhere but NOT a + // slashing risk — this assertion intentionally does not catch it. + spec := &corev1.PodSpec{ + Volumes: []corev1.Volume{ + {Name: "node-key", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: bootstrapTestNodeKeySecret}, + }}, + }, + } + if err := assertNoValidatorSecretsOnBootstrapPod(node, spec); err != nil { + t.Fatalf("node-key is deliberately not part of the assertion, got: %v", err) + } +} diff --git a/internal/task/task.go b/internal/task/task.go index ebc3a999..5c45e909 100644 --- a/internal/task/task.go +++ b/internal/task/task.go @@ -204,13 +204,14 @@ var registry = map[string]taskDeserializer{ TaskTypeCollectAndSetPeers: deserializeCollectAndSetPeers, // Controller-side infrastructure tasks - TaskTypeEnsureDataPVC: deserializeEnsureDataPVC, - TaskTypeApplyStatefulSet: deserializeApplyStatefulSet, - TaskTypeApplyService: deserializeApplyService, - TaskTypeReplacePod: deserializeReplacePod, - TaskTypeObserveImage: deserializeObserveImage, - TaskTypeValidateSigningKey: deserializeValidateSigningKey, - TaskTypeValidateNodeKey: deserializeValidateNodeKey, + TaskTypeEnsureDataPVC: deserializeEnsureDataPVC, + TaskTypeApplyStatefulSet: deserializeApplyStatefulSet, + TaskTypeApplyService: deserializeApplyService, + TaskTypeReplacePod: deserializeReplacePod, + TaskTypeObserveImage: deserializeObserveImage, + TaskTypeValidateSigningKey: deserializeValidateSigningKey, + TaskTypeValidateNodeKey: deserializeValidateNodeKey, + TaskTypeValidateOperatorKeyring: deserializeValidateOperatorKeyring, // Controller-side bootstrap tasks TaskTypeDeployBootstrapSvc: deserializeBootstrapService, diff --git a/internal/task/validate_operator_keyring.go b/internal/task/validate_operator_keyring.go new file mode 100644 index 00000000..bb730cb4 --- /dev/null +++ b/internal/task/validate_operator_keyring.go @@ -0,0 +1,183 @@ +package task + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1" +) + +const TaskTypeValidateOperatorKeyring = "validate-operator-keyring" + +// Cosmos SDK file-backend keyring layout: each entry materializes as +// ".info" (encrypted protobuf-marshaled LocalInfo) plus one or more +// ".address" name→address index files. +const ( + keyringInfoSuffix = ".info" + keyringAddressSuffix = ".address" +) + +type ValidateOperatorKeyringParams struct { + SecretName string `json:"secretName"` + KeyName string `json:"keyName"` + PassphraseSecretName string `json:"passphraseSecretName"` + PassphraseSecretKey string `json:"passphraseSecretKey"` + Namespace string `json:"namespace"` +} + +type validateOperatorKeyringExecution struct { + taskBase + params ValidateOperatorKeyringParams + cfg ExecutionConfig +} + +func deserializeValidateOperatorKeyring(id string, params json.RawMessage, cfg ExecutionConfig) (TaskExecution, error) { + var p ValidateOperatorKeyringParams + if len(params) > 0 { + if err := json.Unmarshal(params, &p); err != nil { + return nil, fmt.Errorf("deserializing validate-operator-keyring params: %w", err) + } + } + return &validateOperatorKeyringExecution{ + taskBase: taskBase{id: id, status: ExecutionRunning}, + params: p, + cfg: cfg, + }, nil +} + +func (e *validateOperatorKeyringExecution) Execute(ctx context.Context) error { + node, err := ResourceAs[*seiv1alpha1.SeiNode](e.cfg) + if err != nil { + return Terminal(err) + } + + err = e.validate(ctx, node) + switch { + case err == nil: + setOperatorKeyringCondition(node, metav1.ConditionTrue, + seiv1alpha1.ReasonOperatorKeyringValidated, + fmt.Sprintf("Secret pair (%q, %q) passes operator-keyring validation", + e.params.SecretName, e.params.PassphraseSecretName)) + e.complete() + return nil + case isTerminal(err): + setOperatorKeyringCondition(node, metav1.ConditionFalse, seiv1alpha1.ReasonOperatorKeyringInvalid, err.Error()) + return err + default: + setOperatorKeyringCondition(node, metav1.ConditionFalse, seiv1alpha1.ReasonOperatorKeyringNotReady, err.Error()) + return nil + } +} + +func (e *validateOperatorKeyringExecution) Status(_ context.Context) ExecutionStatus { + return e.DefaultStatus() +} + +// validate checks the shape of both Secrets. It deliberately does NOT +// attempt to decrypt the keyring with the passphrase — running the Cosmos +// SDK keyring backend inside the controller process would expand the +// controller's TCB. Decryption is the sidecar's startup smoke test. +func (e *validateOperatorKeyringExecution) validate(ctx context.Context, node *seiv1alpha1.SeiNode) error { + if e.params.SecretName == "" { + return Terminal(fmt.Errorf("validate-operator-keyring: secretName is empty")) + } + if e.params.PassphraseSecretName == "" { + return Terminal(fmt.Errorf("validate-operator-keyring: passphraseSecretName is empty")) + } + if e.params.PassphraseSecretKey == "" { + return Terminal(fmt.Errorf("validate-operator-keyring: passphraseSecretKey is empty")) + } + + keyring, err := e.getSecret(ctx, e.params.SecretName, node.Namespace) + if err != nil { + return err + } + if err := validateKeyringShape(keyring, e.params.KeyName); err != nil { + return err + } + + passphrase, err := e.getSecret(ctx, e.params.PassphraseSecretName, node.Namespace) + if err != nil { + return err + } + return validatePassphraseShape(passphrase, e.params.PassphraseSecretKey) +} + +func (e *validateOperatorKeyringExecution) getSecret(ctx context.Context, name, namespace string) (*corev1.Secret, error) { + secret := &corev1.Secret{} + key := types.NamespacedName{Name: name, Namespace: namespace} + if err := e.cfg.KubeClient.Get(ctx, key, secret); err != nil { + if apierrors.IsNotFound(err) { + return nil, fmt.Errorf("secret %q not found in namespace %q", name, namespace) + } + return nil, fmt.Errorf("getting Secret %q: %w", name, err) + } + if secret.DeletionTimestamp != nil { + return nil, fmt.Errorf("secret %q is being deleted (deletionTimestamp=%s)", name, secret.DeletionTimestamp) + } + return secret, nil +} + +// validateKeyringShape walks Secret data keys looking for the Cosmos SDK +// file-backend layout. Empty Secrets and Secrets missing either suffix +// are operator-fixable defects (Terminal). +func validateKeyringShape(secret *corev1.Secret, keyName string) error { + var infoKeys, addressKeys []string + for k, v := range secret.Data { + switch { + case strings.HasSuffix(k, keyringInfoSuffix): + if len(v) == 0 { + return Terminal(fmt.Errorf("secret %q data key %q is empty; file-keyring info blobs must be non-empty", + secret.Name, k)) + } + infoKeys = append(infoKeys, k) + case strings.HasSuffix(k, keyringAddressSuffix): + addressKeys = append(addressKeys, k) + } + } + if len(infoKeys) == 0 { + return Terminal(fmt.Errorf("secret %q has no %q data keys; file-keyring requires at least one encrypted key blob", + secret.Name, "*"+keyringInfoSuffix)) + } + if len(addressKeys) == 0 { + return Terminal(fmt.Errorf("secret %q has no %q data keys; file-keyring requires at least one name→address index", + secret.Name, "*"+keyringAddressSuffix)) + } + if keyName != "" { + want := keyName + keyringInfoSuffix + if _, ok := secret.Data[want]; !ok { + return Terminal(fmt.Errorf("secret %q is missing data key %q for keyName %q", + secret.Name, want, keyName)) + } + } + return nil +} + +func validatePassphraseShape(secret *corev1.Secret, dataKey string) error { + v, ok := secret.Data[dataKey] + if !ok { + return Terminal(fmt.Errorf("passphrase Secret %q missing data key %q", secret.Name, dataKey)) + } + if len(v) == 0 { + return Terminal(fmt.Errorf("passphrase Secret %q data key %q is empty", secret.Name, dataKey)) + } + return nil +} + +func setOperatorKeyringCondition(node *seiv1alpha1.SeiNode, status metav1.ConditionStatus, reason, message string) { + meta.SetStatusCondition(&node.Status.Conditions, metav1.Condition{ + Type: seiv1alpha1.ConditionOperatorKeyringReady, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: node.Generation, + }) +} diff --git a/internal/task/validate_operator_keyring_test.go b/internal/task/validate_operator_keyring_test.go new file mode 100644 index 00000000..b0671975 --- /dev/null +++ b/internal/task/validate_operator_keyring_test.go @@ -0,0 +1,333 @@ +package task + +import ( + "context" + "encoding/json" + "testing" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1" +) + +const ( + opkNs = "default" + opkValidatorName = "validator-0" + opkChainID = "atlantic-2" + opkImage = "sei:v1.0.0" + opkPassphraseKey = "passphrase" + opkAddressDataKey = "deadbeef.address" + opkKeyringSecret = "opk-data" + opkPassphrSecret = "opk-passphrase" + opkDefaultKeyName = "node_admin" +) + +func operatorKeyringNode(keyringSecret, passphraseSecret, keyName string) *seiv1alpha1.SeiNode { //nolint:unparam // test helper designed for reuse + return &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{ + Name: opkValidatorName, + Namespace: opkNs, + UID: "uid-validator-0", + }, + Spec: seiv1alpha1.SeiNodeSpec{ + ChainID: opkChainID, + Image: opkImage, + Validator: &seiv1alpha1.ValidatorSpec{ + OperatorKeyring: &seiv1alpha1.OperatorKeyringSource{ + Secret: &seiv1alpha1.SecretOperatorKeyringSource{ + SecretName: keyringSecret, + KeyName: keyName, + PassphraseSecretRef: seiv1alpha1.PassphraseSecretRef{ + SecretName: passphraseSecret, + Key: opkPassphraseKey, + }, + }, + }, + }, + }, + } +} + +func validKeyringSecret(name, ns, keyName string) *corev1.Secret { //nolint:unparam // test helper designed for reuse + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Data: map[string][]byte{ + keyName + ".info": []byte("encrypted-key-blob"), + opkAddressDataKey: []byte("addr-index"), + }, + } +} + +func validPassphraseSecret(name, ns string) *corev1.Secret { //nolint:unparam // test helper designed for reuse + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Data: map[string][]byte{opkPassphraseKey: []byte("hunter2")}, + } +} + +func newValidateOperatorKeyringExec(t *testing.T, node *seiv1alpha1.SeiNode, objs ...client.Object) (TaskExecution, client.Client) { + t.Helper() + s := validatorScheme(t) + c := fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).Build() + cfg := ExecutionConfig{ + KubeClient: c, + APIReader: c, + Scheme: s, + Resource: node, + } + src := node.Spec.Validator.OperatorKeyring.Secret + params := ValidateOperatorKeyringParams{ + SecretName: src.SecretName, + KeyName: src.KeyName, + PassphraseSecretName: src.PassphraseSecretRef.SecretName, + PassphraseSecretKey: src.PassphraseSecretRef.Key, + Namespace: node.Namespace, + } + raw, _ := json.Marshal(params) + exec, err := deserializeValidateOperatorKeyring("validate-op-1", raw, cfg) + if err != nil { + t.Fatal(err) + } + return exec, c +} + +func operatorKeyringReasonFor(node *seiv1alpha1.SeiNode) string { + for _, c := range node.Status.Conditions { + if c.Type == seiv1alpha1.ConditionOperatorKeyringReady { + return c.Reason + } + } + return "" +} + +func operatorKeyringConditionFor(node *seiv1alpha1.SeiNode) *metav1.Condition { + for i := range node.Status.Conditions { + if node.Status.Conditions[i].Type == seiv1alpha1.ConditionOperatorKeyringReady { + return &node.Status.Conditions[i] + } + } + return nil +} + +// --- Happy path --- + +func TestValidateOperatorKeyring_Valid_Completes(t *testing.T) { + g := NewWithT(t) + node := operatorKeyringNode(opkKeyringSecret, opkPassphrSecret, opkDefaultKeyName) + exec, _ := newValidateOperatorKeyringExec(t, node, + validKeyringSecret(opkKeyringSecret, opkNs, opkDefaultKeyName), + validPassphraseSecret(opkPassphrSecret, opkNs), + ) + + g.Expect(exec.Execute(context.Background())).To(Succeed()) + g.Expect(exec.Status(context.Background())).To(Equal(ExecutionComplete)) + g.Expect(operatorKeyringReasonFor(node)).To(Equal(seiv1alpha1.ReasonOperatorKeyringValidated)) + + cond := operatorKeyringConditionFor(node) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) +} + +// --- Transient (Secret not yet applied) --- + +func TestValidateOperatorKeyring_KeyringSecretNotFound_Transient(t *testing.T) { + g := NewWithT(t) + node := operatorKeyringNode(opkKeyringSecret, opkPassphrSecret, opkDefaultKeyName) + exec, _ := newValidateOperatorKeyringExec(t, node, + validPassphraseSecret(opkPassphrSecret, opkNs), + ) + + err := exec.Execute(context.Background()) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(exec.Status(context.Background())).To(Equal(ExecutionRunning)) + g.Expect(operatorKeyringReasonFor(node)).To(Equal(seiv1alpha1.ReasonOperatorKeyringNotReady)) +} + +func TestValidateOperatorKeyring_PassphraseSecretNotFound_Transient(t *testing.T) { + g := NewWithT(t) + node := operatorKeyringNode(opkKeyringSecret, opkPassphrSecret, opkDefaultKeyName) + exec, _ := newValidateOperatorKeyringExec(t, node, + validKeyringSecret(opkKeyringSecret, opkNs, opkDefaultKeyName), + ) + + err := exec.Execute(context.Background()) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(exec.Status(context.Background())).To(Equal(ExecutionRunning)) + g.Expect(operatorKeyringReasonFor(node)).To(Equal(seiv1alpha1.ReasonOperatorKeyringNotReady)) +} + +func TestValidateOperatorKeyring_SecretTerminating_Transient(t *testing.T) { + g := NewWithT(t) + node := operatorKeyringNode(opkKeyringSecret, opkPassphrSecret, opkDefaultKeyName) + keyring := validKeyringSecret(opkKeyringSecret, opkNs, opkDefaultKeyName) + keyring.Finalizers = []string{"example.com/protect"} + now := metav1.Now() + keyring.DeletionTimestamp = &now + exec, _ := newValidateOperatorKeyringExec(t, node, keyring, + validPassphraseSecret(opkPassphrSecret, opkNs), + ) + + err := exec.Execute(context.Background()) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(exec.Status(context.Background())).To(Equal(ExecutionRunning)) + g.Expect(operatorKeyringReasonFor(node)).To(Equal(seiv1alpha1.ReasonOperatorKeyringNotReady)) +} + +// --- Terminal (operator must fix) --- + +func TestValidateOperatorKeyring_NoInfoKeys_Terminal(t *testing.T) { + g := NewWithT(t) + node := operatorKeyringNode(opkKeyringSecret, opkPassphrSecret, opkDefaultKeyName) + keyring := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: opkKeyringSecret, Namespace: opkNs}, + Data: map[string][]byte{opkAddressDataKey: []byte("addr")}, + } + exec, _ := newValidateOperatorKeyringExec(t, node, keyring, + validPassphraseSecret(opkPassphrSecret, opkNs), + ) + + err := exec.Execute(context.Background()) + g.Expect(err).To(HaveOccurred()) + g.Expect(isTerminal(err)).To(BeTrue()) + g.Expect(operatorKeyringReasonFor(node)).To(Equal(seiv1alpha1.ReasonOperatorKeyringInvalid)) + g.Expect(operatorKeyringConditionFor(node).Message).To(ContainSubstring(".info")) +} + +func TestValidateOperatorKeyring_NoAddressKeys_Terminal(t *testing.T) { + g := NewWithT(t) + node := operatorKeyringNode(opkKeyringSecret, opkPassphrSecret, opkDefaultKeyName) + keyring := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: opkKeyringSecret, Namespace: opkNs}, + Data: map[string][]byte{"node_admin.info": []byte("blob")}, + } + exec, _ := newValidateOperatorKeyringExec(t, node, keyring, + validPassphraseSecret(opkPassphrSecret, opkNs), + ) + + err := exec.Execute(context.Background()) + g.Expect(err).To(HaveOccurred()) + g.Expect(isTerminal(err)).To(BeTrue()) + g.Expect(operatorKeyringConditionFor(node).Message).To(ContainSubstring(".address")) +} + +func TestValidateOperatorKeyring_EmptyInfoBlob_Terminal(t *testing.T) { + g := NewWithT(t) + node := operatorKeyringNode(opkKeyringSecret, opkPassphrSecret, opkDefaultKeyName) + keyring := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: opkKeyringSecret, Namespace: opkNs}, + Data: map[string][]byte{ + "node_admin.info": {}, + opkAddressDataKey: []byte("addr"), + }, + } + exec, _ := newValidateOperatorKeyringExec(t, node, keyring, + validPassphraseSecret(opkPassphrSecret, opkNs), + ) + + err := exec.Execute(context.Background()) + g.Expect(err).To(HaveOccurred()) + g.Expect(isTerminal(err)).To(BeTrue()) + g.Expect(operatorKeyringConditionFor(node).Message).To(ContainSubstring("empty")) +} + +func TestValidateOperatorKeyring_NamedKeyMissing_Terminal(t *testing.T) { + g := NewWithT(t) + node := operatorKeyringNode(opkKeyringSecret, opkPassphrSecret, opkDefaultKeyName) + keyring := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: opkKeyringSecret, Namespace: opkNs}, + Data: map[string][]byte{ + "someone_else.info": []byte("blob"), + opkAddressDataKey: []byte("addr"), + }, + } + exec, _ := newValidateOperatorKeyringExec(t, node, keyring, + validPassphraseSecret(opkPassphrSecret, opkNs), + ) + + err := exec.Execute(context.Background()) + g.Expect(err).To(HaveOccurred()) + g.Expect(isTerminal(err)).To(BeTrue()) + g.Expect(operatorKeyringConditionFor(node).Message).To(ContainSubstring("node_admin.info")) +} + +func TestValidateOperatorKeyring_PassphraseKeyMissing_Terminal(t *testing.T) { + g := NewWithT(t) + node := operatorKeyringNode(opkKeyringSecret, opkPassphrSecret, opkDefaultKeyName) + passphrase := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: opkPassphrSecret, Namespace: opkNs}, + Data: map[string][]byte{"wrong-key": []byte("hunter2")}, + } + exec, _ := newValidateOperatorKeyringExec(t, node, + validKeyringSecret(opkKeyringSecret, opkNs, opkDefaultKeyName), + passphrase, + ) + + err := exec.Execute(context.Background()) + g.Expect(err).To(HaveOccurred()) + g.Expect(isTerminal(err)).To(BeTrue()) + g.Expect(operatorKeyringConditionFor(node).Message).To(ContainSubstring("passphrase")) +} + +func TestValidateOperatorKeyring_PassphraseEmpty_Terminal(t *testing.T) { + g := NewWithT(t) + node := operatorKeyringNode(opkKeyringSecret, opkPassphrSecret, opkDefaultKeyName) + passphrase := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: opkPassphrSecret, Namespace: opkNs}, + Data: map[string][]byte{opkPassphraseKey: {}}, + } + exec, _ := newValidateOperatorKeyringExec(t, node, + validKeyringSecret(opkKeyringSecret, opkNs, opkDefaultKeyName), + passphrase, + ) + + err := exec.Execute(context.Background()) + g.Expect(err).To(HaveOccurred()) + g.Expect(isTerminal(err)).To(BeTrue()) + g.Expect(operatorKeyringConditionFor(node).Message).To(ContainSubstring("empty")) +} + +// --- Empty params --- + +func TestValidateOperatorKeyring_EmptySecretName_Terminal(t *testing.T) { + g := NewWithT(t) + node := operatorKeyringNode("", opkPassphrSecret, opkDefaultKeyName) + exec, _ := newValidateOperatorKeyringExec(t, node) + + err := exec.Execute(context.Background()) + g.Expect(err).To(HaveOccurred()) + g.Expect(isTerminal(err)).To(BeTrue()) +} + +func TestValidateOperatorKeyring_EmptyPassphraseSecretName_Terminal(t *testing.T) { + g := NewWithT(t) + node := operatorKeyringNode(opkKeyringSecret, "", opkDefaultKeyName) + exec, _ := newValidateOperatorKeyringExec(t, node) + + err := exec.Execute(context.Background()) + g.Expect(err).To(HaveOccurred()) + g.Expect(isTerminal(err)).To(BeTrue()) +} + +// --- Convergence: missing-then-applied --- + +func TestValidateOperatorKeyring_MissingThenApplied_Completes(t *testing.T) { + g := NewWithT(t) + node := operatorKeyringNode(opkKeyringSecret, opkPassphrSecret, opkDefaultKeyName) + exec, c := newValidateOperatorKeyringExec(t, node) + + g.Expect(exec.Execute(context.Background())).To(Succeed()) + g.Expect(exec.Status(context.Background())).To(Equal(ExecutionRunning)) + g.Expect(operatorKeyringReasonFor(node)).To(Equal(seiv1alpha1.ReasonOperatorKeyringNotReady)) + + ctx := context.Background() + g.Expect(c.Create(ctx, validKeyringSecret(opkKeyringSecret, opkNs, opkDefaultKeyName))).To(Succeed()) + g.Expect(c.Create(ctx, validPassphraseSecret(opkPassphrSecret, opkNs))).To(Succeed()) + + g.Expect(exec.Execute(ctx)).To(Succeed()) + g.Expect(exec.Status(ctx)).To(Equal(ExecutionComplete)) + g.Expect(operatorKeyringReasonFor(node)).To(Equal(seiv1alpha1.ReasonOperatorKeyringValidated)) +} diff --git a/manifests/sei.io_seinodedeployments.yaml b/manifests/sei.io_seinodedeployments.yaml index f6fa6cf9..86a91f5f 100644 --- a/manifests/sei.io_seinodedeployments.yaml +++ b/manifests/sei.io_seinodedeployments.yaml @@ -683,6 +683,83 @@ spec: x-kubernetes-validations: - message: exactly one node key source must be set rule: '(has(self.secret) ? 1 : 0) == 1' + operatorKeyring: + description: |- + OperatorKeyring declares the source of this validator's operator-account + keyring used by the sidecar to sign and broadcast governance, + MsgEditValidator, withdraw-rewards, and other operator-account + transactions. + + Independently optional from signingKey/nodeKey: a validator may run as a + non-signing observer with operatorKeyring set (governance-only + operations), or as a consensus-signing validator without operatorKeyring + (governance performed out-of-band). + + Mounted exclusively on the sidecar container; the seid main container + and bootstrap pods never carry this material. + properties: + secret: + description: |- + Secret loads a Cosmos SDK file-backend keyring from a Kubernetes Secret + in the SeiNode's namespace. + properties: + keyName: + default: node_admin + description: |- + KeyName is the name of the keyring entry to use when signing + (the name passed to `seid keys add `). Defaults to + "node_admin" to preserve continuity with the seienv convention. + Mutable — rotating to a different entry within the same Secret + is a routine operator-account change, not a slashing risk. + maxLength: 64 + pattern: ^[a-zA-Z0-9_-]+$ + type: string + passphraseSecretRef: + description: |- + PassphraseSecretRef names a separate Secret containing the keyring + unlock passphrase. Required for the file backend. + properties: + key: + default: passphrase + description: Key is the data key inside the + Secret. Defaults to "passphrase". + maxLength: 253 + type: string + secretName: + description: SecretName names the passphrase + Secret in the SeiNode's namespace. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + x-kubernetes-validations: + - message: passphrase secretName is immutable + rule: self == oldSelf + required: + - secretName + type: object + secretName: + description: |- + SecretName names a Secret in the SeiNode's namespace whose data keys + are the on-disk Cosmos SDK file-keyring layout. Minimum required: + .info (armored encrypted key blob) + .address (name→address index) + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + x-kubernetes-validations: + - message: secretName is immutable + rule: self == oldSelf + required: + - passphraseSecretRef + - secretName + type: object + type: object + x-kubernetes-validations: + - message: exactly one operator keyring source must be + set + rule: '(has(self.secret) ? 1 : 0) == 1' signingKey: description: |- SigningKey declares the source of this validator's consensus signing @@ -774,6 +851,19 @@ spec: bootstrap-pod trust boundary rule: '!has(self.signingKey) || !has(self.nodeKey) || self.signingKey.secret.secretName != self.nodeKey.secret.secretName' + - message: operatorKeyring and signingKey must reference distinct + Secrets — collapsing them into one Secret would force + the sidecar/seid trust boundary to evaporate + rule: '!has(self.operatorKeyring) || !has(self.signingKey) + || self.operatorKeyring.secret.secretName != self.signingKey.secret.secretName' + - message: operatorKeyring and nodeKey must reference distinct + Secrets + rule: '!has(self.operatorKeyring) || !has(self.nodeKey) + || self.operatorKeyring.secret.secretName != self.nodeKey.secret.secretName' + - message: operatorKeyring data Secret and passphrase Secret + must be distinct + rule: '!has(self.operatorKeyring) || self.operatorKeyring.secret.secretName + != self.operatorKeyring.secret.passphraseSecretRef.secretName' required: - chainId - image diff --git a/manifests/sei.io_seinodes.yaml b/manifests/sei.io_seinodes.yaml index 5dda9b46..5286662c 100644 --- a/manifests/sei.io_seinodes.yaml +++ b/manifests/sei.io_seinodes.yaml @@ -538,6 +538,82 @@ spec: x-kubernetes-validations: - message: exactly one node key source must be set rule: '(has(self.secret) ? 1 : 0) == 1' + operatorKeyring: + description: |- + OperatorKeyring declares the source of this validator's operator-account + keyring used by the sidecar to sign and broadcast governance, + MsgEditValidator, withdraw-rewards, and other operator-account + transactions. + + Independently optional from signingKey/nodeKey: a validator may run as a + non-signing observer with operatorKeyring set (governance-only + operations), or as a consensus-signing validator without operatorKeyring + (governance performed out-of-band). + + Mounted exclusively on the sidecar container; the seid main container + and bootstrap pods never carry this material. + properties: + secret: + description: |- + Secret loads a Cosmos SDK file-backend keyring from a Kubernetes Secret + in the SeiNode's namespace. + properties: + keyName: + default: node_admin + description: |- + KeyName is the name of the keyring entry to use when signing + (the name passed to `seid keys add `). Defaults to + "node_admin" to preserve continuity with the seienv convention. + Mutable — rotating to a different entry within the same Secret + is a routine operator-account change, not a slashing risk. + maxLength: 64 + pattern: ^[a-zA-Z0-9_-]+$ + type: string + passphraseSecretRef: + description: |- + PassphraseSecretRef names a separate Secret containing the keyring + unlock passphrase. Required for the file backend. + properties: + key: + default: passphrase + description: Key is the data key inside the Secret. + Defaults to "passphrase". + maxLength: 253 + type: string + secretName: + description: SecretName names the passphrase Secret + in the SeiNode's namespace. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + x-kubernetes-validations: + - message: passphrase secretName is immutable + rule: self == oldSelf + required: + - secretName + type: object + secretName: + description: |- + SecretName names a Secret in the SeiNode's namespace whose data keys + are the on-disk Cosmos SDK file-keyring layout. Minimum required: + .info (armored encrypted key blob) + .address (name→address index) + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + x-kubernetes-validations: + - message: secretName is immutable + rule: self == oldSelf + required: + - passphraseSecretRef + - secretName + type: object + type: object + x-kubernetes-validations: + - message: exactly one operator keyring source must be set + rule: '(has(self.secret) ? 1 : 0) == 1' signingKey: description: |- SigningKey declares the source of this validator's consensus signing @@ -628,6 +704,18 @@ spec: trust boundary rule: '!has(self.signingKey) || !has(self.nodeKey) || self.signingKey.secret.secretName != self.nodeKey.secret.secretName' + - message: operatorKeyring and signingKey must reference distinct + Secrets — collapsing them into one Secret would force the sidecar/seid + trust boundary to evaporate + rule: '!has(self.operatorKeyring) || !has(self.signingKey) || self.operatorKeyring.secret.secretName + != self.signingKey.secret.secretName' + - message: operatorKeyring and nodeKey must reference distinct Secrets + rule: '!has(self.operatorKeyring) || !has(self.nodeKey) || self.operatorKeyring.secret.secretName + != self.nodeKey.secret.secretName' + - message: operatorKeyring data Secret and passphrase Secret must + be distinct + rule: '!has(self.operatorKeyring) || self.operatorKeyring.secret.secretName + != self.operatorKeyring.secret.passphraseSecretRef.secretName' required: - chainId - image From 6622404e8fd62b44c351d32304761fab5b2ce399 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Mon, 11 May 2026 17:58:46 -0700 Subject: [PATCH 2/5] review: integrate cross-review findings on operatorKeyring CRD surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Synthesizes platform-engineer / kubernetes-specialist / security-specialist cross-review of #220. Addresses 9 review items; 1 deferred with explicit documentation. CEL invariants tightened (kubernetes-specialist finding): Adds two ValidatorSpec rules forbidding the passphrase Secret from colliding with signingKey.secret.secretName or nodeKey.secret.secretName. Closes a trust-boundary gap where a malicious config could project a Secret used by seid main into the sidecar's env-resolution path. Planner-side defense-in-depth (security-specialist finding): validatorPlanner.Validate now mirrors all five operator-keyring distinctness rules. CEL alone requires K8s 1.25+ with the feature flag; mis-configured webhooks or --validate=false bypass CEL. Production pod-spec runtime guard (security-specialist finding): GenerateStatefulSet asserts operatorKeyringVolumeName never appears on the seid main container or init containers. CI catches regressions the way assertNoValidatorSecretsOnBootstrapPod does for bootstrap pods. Signature changes to (*appsv1.StatefulSet, error); 42 call sites updated. Bootstrap-pod guard extended (security-specialist finding): assertNoValidatorSecretsOnBootstrapPod now walks c.EnvFrom[].SecretRef in addition to c.Env[].ValueFrom.SecretKeyRef. Future-proofs against envFrom-shaped leakage. Default constants exported (platform-engineer finding): api/v1alpha1.DefaultOperatorKeyName and DefaultPassphraseSecretKey replace magic strings in planner + noderesource. Kubebuilder defaults retain literal strings (markers can't reference Go consts) but carry comments tying them to the constants. Operator-facing error wording reworded (platform-engineer finding): Conditions point at Secret data-key shape (*.info, *.address) instead of internal Cosmos SDK backend terminology. Explicit AutomountServiceAccountToken (security-specialist finding): Pod spec now sets ptr.To(true) explicitly with #165 reference comment. Cluster-default flips that disable SA tokens would otherwise silently break the future TokenReview path. ptr.To linter situation cleaned (platform-engineer finding): Restored ptr.To(true)/ptr.To(false) with //nolint:modernize and a reason string. CLAUDE.md "fix lint, don't suppress" applies to real issues; new(true) is invalid Go, so the modernize suggestion is a false-positive and explicit nolint is the idiomatic answer. shareProcessNamespace = true documented as deferred (security blocker downgraded to deferred follow-up per orchestrator decision): A compromised seid container can today read /proc//environ and /proc//mem, exposing the passphrase and unlocked keyring. Not a one-way door — the broader sidecar/seid hardening (drop shareProcessNamespace, harden seid main SecurityContext, separate sidecar SA) is tracked separately. Co-Authored-By: Claude Opus 4.7 (1M context) --- api/v1alpha1/validator_types.go | 18 ++ config/crd/sei.io_seinodedeployments.yaml | 20 ++- config/crd/sei.io_seinodes.yaml | 18 +- internal/controller/node/reconciler_test.go | 3 +- internal/noderesource/noderesource.go | 99 ++++++++-- internal/noderesource/noderesource_test.go | 190 +++++++++++++++----- internal/planner/planner.go | 4 +- internal/planner/validator.go | 37 ++++ internal/planner/validator_test.go | 121 +++++++++++++ internal/task/apply_statefulset.go | 5 +- internal/task/bootstrap_resources.go | 14 +- internal/task/bootstrap_resources_test.go | 26 +++ internal/task/validate_operator_keyring.go | 22 +-- manifests/sei.io_seinodedeployments.yaml | 20 ++- manifests/sei.io_seinodes.yaml | 18 +- 15 files changed, 534 insertions(+), 81 deletions(-) diff --git a/api/v1alpha1/validator_types.go b/api/v1alpha1/validator_types.go index 1d36f07f..61b0bad3 100644 --- a/api/v1alpha1/validator_types.go +++ b/api/v1alpha1/validator_types.go @@ -1,5 +1,15 @@ package v1alpha1 +// Defaults for SecretOperatorKeyringSource fields. Kept in lockstep with the +// +kubebuilder:default markers on the corresponding struct fields and +// referenced by the planner + noderesource packages so defaulting stays +// consistent even when admission webhooks haven't run (e.g., in-memory +// specs in tests, or controllers reading pre-defaulted objects). +const ( + DefaultOperatorKeyName = "node_admin" + DefaultPassphraseSecretKey = "passphrase" +) + // ValidatorSpec configures a consensus-participating validator node. // Validators bootstrap the same way as full nodes but participate in consensus. // @@ -8,6 +18,8 @@ package v1alpha1 // +kubebuilder:validation:XValidation:rule="!has(self.operatorKeyring) || !has(self.signingKey) || self.operatorKeyring.secret.secretName != self.signingKey.secret.secretName",message="operatorKeyring and signingKey must reference distinct Secrets — collapsing them into one Secret would force the sidecar/seid trust boundary to evaporate" // +kubebuilder:validation:XValidation:rule="!has(self.operatorKeyring) || !has(self.nodeKey) || self.operatorKeyring.secret.secretName != self.nodeKey.secret.secretName",message="operatorKeyring and nodeKey must reference distinct Secrets" // +kubebuilder:validation:XValidation:rule="!has(self.operatorKeyring) || self.operatorKeyring.secret.secretName != self.operatorKeyring.secret.passphraseSecretRef.secretName",message="operatorKeyring data Secret and passphrase Secret must be distinct" +// +kubebuilder:validation:XValidation:rule="!has(self.operatorKeyring) || !has(self.signingKey) || self.operatorKeyring.secret.passphraseSecretRef.secretName != self.signingKey.secret.secretName",message="operatorKeyring passphrase Secret must not equal signingKey Secret" +// +kubebuilder:validation:XValidation:rule="!has(self.operatorKeyring) || !has(self.nodeKey) || self.operatorKeyring.secret.passphraseSecretRef.secretName != self.nodeKey.secret.secretName",message="operatorKeyring passphrase Secret must not equal nodeKey Secret" type ValidatorSpec struct { // Snapshot configures how the node obtains its initial chain state. // When absent the node block-syncs from genesis. @@ -174,6 +186,9 @@ type SecretOperatorKeyringSource struct { // Mutable — rotating to a different entry within the same Secret // is a routine operator-account change, not a slashing risk. // + // The default literal below MUST match DefaultOperatorKeyName — + // kubebuilder markers cannot reference Go constants. + // // +optional // +kubebuilder:default="node_admin" // +kubebuilder:validation:MaxLength=64 @@ -197,6 +212,9 @@ type PassphraseSecretRef struct { // Key is the data key inside the Secret. Defaults to "passphrase". // + // The default literal below MUST match DefaultPassphraseSecretKey — + // kubebuilder markers cannot reference Go constants. + // // +optional // +kubebuilder:default="passphrase" // +kubebuilder:validation:MaxLength=253 diff --git a/config/crd/sei.io_seinodedeployments.yaml b/config/crd/sei.io_seinodedeployments.yaml index 86a91f5f..b1d87513 100644 --- a/config/crd/sei.io_seinodedeployments.yaml +++ b/config/crd/sei.io_seinodedeployments.yaml @@ -711,6 +711,9 @@ spec: "node_admin" to preserve continuity with the seienv convention. Mutable — rotating to a different entry within the same Secret is a routine operator-account change, not a slashing risk. + + The default literal below MUST match DefaultOperatorKeyName — + kubebuilder markers cannot reference Go constants. maxLength: 64 pattern: ^[a-zA-Z0-9_-]+$ type: string @@ -721,8 +724,11 @@ spec: properties: key: default: passphrase - description: Key is the data key inside the - Secret. Defaults to "passphrase". + description: |- + Key is the data key inside the Secret. Defaults to "passphrase". + + The default literal below MUST match DefaultPassphraseSecretKey — + kubebuilder markers cannot reference Go constants. maxLength: 253 type: string secretName: @@ -864,6 +870,16 @@ spec: must be distinct rule: '!has(self.operatorKeyring) || self.operatorKeyring.secret.secretName != self.operatorKeyring.secret.passphraseSecretRef.secretName' + - message: operatorKeyring passphrase Secret must not equal + signingKey Secret + rule: '!has(self.operatorKeyring) || !has(self.signingKey) + || self.operatorKeyring.secret.passphraseSecretRef.secretName + != self.signingKey.secret.secretName' + - message: operatorKeyring passphrase Secret must not equal + nodeKey Secret + rule: '!has(self.operatorKeyring) || !has(self.nodeKey) + || self.operatorKeyring.secret.passphraseSecretRef.secretName + != self.nodeKey.secret.secretName' required: - chainId - image diff --git a/config/crd/sei.io_seinodes.yaml b/config/crd/sei.io_seinodes.yaml index 5286662c..5eb04e54 100644 --- a/config/crd/sei.io_seinodes.yaml +++ b/config/crd/sei.io_seinodes.yaml @@ -566,6 +566,9 @@ spec: "node_admin" to preserve continuity with the seienv convention. Mutable — rotating to a different entry within the same Secret is a routine operator-account change, not a slashing risk. + + The default literal below MUST match DefaultOperatorKeyName — + kubebuilder markers cannot reference Go constants. maxLength: 64 pattern: ^[a-zA-Z0-9_-]+$ type: string @@ -576,8 +579,11 @@ spec: properties: key: default: passphrase - description: Key is the data key inside the Secret. - Defaults to "passphrase". + description: |- + Key is the data key inside the Secret. Defaults to "passphrase". + + The default literal below MUST match DefaultPassphraseSecretKey — + kubebuilder markers cannot reference Go constants. maxLength: 253 type: string secretName: @@ -716,6 +722,14 @@ spec: be distinct rule: '!has(self.operatorKeyring) || self.operatorKeyring.secret.secretName != self.operatorKeyring.secret.passphraseSecretRef.secretName' + - message: operatorKeyring passphrase Secret must not equal signingKey + Secret + rule: '!has(self.operatorKeyring) || !has(self.signingKey) || self.operatorKeyring.secret.passphraseSecretRef.secretName + != self.signingKey.secret.secretName' + - message: operatorKeyring passphrase Secret must not equal nodeKey + Secret + rule: '!has(self.operatorKeyring) || !has(self.nodeKey) || self.operatorKeyring.secret.passphraseSecretRef.secretName + != self.nodeKey.secret.secretName' required: - chainId - image diff --git a/internal/controller/node/reconciler_test.go b/internal/controller/node/reconciler_test.go index 6a135dfb..65c7c0f9 100644 --- a/internal/controller/node/reconciler_test.go +++ b/internal/controller/node/reconciler_test.go @@ -198,7 +198,8 @@ func TestNodeReconcile_RunningPhase_UpdatesStatefulSetImage(t *testing.T) { node.Status.Phase = seiv1alpha1.PhaseRunning // Pre-create a StatefulSet with the old image. - oldSts := noderesource.GenerateStatefulSet(node, platformtest.Config()) + oldSts, err := noderesource.GenerateStatefulSet(node, platformtest.Config()) + g.Expect(err).NotTo(HaveOccurred()) oldSts.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("StatefulSet")) r, c := newNodeReconciler(t, node, oldSts) diff --git a/internal/noderesource/noderesource.go b/internal/noderesource/noderesource.go index 3bae3068..6a8d1570 100644 --- a/internal/noderesource/noderesource.go +++ b/internal/noderesource/noderesource.go @@ -28,6 +28,11 @@ const ( dataDir = platform.DataDir defaultSidecarImage = platform.DefaultSidecarImage + // Pod-spec container names. Used as both the .Name on built containers + // and the lookup key for the operator-keyring containment guard. + containerNameSeid = "seid" + containerNameSidecar = "sei-sidecar" + signingKeyVolumeName = "signing-key" privValidatorKeyDataKey = "priv_validator_key.json" @@ -138,9 +143,18 @@ func DefaultResourcesForMode(mode string, p PlatformConfig) corev1.ResourceRequi // --------------------------------------------------------------------------- // GenerateStatefulSet produces the desired StatefulSet for a SeiNode. -func GenerateStatefulSet(node *seiv1alpha1.SeiNode, p PlatformConfig) *appsv1.StatefulSet { +// +// Returns an error if the resulting pod-spec violates the operator-keyring +// containment invariant — only the sidecar container may mount that volume, +// never seid main or any non-sidecar init container. +func GenerateStatefulSet(node *seiv1alpha1.SeiNode, p PlatformConfig) (*appsv1.StatefulSet, error) { one := int32(1) labels := ResourceLabels(node) + podSpec := buildNodePodSpec(node, p) + + if err := assertNoOperatorKeyringOnSeidContainers(node, &podSpec); err != nil { + return nil, err + } return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ @@ -165,10 +179,54 @@ func GenerateStatefulSet(node *seiv1alpha1.SeiNode, p PlatformConfig) *appsv1.St "karpenter.sh/do-not-disrupt": "true", }, }, - Spec: buildNodePodSpec(node, p), + Spec: podSpec, }, }, + }, nil +} + +// assertNoOperatorKeyringOnSeidContainers fails closed if a future refactor +// lands the operator-keyring volume on the seid main container or a +// non-sidecar init container. Operator-keyring material is the sidecar's +// alone: a compromised seid container reading that mount would collapse +// the sidecar/seid trust boundary. +// +// No-op when the node has no operator-keyring configured — the volume is +// only present when SecretOperatorKeyringSource is set, so there is +// nothing to contain. +func assertNoOperatorKeyringOnSeidContainers(node *seiv1alpha1.SeiNode, spec *corev1.PodSpec) error { + if operatorKeyringSecretSource(node) == nil { + return nil + } + + check := func(c *corev1.Container) error { + for _, m := range c.VolumeMounts { + if m.Name == operatorKeyringVolumeName { + return fmt.Errorf("pod-spec for %s/%s mounts operator-keyring volume on container %q; "+ + "operator-keyring is exclusively the sidecar's — mounting on seid would collapse the sidecar/seid trust boundary", + node.Namespace, node.Name, c.Name) + } + } + return nil + } + + for i := range spec.Containers { + if spec.Containers[i].Name == containerNameSidecar { + continue + } + if err := check(&spec.Containers[i]); err != nil { + return err + } + } + for i := range spec.InitContainers { + if spec.InitContainers[i].Name == containerNameSidecar { + continue + } + if err := check(&spec.InitContainers[i]); err != nil { + return err + } } + return nil } // --------------------------------------------------------------------------- @@ -273,10 +331,12 @@ func buildNodePodSpec(node *seiv1alpha1.SeiNode, p PlatformConfig) corev1.PodSpe pool := p.NodepoolForMode(NodeMode(node)) spec := corev1.PodSpec{ - // automountServiceAccountToken is left at the kubelet default (true) - // — the projected token is a hard dependency for Phase 4 TokenReview - // authentication on sidecar HTTP endpoints (see #165). - ServiceAccountName: p.ServiceAccount, + // AutomountServiceAccountToken is explicit here because the sidecar's + // future TokenReview-based authn (sei-protocol/seictl#165) requires + // the SA token mount. Cluster-default flips that silently disable + // this would break the auth path. + AutomountServiceAccountToken: ptr.To(true), //nolint:modernize // ptr.To(true) is idiomatic; new(true) is invalid Go + ServiceAccountName: p.ServiceAccount, Tolerations: []corev1.Toleration{ {Key: p.TolerationKey, Value: pool, Effect: corev1.TaintEffectNoSchedule}, }, @@ -296,6 +356,14 @@ func buildNodePodSpec(node *seiv1alpha1.SeiNode, p PlatformConfig) corev1.PodSpe Volumes: volumes, } + // ShareProcessNamespace is currently enabled across all SeiNode pods. A + // compromised seid container can therefore read /proc//environ + // and /proc//mem, including the operator-keyring passphrase and + // the unlocked in-memory keyring. This is a known limitation of the v1 + // trust boundary — not a one-way door. The broader sidecar/seid isolation + // hardening (drop shareProcessNamespace, harden the seid main container's + // SecurityContext, separate the sidecar's SA) is tracked as a follow-up. + // See PR #220 review thread. spec.ShareProcessNamespace = ptr.To(true) // fsGroup is required so the non-root sidecar (UID 65532) can read // 0o400 Secret-projected files (operator keyring) kubelet owns root:root. @@ -348,7 +416,7 @@ func buildSidecarContainer(node *seiv1alpha1.SeiNode, p PlatformConfig) corev1.C mounts = append(mounts, keyringMounts...) c := corev1.Container{ - Name: "sei-sidecar", + Name: containerNameSidecar, Image: sidecarImage(node), Command: []string{"seictl", "serve"}, RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), @@ -454,7 +522,7 @@ func buildNodeMainContainer(node *seiv1alpha1.SeiNode) corev1.Container { mounts = append(mounts, signingMounts...) mounts = append(mounts, nodeMounts...) container := corev1.Container{ - Name: "seid", + Name: containerNameSeid, Image: node.Spec.Image, Env: []corev1.EnvVar{ {Name: "TMPDIR", Value: dataDir + "/tmp"}, @@ -643,7 +711,7 @@ func operatorKeyringEnvVars(node *seiv1alpha1.SeiNode) []corev1.EnvVar { } key := src.PassphraseSecretRef.Key if key == "" { - key = "passphrase" + key = seiv1alpha1.DefaultPassphraseSecretKey } return []corev1.EnvVar{{ Name: keyringPassphraseEnvVar, @@ -669,15 +737,12 @@ func operatorKeyringSecretSource(node *seiv1alpha1.SeiNode) *seiv1alpha1.SecretO // same to the seid main container is a larger blast-radius change owned // by a different workstream. func sidecarSecurityContext() *corev1.SecurityContext { - yes, no := true, false - uid := sidecarNonRootUID - gid := sidecarNonRootUID return &corev1.SecurityContext{ - RunAsNonRoot: &yes, - RunAsUser: &uid, - RunAsGroup: &gid, - AllowPrivilegeEscalation: &no, - ReadOnlyRootFilesystem: &yes, + RunAsNonRoot: ptr.To(true), //nolint:modernize // ptr.To(true) is idiomatic; new(true) is invalid Go + RunAsUser: ptr.To(sidecarNonRootUID), //nolint:modernize // false-positive: new() takes a type, not a value + RunAsGroup: ptr.To(sidecarNonRootUID), //nolint:modernize // false-positive: new() takes a type, not a value + AllowPrivilegeEscalation: ptr.To(false), //nolint:modernize // ptr.To(false) is idiomatic; new(false) is invalid Go + ReadOnlyRootFilesystem: ptr.To(true), //nolint:modernize // ptr.To(true) is idiomatic; new(true) is invalid Go Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, } diff --git a/internal/noderesource/noderesource_test.go b/internal/noderesource/noderesource_test.go index 0f02501d..9a2a17ab 100644 --- a/internal/noderesource/noderesource_test.go +++ b/internal/noderesource/noderesource_test.go @@ -76,6 +76,18 @@ func envValue(envs []corev1.EnvVar, name string) string { return "" } +// mustGenerateStatefulSet wraps GenerateStatefulSet with a t.Fatal on +// invariant violation. Used by tests that construct a valid SeiNode and +// expect the runtime guard to pass. +func mustGenerateStatefulSet(t *testing.T, node *seiv1alpha1.SeiNode, p PlatformConfig) *appsv1.StatefulSet { + t.Helper() + sts, err := GenerateStatefulSet(node, p) + if err != nil { + t.Fatalf("GenerateStatefulSet: %v", err) + } + return sts +} + // --- Pod labels --- func TestResourceLabelsForNode_DefaultsToNodeOnly(t *testing.T) { @@ -120,7 +132,7 @@ func TestGenerateNodeStatefulSet_PodLabelsPropagate(t *testing.T) { "sei.io/nodedeployment": "my-group", } - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) g.Expect(sts.Labels).To(HaveKeyWithValue("sei.io/nodedeployment", "my-group")) g.Expect(sts.Spec.Template.Labels).To(HaveKeyWithValue("sei.io/nodedeployment", "my-group")) @@ -133,7 +145,7 @@ func TestGenerateNodeStatefulSet_BasicFields(t *testing.T) { g := NewWithT(t) node := newGenesisNode("mynet-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) g.Expect(sts.Name).To(Equal("mynet-0")) g.Expect(sts.Namespace).To(Equal("default")) @@ -148,7 +160,7 @@ func TestGenerateNodeStatefulSet_UsesOnDeleteUpdateStrategy(t *testing.T) { g := NewWithT(t) node := newGenesisNode("mynet-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) g.Expect(sts.Spec.UpdateStrategy.Type).To(Equal(appsv1.OnDeleteStatefulSetStrategyType)) } @@ -157,7 +169,7 @@ func TestGenerateNodeStatefulSet_AlwaysHasSidecar(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("snap-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) initContainers := sts.Spec.Template.Spec.InitContainers g.Expect(initContainers).To(HaveLen(2)) @@ -192,7 +204,7 @@ func TestBuildNodePodSpec_SharedPIDNamespace(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("snap-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) g.Expect(sts.Spec.Template.Spec.ShareProcessNamespace).NotTo(BeNil()) g.Expect(*sts.Spec.Template.Spec.ShareProcessNamespace).To(BeTrue()) @@ -347,7 +359,7 @@ func TestSidecarContainer_DefaultImage(t *testing.T) { node := newSnapshotNode("sc-0", "default") node.Spec.Sidecar = &seiv1alpha1.SidecarConfig{Port: 7777} - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) sc := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") g.Expect(sc.Image).To(Equal(defaultSidecarImage)) @@ -358,7 +370,7 @@ func TestSidecarContainer_CustomImage(t *testing.T) { node := newSnapshotNode("sc-0", "default") node.Spec.Sidecar = &seiv1alpha1.SidecarConfig{Image: "custom/seictl:v3", Port: 7777} - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) sc := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") g.Expect(sc.Image).To(Equal("custom/seictl:v3")) @@ -368,7 +380,7 @@ func TestSidecarContainer_RestartPolicyAlways(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("sc-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) sc := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") g.Expect(sc).NotTo(BeNil()) @@ -380,7 +392,7 @@ func TestSidecarContainer_EnvVars(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("sc-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) sc := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") cfg := platformtest.Config() @@ -395,7 +407,7 @@ func TestSidecarContainer_DataVolumeMount(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("sc-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) sc := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") g.Expect(sc.VolumeMounts).To(HaveLen(1)) @@ -407,7 +419,7 @@ func TestSidecarContainer_CustomPort(t *testing.T) { node := newSnapshotNode("sc-0", "default") node.Spec.Sidecar = &seiv1alpha1.SidecarConfig{Port: 9999} - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) sc := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") g.Expect(sc.Ports).To(HaveLen(1)) @@ -431,7 +443,7 @@ func TestSidecarContainer_CustomResources(t *testing.T) { }, } - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) sc := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") g.Expect(sc.Resources.Requests.Cpu().String()).To(Equal("250m")) @@ -444,7 +456,7 @@ func TestSidecarContainer_NoResources_DefaultsToEmpty(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("sc-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) sc := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") g.Expect(sc.Resources.Requests).To(BeNil()) @@ -457,7 +469,7 @@ func TestSidecarMainContainer_StartupProbeTargetsHealthz(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("sc-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) seid := findContainer(sts.Spec.Template.Spec.Containers, "seid") g.Expect(seid).NotTo(BeNil()) @@ -476,7 +488,7 @@ func TestSidecarMainContainer_StartupProbeUsesCustomPort(t *testing.T) { node := newSnapshotNode("sc-0", "default") node.Spec.Sidecar = &seiv1alpha1.SidecarConfig{Port: 9999} - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) seid := findContainer(sts.Spec.Template.Spec.Containers, "seid") g.Expect(seid.StartupProbe.HTTPGet.Port.IntValue()).To(Equal(9999)) @@ -486,7 +498,7 @@ func TestSidecarMainContainer_ReadinessProbeTargetsLagStatus(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("sc-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) seid := findContainer(sts.Spec.Template.Spec.Containers, "seid") g.Expect(seid).NotTo(BeNil()) @@ -505,7 +517,7 @@ func TestSidecarMainContainer_WaitWrapper_PollsHealthzBeforeExec(t *testing.T) { g := NewWithT(t) node := newGenesisNode("gen-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) seid := findContainer(sts.Spec.Template.Spec.Containers, "seid") g.Expect(seid.Command).To(Equal([]string{"/bin/bash", "-c"})) @@ -524,7 +536,7 @@ func TestSidecarMainContainer_WaitWrapper_IncludesEntrypointArgs(t *testing.T) { Args: []string{"start", "--home", "/sei"}, } - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) seid := findContainer(sts.Spec.Template.Spec.Containers, "seid") g.Expect(seid.Args[0]).To(ContainSubstring(`exec seid "start" "--home" "/sei"`)) @@ -534,7 +546,7 @@ func TestSidecarMainContainer_WaitWrapper_NoEntrypoint_DefaultsSeidStart(t *test g := NewWithT(t) node := newSnapshotNode("sc-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) seid := findContainer(sts.Spec.Template.Spec.Containers, "seid") g.Expect(seid.Command).To(Equal([]string{"/bin/bash", "-c"})) @@ -546,7 +558,7 @@ func TestSidecarMainContainer_NilSidecarConfig_UsesDefaults(t *testing.T) { node := newSnapshotNode("sc-0", "default") node.Spec.Sidecar = nil - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) seid := findContainer(sts.Spec.Template.Spec.Containers, "seid") g.Expect(seid.StartupProbe.HTTPGet.Port.IntValue()).To(Equal(int(seiconfig.PortSidecar))) @@ -562,7 +574,7 @@ func TestSidecarMainContainer_WaitWrapper_UsesCustomPort(t *testing.T) { node := newSnapshotNode("sc-0", "default") node.Spec.Sidecar = &seiv1alpha1.SidecarConfig{Port: 9999} - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) seid := findContainer(sts.Spec.Template.Spec.Containers, "seid") g.Expect(seid.Args[0]).To(ContainSubstring("/dev/tcp/localhost/9999")) @@ -574,7 +586,7 @@ func TestGenesisMode_SidecarPresent(t *testing.T) { g := NewWithT(t) node := newGenesisNode("gen-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) initContainers := sts.Spec.Template.Spec.InitContainers g.Expect(initContainers).To(HaveLen(2)) @@ -586,7 +598,7 @@ func TestGenesisMode_NoSnapshotRestoreInitContainer(t *testing.T) { g := NewWithT(t) node := newGenesisNode("gen-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) initContainers := sts.Spec.Template.Spec.InitContainers g.Expect(findInitContainer(initContainers, "snapshot-restore")).To(BeNil()) @@ -596,7 +608,7 @@ func TestGenesisMode_SharedPIDNamespace(t *testing.T) { g := NewWithT(t) node := newGenesisNode("gen-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) g.Expect(sts.Spec.Template.Spec.ShareProcessNamespace).NotTo(BeNil()) g.Expect(*sts.Spec.Template.Spec.ShareProcessNamespace).To(BeTrue()) @@ -775,7 +787,7 @@ func TestSigningKey_SecretVolumePresentOnPodTemplate(t *testing.T) { g := NewWithT(t) node := newValidatorNodeWithSigningKey("validator-0", "default", "validator-0-key") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) vol := findVolume(sts.Spec.Template.Spec.Volumes, signingKeyVolumeName) g.Expect(vol).NotTo(BeNil(), "signing-key volume must be present on the StatefulSet pod template") @@ -791,7 +803,7 @@ func TestSigningKey_SeidContainerHasSubPathMount(t *testing.T) { g := NewWithT(t) node := newValidatorNodeWithSigningKey("validator-0", "default", "validator-0-key") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) seid := findContainer(sts.Spec.Template.Spec.Containers, "seid") g.Expect(seid).NotTo(BeNil(), "seid main container must exist") @@ -806,7 +818,7 @@ func TestSigningKey_SidecarContainerHasNoSigningMount(t *testing.T) { g := NewWithT(t) node := newValidatorNodeWithSigningKey("validator-0", "default", "validator-0-key") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) sidecar := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") g.Expect(sidecar).NotTo(BeNil(), "sei-sidecar init container must exist") @@ -818,7 +830,7 @@ func TestSigningKey_Unset_NoSigningVolume(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("snap-0", "default") // FullNode mode, no SigningKey - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) g.Expect(findVolume(sts.Spec.Template.Spec.Volumes, signingKeyVolumeName)).To(BeNil(), "non-signing-key SeiNode must not have a signing-key volume") @@ -835,7 +847,7 @@ func TestNodeKey_SecretVolumePresentOnPodTemplate(t *testing.T) { g := NewWithT(t) node := newValidatorNodeWithSigningKey("validator-0", "default", "validator-0-key") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) vol := findVolume(sts.Spec.Template.Spec.Volumes, nodeKeyVolumeName) g.Expect(vol).NotTo(BeNil(), "node-key volume must be present on the StatefulSet pod template") @@ -851,7 +863,7 @@ func TestNodeKey_SeidContainerHasSubPathMount(t *testing.T) { g := NewWithT(t) node := newValidatorNodeWithSigningKey("validator-0", "default", "validator-0-key") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) seid := findContainer(sts.Spec.Template.Spec.Containers, "seid") g.Expect(seid).NotTo(BeNil(), "seid main container must exist") @@ -866,7 +878,7 @@ func TestNodeKey_SidecarContainerHasNoNodeKeyMount(t *testing.T) { g := NewWithT(t) node := newValidatorNodeWithSigningKey("validator-0", "default", "validator-0-key") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) sidecar := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") g.Expect(sidecar).NotTo(BeNil(), "sei-sidecar init container must exist") @@ -878,7 +890,7 @@ func TestNodeKey_Unset_NoNodeKeyVolume(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("snap-0", "default") // FullNode mode, no NodeKey - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) g.Expect(findVolume(sts.Spec.Template.Spec.Volumes, nodeKeyVolumeName)).To(BeNil(), "non-validator SeiNode must not have a node-key volume") @@ -893,7 +905,7 @@ func TestNodeKey_BothMountsCoexist(t *testing.T) { g := NewWithT(t) node := newValidatorNodeWithSigningKey("validator-0", "default", "validator-0-key") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) seid := findContainer(sts.Spec.Template.Spec.Containers, "seid") g.Expect(seid).NotTo(BeNil()) @@ -933,7 +945,7 @@ func TestOperatorKeyring_SecretVolumePresentOnPodTemplate(t *testing.T) { g := NewWithT(t) node := newValidatorNodeWithOperatorKeyring("validator-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) vol := findVolume(sts.Spec.Template.Spec.Volumes, operatorKeyringVolumeName) g.Expect(vol).NotTo(BeNil(), "operator-keyring volume must be present on the StatefulSet pod template") @@ -948,7 +960,7 @@ func TestOperatorKeyring_SidecarContainerHasMountAndEnv(t *testing.T) { g := NewWithT(t) node := newValidatorNodeWithOperatorKeyring("validator-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) sidecar := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") g.Expect(sidecar).NotTo(BeNil(), "sei-sidecar init container must exist") @@ -976,7 +988,7 @@ func TestOperatorKeyring_SeidMainContainerHasNoMountOrEnv(t *testing.T) { g := NewWithT(t) node := newValidatorNodeWithOperatorKeyring("validator-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) seid := findContainer(sts.Spec.Template.Spec.Containers, "seid") g.Expect(seid).NotTo(BeNil(), "seid main container must exist") @@ -995,7 +1007,7 @@ func TestOperatorKeyring_Unset_NoVolumeOrMount(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("snap-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) g.Expect(findVolume(sts.Spec.Template.Spec.Volumes, operatorKeyringVolumeName)).To(BeNil()) @@ -1010,7 +1022,7 @@ func TestSidecarContainer_SecurityContext(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("snap-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) sidecar := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") g.Expect(sidecar).NotTo(BeNil()) g.Expect(sidecar.SecurityContext).NotTo(BeNil()) @@ -1031,7 +1043,7 @@ func TestSeidMainContainer_NoSecurityContextChange(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("snap-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) seid := findContainer(sts.Spec.Template.Spec.Containers, "seid") g.Expect(seid).NotTo(BeNil()) // seid main container hardening is an out-of-scope, larger blast-radius @@ -1047,8 +1059,106 @@ func TestPodSpec_FSGroup(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("snap-0", "default") - sts := GenerateStatefulSet(node, platformtest.Config()) + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) g.Expect(sts.Spec.Template.Spec.SecurityContext).NotTo(BeNil()) g.Expect(*sts.Spec.Template.Spec.SecurityContext.FSGroup).To(Equal(int64(65532)), "pod-level fsGroup must match sidecar UID so non-root sidecar can read 0o400 Secret mounts") } + +// --- assertNoOperatorKeyringOnSeidContainers --- + +const ( + keyringTestSeidInitName = "seid-init" + keyringTestSidecarName = "sei-sidecar" + keyringTestMountPath = "/sei/keyring-file" +) + +func validatorNodeWithOperatorKeyring() *seiv1alpha1.SeiNode { + return &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: "v-0", Namespace: "default"}, + Spec: seiv1alpha1.SeiNodeSpec{ + ChainID: "sei-test", + Image: "ghcr.io/sei-protocol/seid:latest", + Validator: &seiv1alpha1.ValidatorSpec{ + OperatorKeyring: &seiv1alpha1.OperatorKeyringSource{ + Secret: &seiv1alpha1.SecretOperatorKeyringSource{ + SecretName: "validator-0-opk", + PassphraseSecretRef: seiv1alpha1.PassphraseSecretRef{ + SecretName: "validator-0-opk-pass", + Key: "passphrase", + }, + }, + }, + }, + }, + } +} + +func TestAssertNoOperatorKeyringOnSeidContainers_NoKeyringConfigured(t *testing.T) { + g := NewWithT(t) + node := newGenesisNode("v-0", "default") + // Even a deliberately mis-mounted volume must be ignored when no + // operator-keyring is configured — the guard is opt-in by spec. + spec := &corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: containerNameSeid, VolumeMounts: []corev1.VolumeMount{ + {Name: operatorKeyringVolumeName, MountPath: "/somewhere"}, + }}, + }, + } + g.Expect(assertNoOperatorKeyringOnSeidContainers(node, spec)).To(Succeed()) +} + +func TestAssertNoOperatorKeyringOnSeidContainers_SidecarOnlyMount_Passes(t *testing.T) { + g := NewWithT(t) + node := validatorNodeWithOperatorKeyring() + spec := &corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: containerNameSeid}, + }, + InitContainers: []corev1.Container{ + {Name: keyringTestSeidInitName}, + {Name: keyringTestSidecarName, VolumeMounts: []corev1.VolumeMount{ + {Name: operatorKeyringVolumeName, MountPath: keyringTestMountPath}, + }}, + }, + } + g.Expect(assertNoOperatorKeyringOnSeidContainers(node, spec)).To(Succeed()) +} + +func TestAssertNoOperatorKeyringOnSeidContainers_SeidMainMisMounted_Rejects(t *testing.T) { + g := NewWithT(t) + node := validatorNodeWithOperatorKeyring() + spec := &corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: containerNameSeid, VolumeMounts: []corev1.VolumeMount{ + {Name: operatorKeyringVolumeName, MountPath: keyringTestMountPath}, + }}, + }, + } + err := assertNoOperatorKeyringOnSeidContainers(node, spec) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(`container "seid"`)) +} + +func TestAssertNoOperatorKeyringOnSeidContainers_SeidInitMisMounted_Rejects(t *testing.T) { + g := NewWithT(t) + node := validatorNodeWithOperatorKeyring() + spec := &corev1.PodSpec{ + InitContainers: []corev1.Container{ + {Name: keyringTestSeidInitName, VolumeMounts: []corev1.VolumeMount{ + {Name: operatorKeyringVolumeName, MountPath: keyringTestMountPath}, + }}, + }, + } + err := assertNoOperatorKeyringOnSeidContainers(node, spec) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(`container "seid-init"`)) +} + +func TestGenerateStatefulSet_ProductionPodSpec_PassesGuard(t *testing.T) { + g := NewWithT(t) + node := validatorNodeWithOperatorKeyring() + _, err := GenerateStatefulSet(node, platformtest.Config()) + g.Expect(err).NotTo(HaveOccurred()) +} diff --git a/internal/planner/planner.go b/internal/planner/planner.go index ef6c0583..35b27282 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -368,11 +368,11 @@ func validateOperatorKeyringParams(node *seiv1alpha1.SeiNode) any { // from a partially-defaulted in-memory spec still get the right values. keyName := s.KeyName if keyName == "" { - keyName = "node_admin" + keyName = seiv1alpha1.DefaultOperatorKeyName } passphraseKey := s.PassphraseSecretRef.Key if passphraseKey == "" { - passphraseKey = "passphrase" + passphraseKey = seiv1alpha1.DefaultPassphraseSecretKey } return &task.ValidateOperatorKeyringParams{ SecretName: s.SecretName, diff --git a/internal/planner/validator.go b/internal/planner/validator.go index aa7eb7f0..88157b5b 100644 --- a/internal/planner/validator.go +++ b/internal/planner/validator.go @@ -38,6 +38,9 @@ func (p *validatorPlanner) Validate(node *seiv1alpha1.SeiNode) error { if node.Spec.Validator.NodeKey != nil && node.Spec.Validator.SigningKey == nil { return fmt.Errorf("validator: nodeKey requires signingKey to be set") } + if err := validateOperatorKeyringDistinctness(node.Spec.Validator); err != nil { + return err + } if gc := node.Spec.Validator.GenesisCeremony; gc != nil { if gc.ChainID == "" { return fmt.Errorf("validator: genesisCeremony.chainId is required") @@ -55,6 +58,40 @@ func (p *validatorPlanner) Validate(node *seiv1alpha1.SeiNode) error { return nil } +// validateOperatorKeyringDistinctness mirrors the CRD XValidation rules +// so the planner rejects identical specs even when admission webhooks +// haven't run (in-memory specs in tests, or stale objects predating the +// CRD update). The CEL rules remain the canonical surface — these checks +// are defense in depth. +func validateOperatorKeyringDistinctness(v *seiv1alpha1.ValidatorSpec) error { + if v.OperatorKeyring == nil || v.OperatorKeyring.Secret == nil { + return nil + } + opk := v.OperatorKeyring.Secret + pass := opk.PassphraseSecretRef.SecretName + + if opk.SecretName != "" && opk.SecretName == pass { + return fmt.Errorf("validator: operatorKeyring data Secret %q must differ from its passphrase Secret", opk.SecretName) + } + if sk := v.SigningKey; sk != nil && sk.Secret != nil && sk.Secret.SecretName != "" { + if opk.SecretName == sk.Secret.SecretName { + return fmt.Errorf("validator: operatorKeyring Secret %q must differ from signingKey Secret", opk.SecretName) + } + if pass != "" && pass == sk.Secret.SecretName { + return fmt.Errorf("validator: operatorKeyring passphrase Secret %q must differ from signingKey Secret", pass) + } + } + if nk := v.NodeKey; nk != nil && nk.Secret != nil && nk.Secret.SecretName != "" { + if opk.SecretName == nk.Secret.SecretName { + return fmt.Errorf("validator: operatorKeyring Secret %q must differ from nodeKey Secret", opk.SecretName) + } + if pass != "" && pass == nk.Secret.SecretName { + return fmt.Errorf("validator: operatorKeyring passphrase Secret %q must differ from nodeKey Secret", pass) + } + } + return nil +} + func (p *validatorPlanner) BuildPlan(node *seiv1alpha1.SeiNode) (*seiv1alpha1.TaskPlan, error) { if node.Status.Phase == seiv1alpha1.PhaseRunning { return buildRunningPlan(node) diff --git a/internal/planner/validator_test.go b/internal/planner/validator_test.go index ca744c77..a15b2655 100644 --- a/internal/planner/validator_test.go +++ b/internal/planner/validator_test.go @@ -139,6 +139,127 @@ func TestValidatorPlanner_Validate_SigningKey(t *testing.T) { } } +func TestValidatorPlanner_Validate_OperatorKeyringDistinctness(t *testing.T) { + const ( + sk = "validator-0-signing" + nk = "validator-0-nodekey" + opk = "validator-0-opk" + pass = "validator-0-opk-pass" + ) + cases := []struct { + name string + spec seiv1alpha1.ValidatorSpec + wantErr string + }{ + { + name: "all four secrets distinct is valid", + spec: seiv1alpha1.ValidatorSpec{ + SigningKey: &seiv1alpha1.SigningKeySource{Secret: &seiv1alpha1.SecretSigningKeySource{SecretName: sk}}, + NodeKey: &seiv1alpha1.NodeKeySource{Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: nk}}, + OperatorKeyring: &seiv1alpha1.OperatorKeyringSource{ + Secret: &seiv1alpha1.SecretOperatorKeyringSource{ + SecretName: opk, + PassphraseSecretRef: seiv1alpha1.PassphraseSecretRef{SecretName: pass}, + }, + }, + }, + }, + { + name: "operatorKeyring Secret equals signingKey Secret is rejected", + spec: seiv1alpha1.ValidatorSpec{ + SigningKey: &seiv1alpha1.SigningKeySource{Secret: &seiv1alpha1.SecretSigningKeySource{SecretName: sk}}, + NodeKey: &seiv1alpha1.NodeKeySource{Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: nk}}, + OperatorKeyring: &seiv1alpha1.OperatorKeyringSource{ + Secret: &seiv1alpha1.SecretOperatorKeyringSource{ + SecretName: sk, + PassphraseSecretRef: seiv1alpha1.PassphraseSecretRef{SecretName: pass}, + }, + }, + }, + wantErr: "operatorKeyring Secret", + }, + { + name: "operatorKeyring Secret equals nodeKey Secret is rejected", + spec: seiv1alpha1.ValidatorSpec{ + SigningKey: &seiv1alpha1.SigningKeySource{Secret: &seiv1alpha1.SecretSigningKeySource{SecretName: sk}}, + NodeKey: &seiv1alpha1.NodeKeySource{Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: nk}}, + OperatorKeyring: &seiv1alpha1.OperatorKeyringSource{ + Secret: &seiv1alpha1.SecretOperatorKeyringSource{ + SecretName: nk, + PassphraseSecretRef: seiv1alpha1.PassphraseSecretRef{SecretName: pass}, + }, + }, + }, + wantErr: "must differ from nodeKey Secret", + }, + { + name: "operatorKeyring Secret equals its own passphrase Secret is rejected", + spec: seiv1alpha1.ValidatorSpec{ + OperatorKeyring: &seiv1alpha1.OperatorKeyringSource{ + Secret: &seiv1alpha1.SecretOperatorKeyringSource{ + SecretName: opk, + PassphraseSecretRef: seiv1alpha1.PassphraseSecretRef{SecretName: opk}, + }, + }, + }, + wantErr: "must differ from its passphrase Secret", + }, + { + name: "passphrase Secret equals signingKey Secret is rejected", + spec: seiv1alpha1.ValidatorSpec{ + SigningKey: &seiv1alpha1.SigningKeySource{Secret: &seiv1alpha1.SecretSigningKeySource{SecretName: sk}}, + NodeKey: &seiv1alpha1.NodeKeySource{Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: nk}}, + OperatorKeyring: &seiv1alpha1.OperatorKeyringSource{ + Secret: &seiv1alpha1.SecretOperatorKeyringSource{ + SecretName: opk, + PassphraseSecretRef: seiv1alpha1.PassphraseSecretRef{SecretName: sk}, + }, + }, + }, + wantErr: "passphrase Secret", + }, + { + name: "passphrase Secret equals nodeKey Secret is rejected", + spec: seiv1alpha1.ValidatorSpec{ + SigningKey: &seiv1alpha1.SigningKeySource{Secret: &seiv1alpha1.SecretSigningKeySource{SecretName: sk}}, + NodeKey: &seiv1alpha1.NodeKeySource{Secret: &seiv1alpha1.SecretNodeKeySource{SecretName: nk}}, + OperatorKeyring: &seiv1alpha1.OperatorKeyringSource{ + Secret: &seiv1alpha1.SecretOperatorKeyringSource{ + SecretName: opk, + PassphraseSecretRef: seiv1alpha1.PassphraseSecretRef{SecretName: nk}, + }, + }, + }, + wantErr: "passphrase Secret", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + node := &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: "validator-0", Namespace: "pacific-1"}, + Spec: seiv1alpha1.SeiNodeSpec{ + ChainID: "pacific-1", + Image: "seid:v6.4.1", + Validator: &tc.spec, + }, + } + err := (&validatorPlanner{}).Validate(node) + if tc.wantErr == "" { + if err != nil { + t.Fatalf("Validate: unexpected error: %v", err) + } + return + } + if err == nil { + t.Fatalf("Validate: expected error containing %q, got nil", tc.wantErr) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("Validate: error = %q, want containing %q", err.Error(), tc.wantErr) + } + }) + } +} + // taskTypes returns the ordered list of task types in a plan, for assertions. func taskTypes(plan *seiv1alpha1.TaskPlan) []string { out := make([]string, len(plan.Tasks)) diff --git a/internal/task/apply_statefulset.go b/internal/task/apply_statefulset.go index f2939ce7..4b8c745c 100644 --- a/internal/task/apply_statefulset.go +++ b/internal/task/apply_statefulset.go @@ -49,7 +49,10 @@ func (e *applyStatefulSetExecution) Execute(ctx context.Context) error { return err } - desired := noderesource.GenerateStatefulSet(node, e.cfg.Platform) + desired, err := noderesource.GenerateStatefulSet(node, e.cfg.Platform) + if err != nil { + return Terminal(fmt.Errorf("generating statefulset: %w", err)) + } desired.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("StatefulSet")) if err := ctrl.SetControllerReference(node, desired, e.cfg.Scheme); err != nil { return fmt.Errorf("setting owner reference on statefulset: %w", err) diff --git a/internal/task/bootstrap_resources.go b/internal/task/bootstrap_resources.go index 09ca3cf6..09e9bda0 100644 --- a/internal/task/bootstrap_resources.go +++ b/internal/task/bootstrap_resources.go @@ -361,7 +361,7 @@ func assertNoValidatorSecretsOnBootstrapPod(node *seiv1alpha1.SeiNode, spec *cor } // Env injection is a separate leakage path — kubelet resolves - // valueFrom.secretKeyRef regardless of volume mounts. + // valueFrom.secretKeyRef and envFrom.secretRef regardless of volume mounts. allContainers := make([]corev1.Container, 0, len(spec.Containers)+len(spec.InitContainers)) allContainers = append(allContainers, spec.Containers...) allContainers = append(allContainers, spec.InitContainers...) @@ -378,6 +378,18 @@ func assertNoValidatorSecretsOnBootstrapPod(node *seiv1alpha1.SeiNode, spec *cor } } } + for _, ef := range c.EnvFrom { + if ef.SecretRef == nil { + continue + } + for _, f := range forbiddens { + if ef.SecretRef.Name == f.name { + return fmt.Errorf("bootstrap pod-spec for %s/%s references %s Secret %q in container %q envFrom; "+ + "bootstrap pods must never carry validator-owned credentials", + node.Namespace, node.Name, f.kind, f.name, c.Name) + } + } + } } return nil diff --git a/internal/task/bootstrap_resources_test.go b/internal/task/bootstrap_resources_test.go index 624e176a..0f1e0d09 100644 --- a/internal/task/bootstrap_resources_test.go +++ b/internal/task/bootstrap_resources_test.go @@ -168,6 +168,32 @@ func TestAssertNoValidatorSecretsOnBootstrapPod_PassphraseEnvInInitContainer_Rej } } +func TestAssertNoValidatorSecretsOnBootstrapPod_PassphraseEnvFrom_Rejects(t *testing.T) { + node := validatorNodeWithSecrets("sk", "opk", bootstrapTestPassphraseSecret) + spec := &corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: bootstrapTestSidecarContainer, + EnvFrom: []corev1.EnvFromSource{ + {SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: bootstrapTestPassphraseSecret}, + }}, + }, + }, + }, + } + err := assertNoValidatorSecretsOnBootstrapPod(node, spec) + if err == nil { + t.Fatal("expected error for passphrase envFrom on bootstrap pod, got nil") + } + if !strings.Contains(err.Error(), "operator-keyring-passphrase") || !strings.Contains(err.Error(), bootstrapTestSidecarContainer) { + t.Fatalf("expected error to mention passphrase kind and container name, got: %v", err) + } + if !strings.Contains(err.Error(), "envFrom") { + t.Fatalf("expected error to mention envFrom path, got: %v", err) + } +} + func TestAssertNoValidatorSecretsOnBootstrapPod_NodeKeyExcluded(t *testing.T) { node := &seiv1alpha1.SeiNode{ ObjectMeta: metav1.ObjectMeta{Name: opkValidatorName, Namespace: opkNs}, diff --git a/internal/task/validate_operator_keyring.go b/internal/task/validate_operator_keyring.go index bb730cb4..58d0aac3 100644 --- a/internal/task/validate_operator_keyring.go +++ b/internal/task/validate_operator_keyring.go @@ -17,9 +17,9 @@ import ( const TaskTypeValidateOperatorKeyring = "validate-operator-keyring" -// Cosmos SDK file-backend keyring layout: each entry materializes as -// ".info" (encrypted protobuf-marshaled LocalInfo) plus one or more -// ".address" name→address index files. +// File-keyring layout (documented in the operator runbook): each keyring +// entry materializes as a ".info" data key plus one or more +// ".address" name→address index keys. const ( keyringInfoSuffix = ".info" keyringAddressSuffix = ".address" @@ -82,8 +82,8 @@ func (e *validateOperatorKeyringExecution) Status(_ context.Context) ExecutionSt } // validate checks the shape of both Secrets. It deliberately does NOT -// attempt to decrypt the keyring with the passphrase — running the Cosmos -// SDK keyring backend inside the controller process would expand the +// attempt to decrypt the keyring with the passphrase — running the +// keyring backend inside the controller process would expand the // controller's TCB. Decryption is the sidecar's startup smoke test. func (e *validateOperatorKeyringExecution) validate(ctx context.Context, node *seiv1alpha1.SeiNode) error { if e.params.SecretName == "" { @@ -126,16 +126,16 @@ func (e *validateOperatorKeyringExecution) getSecret(ctx context.Context, name, return secret, nil } -// validateKeyringShape walks Secret data keys looking for the Cosmos SDK -// file-backend layout. Empty Secrets and Secrets missing either suffix -// are operator-fixable defects (Terminal). +// validateKeyringShape walks Secret data keys looking for the file-keyring +// layout. Empty Secrets and Secrets missing either suffix are +// operator-fixable defects (Terminal). func validateKeyringShape(secret *corev1.Secret, keyName string) error { var infoKeys, addressKeys []string for k, v := range secret.Data { switch { case strings.HasSuffix(k, keyringInfoSuffix): if len(v) == 0 { - return Terminal(fmt.Errorf("secret %q data key %q is empty; file-keyring info blobs must be non-empty", + return Terminal(fmt.Errorf("secret %q data key %q is empty (expected non-empty keyring entry payload)", secret.Name, k)) } infoKeys = append(infoKeys, k) @@ -144,11 +144,11 @@ func validateKeyringShape(secret *corev1.Secret, keyName string) error { } } if len(infoKeys) == 0 { - return Terminal(fmt.Errorf("secret %q has no %q data keys; file-keyring requires at least one encrypted key blob", + return Terminal(fmt.Errorf("secret %q has no %q data keys (expected at least one keyring entry)", secret.Name, "*"+keyringInfoSuffix)) } if len(addressKeys) == 0 { - return Terminal(fmt.Errorf("secret %q has no %q data keys; file-keyring requires at least one name→address index", + return Terminal(fmt.Errorf("secret %q has no %q data keys (expected name→address index for at least one keyring entry)", secret.Name, "*"+keyringAddressSuffix)) } if keyName != "" { diff --git a/manifests/sei.io_seinodedeployments.yaml b/manifests/sei.io_seinodedeployments.yaml index 86a91f5f..b1d87513 100644 --- a/manifests/sei.io_seinodedeployments.yaml +++ b/manifests/sei.io_seinodedeployments.yaml @@ -711,6 +711,9 @@ spec: "node_admin" to preserve continuity with the seienv convention. Mutable — rotating to a different entry within the same Secret is a routine operator-account change, not a slashing risk. + + The default literal below MUST match DefaultOperatorKeyName — + kubebuilder markers cannot reference Go constants. maxLength: 64 pattern: ^[a-zA-Z0-9_-]+$ type: string @@ -721,8 +724,11 @@ spec: properties: key: default: passphrase - description: Key is the data key inside the - Secret. Defaults to "passphrase". + description: |- + Key is the data key inside the Secret. Defaults to "passphrase". + + The default literal below MUST match DefaultPassphraseSecretKey — + kubebuilder markers cannot reference Go constants. maxLength: 253 type: string secretName: @@ -864,6 +870,16 @@ spec: must be distinct rule: '!has(self.operatorKeyring) || self.operatorKeyring.secret.secretName != self.operatorKeyring.secret.passphraseSecretRef.secretName' + - message: operatorKeyring passphrase Secret must not equal + signingKey Secret + rule: '!has(self.operatorKeyring) || !has(self.signingKey) + || self.operatorKeyring.secret.passphraseSecretRef.secretName + != self.signingKey.secret.secretName' + - message: operatorKeyring passphrase Secret must not equal + nodeKey Secret + rule: '!has(self.operatorKeyring) || !has(self.nodeKey) + || self.operatorKeyring.secret.passphraseSecretRef.secretName + != self.nodeKey.secret.secretName' required: - chainId - image diff --git a/manifests/sei.io_seinodes.yaml b/manifests/sei.io_seinodes.yaml index 5286662c..5eb04e54 100644 --- a/manifests/sei.io_seinodes.yaml +++ b/manifests/sei.io_seinodes.yaml @@ -566,6 +566,9 @@ spec: "node_admin" to preserve continuity with the seienv convention. Mutable — rotating to a different entry within the same Secret is a routine operator-account change, not a slashing risk. + + The default literal below MUST match DefaultOperatorKeyName — + kubebuilder markers cannot reference Go constants. maxLength: 64 pattern: ^[a-zA-Z0-9_-]+$ type: string @@ -576,8 +579,11 @@ spec: properties: key: default: passphrase - description: Key is the data key inside the Secret. - Defaults to "passphrase". + description: |- + Key is the data key inside the Secret. Defaults to "passphrase". + + The default literal below MUST match DefaultPassphraseSecretKey — + kubebuilder markers cannot reference Go constants. maxLength: 253 type: string secretName: @@ -716,6 +722,14 @@ spec: be distinct rule: '!has(self.operatorKeyring) || self.operatorKeyring.secret.secretName != self.operatorKeyring.secret.passphraseSecretRef.secretName' + - message: operatorKeyring passphrase Secret must not equal signingKey + Secret + rule: '!has(self.operatorKeyring) || !has(self.signingKey) || self.operatorKeyring.secret.passphraseSecretRef.secretName + != self.signingKey.secret.secretName' + - message: operatorKeyring passphrase Secret must not equal nodeKey + Secret + rule: '!has(self.operatorKeyring) || !has(self.nodeKey) || self.operatorKeyring.secret.passphraseSecretRef.secretName + != self.nodeKey.secret.secretName' required: - chainId - image From 1792267de6c3755f6b016ee0b435942ce336a2a5 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Mon, 11 May 2026 18:05:49 -0700 Subject: [PATCH 3/5] review: address Cursor findings on sidecar hardening Cursor bugbot flagged two real regressions introduced by the sidecar SecurityContext + pod-level FSGroup additions: 1. ReadOnlyRootFilesystem: true breaks any Go stdlib path that writes to /tmp (os.CreateTemp, net/http multipart, encoding/json large buffers). Adds an emptyDir volume mounted at /tmp on the sidecar container only. Tmpfs-backed, isolated from the seid container's TMPDIR=/sei/tmp. 2. Pod-level FSGroup without FSGroupChangePolicy defaults to "Always", causing kubelet to recursively chown every file on every mounted PVC at every pod start. For Sei archive PVCs (TBs of chain data) this adds minutes of startup latency on every restart. Setting FSGroupChangePolicy: OnRootMismatch chowns on first creation only. Co-Authored-By: Claude Opus 4.7 (1M context) --- internal/noderesource/noderesource.go | 28 ++++++++++++++++------ internal/noderesource/noderesource_test.go | 8 +++++-- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/internal/noderesource/noderesource.go b/internal/noderesource/noderesource.go index 6a8d1570..92c80925 100644 --- a/internal/noderesource/noderesource.go +++ b/internal/noderesource/noderesource.go @@ -46,6 +46,10 @@ const ( operatorKeyringDirName = "keyring-file" keyringPassphraseEnvVar = "SEI_KEYRING_PASSPHRASE" + // sidecarTmpVolumeName backs an emptyDir at /tmp — required because the + // sidecar runs with ReadOnlyRootFilesystem and Go stdlib defaults to /tmp. + sidecarTmpVolumeName = "sidecar-tmp" + // sidecarNonRootUID is the nonroot UID/GID baked into distroless and // chainguard static-debian12 base images. Pod-level fsGroup matches so // the non-root sidecar can read kubelet-projected 0o400 Secret files. @@ -322,8 +326,12 @@ func buildNodePodSpec(node *seiv1alpha1.SeiNode, p PlatformConfig) corev1.PodSpe signingVolumes := signingKeyVolumes(node) nodeVolumes := nodeKeyVolumes(node) keyringVolumes := operatorKeyringVolumes(node) - volumes := make([]corev1.Volume, 0, 1+len(signingVolumes)+len(nodeVolumes)+len(keyringVolumes)) - volumes = append(volumes, dataVolume) + sidecarTmpVolume := corev1.Volume{ + Name: sidecarTmpVolumeName, + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + } + volumes := make([]corev1.Volume, 0, 2+len(signingVolumes)+len(nodeVolumes)+len(keyringVolumes)) + volumes = append(volumes, dataVolume, sidecarTmpVolume) volumes = append(volumes, signingVolumes...) volumes = append(volumes, nodeVolumes...) volumes = append(volumes, keyringVolumes...) @@ -365,11 +373,14 @@ func buildNodePodSpec(node *seiv1alpha1.SeiNode, p PlatformConfig) corev1.PodSpe // SecurityContext, separate the sidecar's SA) is tracked as a follow-up. // See PR #220 review thread. spec.ShareProcessNamespace = ptr.To(true) - // fsGroup is required so the non-root sidecar (UID 65532) can read - // 0o400 Secret-projected files (operator keyring) kubelet owns root:root. + // FSGroup grants the non-root sidecar UID read access to 0o400 + // Secret-projected files. ChangePolicy=OnRootMismatch avoids recursive + // chown on every pod start (the "Always" default is costly on archive PVCs). fsGroup := sidecarNonRootUID + fsGroupChangePolicy := corev1.FSGroupChangeOnRootMismatch spec.SecurityContext = &corev1.PodSecurityContext{ - FSGroup: &fsGroup, + FSGroup: &fsGroup, + FSGroupChangePolicy: &fsGroupChangePolicy, } spec.InitContainers = []corev1.Container{ buildSeidInitContainer(node), @@ -411,8 +422,11 @@ func buildSidecarContainer(node *seiv1alpha1.SeiNode, p PlatformConfig) corev1.C env = append(env, keyringEnv...) keyringMounts := operatorKeyringMounts(node) - mounts := make([]corev1.VolumeMount, 0, 1+len(keyringMounts)) - mounts = append(mounts, corev1.VolumeMount{Name: "data", MountPath: dataDir}) + mounts := make([]corev1.VolumeMount, 0, 2+len(keyringMounts)) + mounts = append(mounts, + corev1.VolumeMount{Name: "data", MountPath: dataDir}, + corev1.VolumeMount{Name: sidecarTmpVolumeName, MountPath: "/tmp"}, + ) mounts = append(mounts, keyringMounts...) c := corev1.Container{ diff --git a/internal/noderesource/noderesource_test.go b/internal/noderesource/noderesource_test.go index 9a2a17ab..3cb3ad55 100644 --- a/internal/noderesource/noderesource_test.go +++ b/internal/noderesource/noderesource_test.go @@ -187,8 +187,10 @@ func TestBuildNodePodSpec_Genesis_MountsExistingPVC(t *testing.T) { spec := buildNodePodSpec(node, platformtest.Config()) g.Expect(spec.ServiceAccountName).To(Equal(platformtest.Config().ServiceAccount)) - g.Expect(spec.Volumes).To(HaveLen(1)) + g.Expect(spec.Volumes).To(HaveLen(2)) // data PVC + sidecar-tmp emptyDir g.Expect(spec.Volumes[0].PersistentVolumeClaim.ClaimName).To(Equal("data-mynet-0")) + g.Expect(spec.Volumes[1].Name).To(Equal(sidecarTmpVolumeName)) + g.Expect(spec.Volumes[1].EmptyDir).NotTo(BeNil()) } func TestBuildNodePodSpec_Snapshot_MountsNodePVC(t *testing.T) { @@ -410,8 +412,10 @@ func TestSidecarContainer_DataVolumeMount(t *testing.T) { sts := mustGenerateStatefulSet(t, node, platformtest.Config()) sc := findInitContainer(sts.Spec.Template.Spec.InitContainers, "sei-sidecar") - g.Expect(sc.VolumeMounts).To(HaveLen(1)) + g.Expect(sc.VolumeMounts).To(HaveLen(2)) g.Expect(sc.VolumeMounts[0].MountPath).To(Equal(dataDir)) + g.Expect(sc.VolumeMounts[1].Name).To(Equal(sidecarTmpVolumeName)) + g.Expect(sc.VolumeMounts[1].MountPath).To(Equal("/tmp")) } func TestSidecarContainer_CustomPort(t *testing.T) { From d55487ae57b6588b638e85a3a6f638b65fd4b9d9 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 12 May 2026 07:26:06 -0700 Subject: [PATCH 4/5] review: address PR #220 line comments - Cut speculative "future siblings" doc text on OperatorKeyringSource (per YAGNI; the additive CEL shape `(has(self.secret) ? 1 : 0) == 1` speaks for itself, doesn't need prose elaboration about hypothetical variants). - Remove the `+kubebuilder:default="passphrase"` on PassphraseSecretRef.Key and make Key required. Operators declare the data key explicitly rather than relying on a default that hides where the passphrase actually lives. KeyName's "node_admin" default is kept as the seienv-continuity convention. - Drop the now-unused DefaultPassphraseSecretKey constant and the planner+noderesource fallback code (Key is required at admission; no in-code fallback needed). - Refine the operatorKeyringDirName comment to make clear it's an SDK contract, not a controller choice. - Move the `forbiddenSecret` type declaration to the top of `bootstrap_resources.go` per Go convention (types alongside package declarations). - Update the AutomountServiceAccountToken comment to reflect rev2 of the design (the proxy uses the SA token for TokenReview, not the sidecar's in-process middleware). No substantive change to the keyring CRD design. The kube-rbac-proxy work from rev2 is fast-follow scope (#165 expanded). Co-Authored-By: Claude Opus 4.7 (1M context) --- api/v1alpha1/validator_types.go | 32 +++++++---------------- config/crd/sei.io_seinodedeployments.yaml | 10 +++---- config/crd/sei.io_seinodes.yaml | 10 +++---- internal/noderesource/noderesource.go | 21 +++++++-------- internal/planner/planner.go | 11 +++----- internal/task/bootstrap_resources.go | 7 +++-- manifests/sei.io_seinodedeployments.yaml | 10 +++---- manifests/sei.io_seinodes.yaml | 10 +++---- 8 files changed, 48 insertions(+), 63 deletions(-) diff --git a/api/v1alpha1/validator_types.go b/api/v1alpha1/validator_types.go index 61b0bad3..23547281 100644 --- a/api/v1alpha1/validator_types.go +++ b/api/v1alpha1/validator_types.go @@ -1,14 +1,10 @@ package v1alpha1 -// Defaults for SecretOperatorKeyringSource fields. Kept in lockstep with the -// +kubebuilder:default markers on the corresponding struct fields and -// referenced by the planner + noderesource packages so defaulting stays -// consistent even when admission webhooks haven't run (e.g., in-memory -// specs in tests, or controllers reading pre-defaulted objects). -const ( - DefaultOperatorKeyName = "node_admin" - DefaultPassphraseSecretKey = "passphrase" -) +// DefaultOperatorKeyName matches the +kubebuilder:default on +// SecretOperatorKeyringSource.KeyName. Referenced by the planner and +// noderesource packages so defaulting stays consistent when admission +// webhooks haven't run (e.g. in-memory specs in tests). +const DefaultOperatorKeyName = "node_admin" // ValidatorSpec configures a consensus-participating validator node. // Validators bootstrap the same way as full nodes but participate in consensus. @@ -144,18 +140,12 @@ type SecretNodeKeySource struct { // withdraw-rewards, and other operator-account transactions) comes from. // Exactly one variant must be set; variants are mutually exclusive. // -// The CEL rule is shaped as a sum across all variants so adding a future -// sibling (TMKMS, Vault, KMS) is purely additive — bump the sum by one term. -// // +kubebuilder:validation:XValidation:rule="(has(self.secret) ? 1 : 0) == 1",message="exactly one operator keyring source must be set" type OperatorKeyringSource struct { // Secret loads a Cosmos SDK file-backend keyring from a Kubernetes Secret // in the SeiNode's namespace. // +optional Secret *SecretOperatorKeyringSource `json:"secret,omitempty"` - - // Future siblings: TMKMS, Vault, KMS. When added, bump the - // XValidation sum-of-variants rule. } // SecretOperatorKeyringSource references the Kubernetes Secrets that supply @@ -210,15 +200,13 @@ type PassphraseSecretRef struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="passphrase secretName is immutable" SecretName string `json:"secretName"` - // Key is the data key inside the Secret. Defaults to "passphrase". - // - // The default literal below MUST match DefaultPassphraseSecretKey — - // kubebuilder markers cannot reference Go constants. + // Key is the data key inside the Secret holding the passphrase. + // Required — operators declare this explicitly rather than relying on + // a default that hides where the passphrase actually lives. // - // +optional - // +kubebuilder:default="passphrase" + // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=253 - Key string `json:"key,omitempty"` + Key string `json:"key"` } // GenesisCeremonyNodeConfig holds per-node genesis ceremony parameters. diff --git a/config/crd/sei.io_seinodedeployments.yaml b/config/crd/sei.io_seinodedeployments.yaml index b1d87513..14b074e6 100644 --- a/config/crd/sei.io_seinodedeployments.yaml +++ b/config/crd/sei.io_seinodedeployments.yaml @@ -723,13 +723,12 @@ spec: unlock passphrase. Required for the file backend. properties: key: - default: passphrase description: |- - Key is the data key inside the Secret. Defaults to "passphrase". - - The default literal below MUST match DefaultPassphraseSecretKey — - kubebuilder markers cannot reference Go constants. + Key is the data key inside the Secret holding the passphrase. + Required — operators declare this explicitly rather than relying on + a default that hides where the passphrase actually lives. maxLength: 253 + minLength: 1 type: string secretName: description: SecretName names the passphrase @@ -742,6 +741,7 @@ spec: - message: passphrase secretName is immutable rule: self == oldSelf required: + - key - secretName type: object secretName: diff --git a/config/crd/sei.io_seinodes.yaml b/config/crd/sei.io_seinodes.yaml index 5eb04e54..11e1b88e 100644 --- a/config/crd/sei.io_seinodes.yaml +++ b/config/crd/sei.io_seinodes.yaml @@ -578,13 +578,12 @@ spec: unlock passphrase. Required for the file backend. properties: key: - default: passphrase description: |- - Key is the data key inside the Secret. Defaults to "passphrase". - - The default literal below MUST match DefaultPassphraseSecretKey — - kubebuilder markers cannot reference Go constants. + Key is the data key inside the Secret holding the passphrase. + Required — operators declare this explicitly rather than relying on + a default that hides where the passphrase actually lives. maxLength: 253 + minLength: 1 type: string secretName: description: SecretName names the passphrase Secret @@ -597,6 +596,7 @@ spec: - message: passphrase secretName is immutable rule: self == oldSelf required: + - key - secretName type: object secretName: diff --git a/internal/noderesource/noderesource.go b/internal/noderesource/noderesource.go index 92c80925..315d7c80 100644 --- a/internal/noderesource/noderesource.go +++ b/internal/noderesource/noderesource.go @@ -40,9 +40,9 @@ const ( nodeKeyDataKey = "node_key.json" operatorKeyringVolumeName = "operator-keyring" - // operatorKeyringDirName matches the on-disk directory the Cosmos SDK - // file-backend keyring appends to $SEI_HOME — keyring.New(name, BackendFile, - // homeDir, ...) implicitly opens homeDir/keyring-file/. + // operatorKeyringDirName is fixed by the Cosmos SDK file-backend keyring: + // keyring.New(name, BackendFile, homeDir, ...) opens homeDir/keyring-file/. + // Not a controller choice; this constant mirrors the SDK contract. operatorKeyringDirName = "keyring-file" keyringPassphraseEnvVar = "SEI_KEYRING_PASSPHRASE" @@ -339,10 +339,11 @@ func buildNodePodSpec(node *seiv1alpha1.SeiNode, p PlatformConfig) corev1.PodSpe pool := p.NodepoolForMode(NodeMode(node)) spec := corev1.PodSpec{ - // AutomountServiceAccountToken is explicit here because the sidecar's - // future TokenReview-based authn (sei-protocol/seictl#165) requires - // the SA token mount. Cluster-default flips that silently disable - // this would break the auth path. + // AutomountServiceAccountToken is explicit here because the future + // kube-rbac-proxy fronting the sidecar API (sei-protocol/seictl#165) + // calls TokenReview + SubjectAccessReview against the K8s API using + // the pod's projected SA token. Cluster-default flips that silently + // disable this would break the auth path. AutomountServiceAccountToken: ptr.To(true), //nolint:modernize // ptr.To(true) is idiomatic; new(true) is invalid Go ServiceAccountName: p.ServiceAccount, Tolerations: []corev1.Toleration{ @@ -723,16 +724,12 @@ func operatorKeyringEnvVars(node *seiv1alpha1.SeiNode) []corev1.EnvVar { if src == nil { return nil } - key := src.PassphraseSecretRef.Key - if key == "" { - key = seiv1alpha1.DefaultPassphraseSecretKey - } return []corev1.EnvVar{{ Name: keyringPassphraseEnvVar, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: src.PassphraseSecretRef.SecretName}, - Key: key, + Key: src.PassphraseSecretRef.Key, }, }, }} diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 35b27282..2bfc9b28 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -364,21 +364,18 @@ func validateOperatorKeyringParams(node *seiv1alpha1.SeiNode) any { return nil } s := node.Spec.Validator.OperatorKeyring.Secret - // Defaults mirror the CRD kubebuilder:default markers so plans built - // from a partially-defaulted in-memory spec still get the right values. + // KeyName falls back to the CRD default for in-memory specs that + // haven't been through admission defaulting; PassphraseSecretRef.Key + // is required (no fallback). keyName := s.KeyName if keyName == "" { keyName = seiv1alpha1.DefaultOperatorKeyName } - passphraseKey := s.PassphraseSecretRef.Key - if passphraseKey == "" { - passphraseKey = seiv1alpha1.DefaultPassphraseSecretKey - } return &task.ValidateOperatorKeyringParams{ SecretName: s.SecretName, KeyName: keyName, PassphraseSecretName: s.PassphraseSecretRef.SecretName, - PassphraseSecretKey: passphraseKey, + PassphraseSecretKey: s.PassphraseSecretRef.Key, Namespace: node.Namespace, } } diff --git a/internal/task/bootstrap_resources.go b/internal/task/bootstrap_resources.go index 09e9bda0..ee98fda1 100644 --- a/internal/task/bootstrap_resources.go +++ b/internal/task/bootstrap_resources.go @@ -23,6 +23,11 @@ const ( bootstrapComponentLabel = "sei.io/component" ) +// forbiddenSecret pairs a Secret name with a human-readable kind, used by the +// bootstrap-pod isolation guard to reject any validator-owned credential +// material on the bootstrap pod-spec. +type forbiddenSecret struct{ name, kind string } + // BootstrapJobName returns the bootstrap Job name for a node. func BootstrapJobName(node *seiv1alpha1.SeiNode) string { return fmt.Sprintf("%s-bootstrap", node.Name) @@ -395,8 +400,6 @@ func assertNoValidatorSecretsOnBootstrapPod(node *seiv1alpha1.SeiNode, spec *cor return nil } -type forbiddenSecret struct{ name, kind string } - func forbiddenBootstrapSecrets(node *seiv1alpha1.SeiNode) []forbiddenSecret { var out []forbiddenSecret if sk := node.Spec.Validator.SigningKey; sk != nil && sk.Secret != nil { diff --git a/manifests/sei.io_seinodedeployments.yaml b/manifests/sei.io_seinodedeployments.yaml index b1d87513..14b074e6 100644 --- a/manifests/sei.io_seinodedeployments.yaml +++ b/manifests/sei.io_seinodedeployments.yaml @@ -723,13 +723,12 @@ spec: unlock passphrase. Required for the file backend. properties: key: - default: passphrase description: |- - Key is the data key inside the Secret. Defaults to "passphrase". - - The default literal below MUST match DefaultPassphraseSecretKey — - kubebuilder markers cannot reference Go constants. + Key is the data key inside the Secret holding the passphrase. + Required — operators declare this explicitly rather than relying on + a default that hides where the passphrase actually lives. maxLength: 253 + minLength: 1 type: string secretName: description: SecretName names the passphrase @@ -742,6 +741,7 @@ spec: - message: passphrase secretName is immutable rule: self == oldSelf required: + - key - secretName type: object secretName: diff --git a/manifests/sei.io_seinodes.yaml b/manifests/sei.io_seinodes.yaml index 5eb04e54..11e1b88e 100644 --- a/manifests/sei.io_seinodes.yaml +++ b/manifests/sei.io_seinodes.yaml @@ -578,13 +578,12 @@ spec: unlock passphrase. Required for the file backend. properties: key: - default: passphrase description: |- - Key is the data key inside the Secret. Defaults to "passphrase". - - The default literal below MUST match DefaultPassphraseSecretKey — - kubebuilder markers cannot reference Go constants. + Key is the data key inside the Secret holding the passphrase. + Required — operators declare this explicitly rather than relying on + a default that hides where the passphrase actually lives. maxLength: 253 + minLength: 1 type: string secretName: description: SecretName names the passphrase Secret @@ -597,6 +596,7 @@ spec: - message: passphrase secretName is immutable rule: self == oldSelf required: + - key - secretName type: object secretName: From 1028fb17c9d381aea2e17861bb6d828388e56db9 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 12 May 2026 07:39:56 -0700 Subject: [PATCH 5/5] review: production pod guard walks env vars for passphrase Secret MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cursor bugbot flagged an asymmetric gap — assertNoOperatorKeyringOnSeidContainers only walked volume mounts, while its bootstrap-pod sibling (assertNoValidatorSecretsOnBootstrapPod) walks both volumes AND env vars. The passphrase env is equally sensitive operator-keyring material: a future refactor that accidentally injects SEI_KEYRING_PASSPHRASE into the seid main container (or a non-sidecar init container) would slip past the volume-only check. Extends the production guard to also walk Env[].ValueFrom.SecretKeyRef.Name and EnvFrom[].SecretRef.Name for matches against the operator-keyring passphrase Secret name. Adds two regression tests covering both leakage paths. Co-Authored-By: Claude Opus 4.7 (1M context) --- internal/noderesource/noderesource.go | 31 +++++++++---- internal/noderesource/noderesource_test.go | 52 +++++++++++++++++++--- 2 files changed, 70 insertions(+), 13 deletions(-) diff --git a/internal/noderesource/noderesource.go b/internal/noderesource/noderesource.go index 315d7c80..e4371649 100644 --- a/internal/noderesource/noderesource.go +++ b/internal/noderesource/noderesource.go @@ -190,18 +190,18 @@ func GenerateStatefulSet(node *seiv1alpha1.SeiNode, p PlatformConfig) (*appsv1.S } // assertNoOperatorKeyringOnSeidContainers fails closed if a future refactor -// lands the operator-keyring volume on the seid main container or a -// non-sidecar init container. Operator-keyring material is the sidecar's -// alone: a compromised seid container reading that mount would collapse -// the sidecar/seid trust boundary. +// lands operator-keyring material on the seid main container or a non-sidecar +// init container. Checks both the keyring volume mount AND env-var references +// to the passphrase Secret — either alone is enough for a compromised seid +// container to recover the unlocked operator key. // -// No-op when the node has no operator-keyring configured — the volume is -// only present when SecretOperatorKeyringSource is set, so there is -// nothing to contain. +// No-op when the node has no operator-keyring configured. func assertNoOperatorKeyringOnSeidContainers(node *seiv1alpha1.SeiNode, spec *corev1.PodSpec) error { - if operatorKeyringSecretSource(node) == nil { + src := operatorKeyringSecretSource(node) + if src == nil { return nil } + passphraseSecretName := src.PassphraseSecretRef.SecretName check := func(c *corev1.Container) error { for _, m := range c.VolumeMounts { @@ -211,6 +211,21 @@ func assertNoOperatorKeyringOnSeidContainers(node *seiv1alpha1.SeiNode, spec *co node.Namespace, node.Name, c.Name) } } + for _, ev := range c.Env { + if ev.ValueFrom != nil && ev.ValueFrom.SecretKeyRef != nil && + ev.ValueFrom.SecretKeyRef.Name == passphraseSecretName { + return fmt.Errorf("pod-spec for %s/%s references operator-keyring passphrase Secret %q in env %q on container %q; "+ + "the passphrase is exclusively the sidecar's", + node.Namespace, node.Name, passphraseSecretName, ev.Name, c.Name) + } + } + for _, ef := range c.EnvFrom { + if ef.SecretRef != nil && ef.SecretRef.Name == passphraseSecretName { + return fmt.Errorf("pod-spec for %s/%s references operator-keyring passphrase Secret %q via envFrom on container %q; "+ + "the passphrase is exclusively the sidecar's", + node.Namespace, node.Name, passphraseSecretName, c.Name) + } + } return nil } diff --git a/internal/noderesource/noderesource_test.go b/internal/noderesource/noderesource_test.go index 3cb3ad55..4d2adea6 100644 --- a/internal/noderesource/noderesource_test.go +++ b/internal/noderesource/noderesource_test.go @@ -1072,9 +1072,11 @@ func TestPodSpec_FSGroup(t *testing.T) { // --- assertNoOperatorKeyringOnSeidContainers --- const ( - keyringTestSeidInitName = "seid-init" - keyringTestSidecarName = "sei-sidecar" - keyringTestMountPath = "/sei/keyring-file" + keyringTestSeidInitName = "seid-init" + keyringTestSidecarName = "sei-sidecar" + keyringTestMountPath = "/sei/keyring-file" + keyringTestPassphraseSecret = "validator-0-opk-pass" + keyringTestPassphraseSecretKey = "passphrase" ) func validatorNodeWithOperatorKeyring() *seiv1alpha1.SeiNode { @@ -1088,8 +1090,8 @@ func validatorNodeWithOperatorKeyring() *seiv1alpha1.SeiNode { Secret: &seiv1alpha1.SecretOperatorKeyringSource{ SecretName: "validator-0-opk", PassphraseSecretRef: seiv1alpha1.PassphraseSecretRef{ - SecretName: "validator-0-opk-pass", - Key: "passphrase", + SecretName: keyringTestPassphraseSecret, + Key: keyringTestPassphraseSecretKey, }, }, }, @@ -1160,6 +1162,46 @@ func TestAssertNoOperatorKeyringOnSeidContainers_SeidInitMisMounted_Rejects(t *t g.Expect(err.Error()).To(ContainSubstring(`container "seid-init"`)) } +func TestAssertNoOperatorKeyringOnSeidContainers_PassphraseEnvOnSeidMain_Rejects(t *testing.T) { + g := NewWithT(t) + node := validatorNodeWithOperatorKeyring() + spec := &corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: containerNameSeid, Env: []corev1.EnvVar{{ + Name: "SEI_KEYRING_PASSPHRASE", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: keyringTestPassphraseSecret}, + Key: keyringTestPassphraseSecretKey, + }, + }, + }}}, + }, + } + err := assertNoOperatorKeyringOnSeidContainers(node, spec) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(`passphrase Secret "` + keyringTestPassphraseSecret + `"`)) + g.Expect(err.Error()).To(ContainSubstring(`container "seid"`)) +} + +func TestAssertNoOperatorKeyringOnSeidContainers_PassphraseEnvFromOnSeidInit_Rejects(t *testing.T) { + g := NewWithT(t) + node := validatorNodeWithOperatorKeyring() + spec := &corev1.PodSpec{ + InitContainers: []corev1.Container{ + {Name: keyringTestSeidInitName, EnvFrom: []corev1.EnvFromSource{{ + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: keyringTestPassphraseSecret}, + }, + }}}, + }, + } + err := assertNoOperatorKeyringOnSeidContainers(node, spec) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(`envFrom`)) + g.Expect(err.Error()).To(ContainSubstring(`container "seid-init"`)) +} + func TestGenerateStatefulSet_ProductionPodSpec_PassesGuard(t *testing.T) { g := NewWithT(t) node := validatorNodeWithOperatorKeyring()