From cfa517ab4f06b3ca7f575338ee0f052ba558d840 Mon Sep 17 00:00:00 2001 From: GatewayJ <18332154+GatewayJ@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:39:00 +0800 Subject: [PATCH] fix(sts): rotate TLS certificates --- Cargo.lock | 131 ++++++++++++ Cargo.toml | 4 +- deploy/k8s-dev/operator-rbac.yaml | 4 +- deploy/rustfs-operator/README.md | 6 +- deploy/rustfs-operator/values.yaml | 6 +- docs/operator-user-guide.md | 3 +- docs/operator-user-guide.zh-CN.md | 3 +- e2e/Cargo.lock | 122 +++++++++++ src/lib.rs | 20 +- src/metrics.rs | 41 ++++ src/sts/tls.rs | 324 +++++++++++++++++++++++++++-- 11 files changed, 634 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 201012d..afb0733 100755 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,6 +104,45 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.17", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-broadcast" version = "0.7.2" @@ -536,6 +575,26 @@ dependencies = [ "syn", ] +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.5.5" @@ -1555,6 +1614,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1576,6 +1641,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1585,12 +1660,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1609,6 +1703,15 @@ dependencies = [ "libc", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -1663,6 +1766,7 @@ dependencies = [ "shadow-rs", "snafu", "strum", + "time", "tokio", "tokio-rustls", "tokio-stream", @@ -1674,6 +1778,7 @@ dependencies = [ "url", "utoipa", "utoipa-swagger-ui", + "x509-parser", ] [[package]] @@ -2106,6 +2211,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustls" version = "0.23.35" @@ -3370,6 +3484,23 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 2.0.17", + "time", +] + [[package]] name = "yasna" version = "0.5.2" diff --git a/Cargo.toml b/Cargo.toml index 84c682e..dac3f38 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ homepage = "https://rustfs.com" chrono = { version = "0.4", features = ["serde"] } const-str = "1.0.0" serde = { version = "1.0.228", features = ["derive"] } -tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs", "io-std", "io-util"] } +tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs", "io-std", "io-util", "sync"] } tokio-rustls = "0.26" tokio-stream = { version = "0.1", features = ["sync"] } tokio-util = { version = "0.7", features = ["io", "compat", "rt"] } @@ -30,6 +30,8 @@ rustls = { version = "0.23", default-features = false, features = ["ring"] } rustls-pemfile = "2.2.0" webpki = { package = "rustls-webpki", version = "0.103" } rcgen = "0.13" +x509-parser = "0.18.1" +time = "0.3.44" sha2 = "0.10" hmac = "0.12" hex = "0.4" diff --git a/deploy/k8s-dev/operator-rbac.yaml b/deploy/k8s-dev/operator-rbac.yaml index 7e894b2..b444497 100755 --- a/deploy/k8s-dev/operator-rbac.yaml +++ b/deploy/k8s-dev/operator-rbac.yaml @@ -92,8 +92,8 @@ subjects: name: rustfs-operator namespace: rustfs-system --- -# STS auto TLS needs to create one Secret and replace it only when a previously generated -# Secret becomes invalid. Keep write access in the operator namespace. +# STS auto TLS needs to create one Secret and replace it when a previously generated Secret +# becomes invalid, uses the legacy policy, or approaches expiry. Keep write access namespaced. apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: diff --git a/deploy/rustfs-operator/README.md b/deploy/rustfs-operator/README.md index 47c128d..8c062c6 100755 --- a/deploy/rustfs-operator/README.md +++ b/deploy/rustfs-operator/README.md @@ -123,7 +123,7 @@ manifests remain consistent. | `sts.audience` | Kubernetes TokenReview audience expected by the operator STS endpoint | `sts.rustfs.com` | | `sts.port` | Operator container port for STS | `4223` | | `sts.tls.enabled` | Serve the operator STS endpoint over TLS | `true` | -| `sts.tls.auto` | Create and replace an invalid Operator-managed STS TLS Secret with namespaced write RBAC | `true` | +| `sts.tls.auto` | Create and rotate an invalid, legacy, or soon-to-expire Operator-managed STS TLS Secret with namespaced write RBAC | `true` | | `sts.service.type` | Kubernetes Service type for STS | `ClusterIP` | | `sts.service.port` | Kubernetes Service port for STS | `4223` | @@ -135,7 +135,9 @@ POST /sts/{tenantNamespace}/{tenantName} This differs from MinIO Operator's namespace-only route. A `PolicyBinding` still lives in the Tenant namespace, but the workload must call STS with both the Tenant namespace and the Tenant name. -The STS service is HTTPS by default. When `sts.tls.auto=true`, the operator creates the fixed `sts-tls` Secret in the operator namespace with `tls.crt`, `tls.key`, and `ca.crt`. With `rbac.create=true`, the chart creates a namespaced Role that can create Secrets and update only `sts-tls`; the ClusterRole keeps all Secret and ConfigMap access read-only. If `rbac.create=false`, you must provide an equivalent Role and RoleBinding for the operator ServiceAccount: namespaced Secret `create`, plus `get` and `update` restricted to the `sts-tls` resource name. Workloads must trust that CA. To use an externally issued certificate, pre-create `sts-tls` with a certificate signed by a CA already trusted by the workload and set `sts.tls.auto=false`; the chart then omits the namespaced Secret write Role. +The STS service is HTTPS by default. When `sts.tls.auto=true`, the operator creates the fixed `sts-tls` Secret in the operator namespace with `tls.crt`, `tls.key`, and `ca.crt`. Generated certificates are valid for one year and are replaced 30 days before expiry. The operator checks the Secret every five minutes and hot-loads a valid replacement for new TLS connections while retaining the last valid configuration on refresh failures. Existing Operator-managed Secrets created with the legacy long-lived policy are replaced once after upgrade; refresh every STS client's trusted `ca.crt` as part of that upgrade. With `rbac.create=true`, the chart creates a namespaced Role that can create Secrets and update only `sts-tls`; the ClusterRole keeps all Secret and ConfigMap access read-only. If `rbac.create=false`, you must provide an equivalent Role and RoleBinding for the operator ServiceAccount: namespaced Secret `create`, plus `get` and `update` restricted to the `sts-tls` resource name. Workloads must trust that CA. To use an externally issued certificate, pre-create `sts-tls` with a certificate signed by a CA already trusted by the workload and set `sts.tls.auto=false`; update that Secret to rotate it manually, and the operator hot-loads the valid replacement within five minutes. The chart then omits the namespaced Secret write Role. + +Monitor `rustfs_operator_sts_tls_certificate_expiry_timestamp_seconds` and `rustfs_operator_sts_tls_ca_expiry_timestamp_seconds` and alert before either timestamp is reached. STS only issues credentials for TLS-enabled Tenants. For Tenant upstream calls, the operator selects the Tenant HTTPS service endpoint and trusts the CA recorded in `status.certificates.tls.caSecretRef`. diff --git a/deploy/rustfs-operator/values.yaml b/deploy/rustfs-operator/values.yaml index c7d122b..7342951 100755 --- a/deploy/rustfs-operator/values.yaml +++ b/deploy/rustfs-operator/values.yaml @@ -128,9 +128,9 @@ sts: # Operator STS listens with TLS by default and uses this Secret for server certs. enabled: true # When true, the chart grants namespaced create access plus update access to the fixed - # sts-tls Secret. The operator creates a missing Secret and replaces an invalid Secret only - # when it is marked as Operator-managed. If rbac.create=false, provide an equivalent Role and - # RoleBinding in the operator namespace. + # sts-tls Secret. The operator creates a missing Secret and replaces an invalid, legacy, or + # soon-to-expire Secret only when it is marked as Operator-managed. If rbac.create=false, + # provide an equivalent Role and RoleBinding in the operator namespace. auto: true service: type: ClusterIP diff --git a/docs/operator-user-guide.md b/docs/operator-user-guide.md index 8829943..914c1e1 100644 --- a/docs/operator-user-guide.md +++ b/docs/operator-user-guide.md @@ -327,7 +327,8 @@ Notes: - `operator.leaderElect` can be unset. The chart enables leader election automatically when `operator.replicas > 1`. - Keep `console.jwtSecret` stable when running multiple Console replicas. If unset, the chart generates or reuses a Secret. - Keep `CONSOLE_COOKIE_SECURE` enabled for production HTTPS. Only disable it for local HTTP testing. -- `sts.tls.auto=true` lets the operator create or repair `sts-tls`; with `rbac.create=true`, the chart isolates write access in a namespaced Role while keeping cluster-wide Secret and ConfigMap access read-only. With `rbac.create=false`, provide an equivalent Role and RoleBinding for the operator ServiceAccount: namespaced Secret `create`, plus `get` and `update` restricted to the `sts-tls` resource name. +- `sts.tls.auto=true` lets the operator create or repair `sts-tls`. Generated certificates are valid for one year and rotate 30 days before expiry. The operator checks every five minutes and hot-loads valid changes for new connections while retaining the last valid configuration on errors. A legacy Operator-managed Secret is replaced once after upgrade, so refresh clients that trust its `ca.crt`. With `rbac.create=true`, the chart isolates write access in a namespaced Role while keeping cluster-wide Secret and ConfigMap access read-only. With `rbac.create=false`, provide an equivalent Role and RoleBinding for the operator ServiceAccount: namespaced Secret `create`, plus `get` and `update` restricted to the `sts-tls` resource name. +- With `sts.tls.auto=false`, replace `sts-tls` to rotate an externally issued certificate manually; a valid replacement is hot-loaded within five minutes. Monitor `rustfs_operator_sts_tls_certificate_expiry_timestamp_seconds` and `rustfs_operator_sts_tls_ca_expiry_timestamp_seconds`. ## 6. Create a Tenant diff --git a/docs/operator-user-guide.zh-CN.md b/docs/operator-user-guide.zh-CN.md index 36280b8..76ec85b 100644 --- a/docs/operator-user-guide.zh-CN.md +++ b/docs/operator-user-guide.zh-CN.md @@ -304,7 +304,8 @@ sts: - `operator.leaderElect` 可以不配置；当 `operator.replicas > 1` 时 Chart 会自动启用 leader election。 - 多副本 Console 部署需要保持 `console.jwtSecret` 稳定；不设置时 Chart 会生成或复用已有 Secret。 - 生产环境应使用 HTTPS 并保持 `CONSOLE_COOKIE_SECURE` 启用。仅本地 HTTP 调试时才关闭。 -- `sts.tls.auto=true` 时，Operator 会创建或修复 `sts-tls`；当 `rbac.create=true` 时，Chart 通过命名空间级 Role 隔离写权限，全集群 Secret 和 ConfigMap 权限保持只读。当 `rbac.create=false` 时，必须为 Operator ServiceAccount 自行提供等效的 Role 和 RoleBinding：允许在 Operator namespace 内 `create` Secret，并将 `get`、`update` 限定到名为 `sts-tls` 的资源。 +- `sts.tls.auto=true` 时，Operator 会创建或修复 `sts-tls`。自动生成的证书有效期为一年，并在到期前 30 天轮换。Operator 每五分钟检查一次 Secret，对新连接热加载有效变更；检查失败时继续使用最后一个有效配置。升级后，旧策略生成的 Operator 托管 Secret 会被替换一次，因此需要同步刷新客户端信任的 `ca.crt`。当 `rbac.create=true` 时，Chart 通过命名空间级 Role 隔离写权限，全集群 Secret 和 ConfigMap 权限保持只读。当 `rbac.create=false` 时，必须为 Operator ServiceAccount 自行提供等效的 Role 和 RoleBinding：允许在 Operator namespace 内 `create` Secret，并将 `get`、`update` 限定到名为 `sts-tls` 的资源。 +- `sts.tls.auto=false` 时，可通过替换 `sts-tls` 手动轮换外部签发证书；有效的新证书会在五分钟内热加载。请监控 `rustfs_operator_sts_tls_certificate_expiry_timestamp_seconds` 和 `rustfs_operator_sts_tls_ca_expiry_timestamp_seconds`。 ## 6. 创建 Tenant diff --git a/e2e/Cargo.lock b/e2e/Cargo.lock index b338af3..2584d41 100644 --- a/e2e/Cargo.lock +++ b/e2e/Cargo.lock @@ -119,6 +119,45 @@ dependencies = [ "rustversion", ] +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-broadcast" version = "0.7.2" @@ -1140,6 +1179,12 @@ dependencies = [ "syn", ] +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "der" version = "0.7.10" @@ -1151,6 +1196,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.5.8" @@ -2433,6 +2492,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2454,6 +2519,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2463,6 +2538,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.1" @@ -2496,6 +2581,15 @@ dependencies = [ "libc", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2550,6 +2644,7 @@ dependencies = [ "shadow-rs", "snafu", "strum", + "time", "tokio", "tokio-rustls 0.26.4", "tokio-stream", @@ -2561,6 +2656,7 @@ dependencies = [ "url", "utoipa", "utoipa-swagger-ui", + "x509-parser", ] [[package]] @@ -3135,6 +3231,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "1.1.4" @@ -4695,6 +4800,23 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror", + "time", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/src/lib.rs b/src/lib.rs index 914ee51..e5cfdc9 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -151,9 +151,15 @@ pub async fn run(options: ServerOptions) -> Result<(), Box>, + tls_config: Option>>, admission_config: http_admission::AdmissionConfig, ) -> Result<(), Box> { let app = Router::new() @@ -806,13 +812,11 @@ async fn run_sts_server( async fn serve_tls_sts_server( listener: tokio::net::TcpListener, app: Router, - tls_config: Arc, + tls_config: tokio::sync::watch::Receiver>, ) -> Result<(), Box> { - let acceptor = TlsAcceptor::from(tls_config); - loop { let (tcp_stream, remote_addr) = listener.accept().await?; - let acceptor = acceptor.clone(); + let acceptor = TlsAcceptor::from(tls_config.borrow().clone()); let service = app.clone(); tokio::spawn(async move { diff --git a/src/metrics.rs b/src/metrics.rs index 09044c6..1e3c6f2 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -39,6 +39,8 @@ struct Metrics { operator_leader: AtomicU64, sts_requests_total: Mutex>, sts_request_duration: Mutex>, + sts_tls_certificate_expiry_timestamp_seconds: AtomicU64, + sts_tls_ca_expiry_timestamp_seconds: AtomicU64, unauthenticated_admission_rejections_total: Mutex>, unauthenticated_requests_total: Mutex>, http_requests_total: Mutex>, @@ -226,6 +228,15 @@ pub fn record_sts_request(success: bool, duration: Duration) { }); } +pub fn set_sts_tls_expiry_timestamps(certificate: i64, ca: i64) { + metrics() + .sts_tls_certificate_expiry_timestamp_seconds + .store(certificate.max(0) as u64, Ordering::Relaxed); + metrics() + .sts_tls_ca_expiry_timestamp_seconds + .store(ca.max(0) as u64, Ordering::Relaxed); +} + pub(crate) fn record_unauthenticated_request( endpoint: AdmissionEndpoint, outcome: UnauthenticatedRequestOutcome, @@ -456,6 +467,22 @@ pub fn render() -> String { "result", &metrics().sts_request_duration, ); + render_gauge( + &mut output, + "rustfs_operator_sts_tls_certificate_expiry_timestamp_seconds", + "Unix timestamp when the active operator STS TLS server certificate expires.", + metrics() + .sts_tls_certificate_expiry_timestamp_seconds + .load(Ordering::Relaxed) as f64, + ); + render_gauge( + &mut output, + "rustfs_operator_sts_tls_ca_expiry_timestamp_seconds", + "Unix timestamp when the active operator STS TLS CA certificate expires.", + metrics() + .sts_tls_ca_expiry_timestamp_seconds + .load(Ordering::Relaxed) as f64, + ); render_admission_rejection_counter(&mut output); render_unauthenticated_request_counter(&mut output); @@ -849,6 +876,20 @@ mod tests { ); } + #[test] + fn sts_tls_expiry_metrics_are_rendered() { + set_sts_tls_expiry_timestamps(1_800_000_000, 1_800_000_001); + + let rendered = render(); + assert!(rendered.contains( + "rustfs_operator_sts_tls_certificate_expiry_timestamp_seconds 1800000000.000000" + )); + assert!( + rendered + .contains("rustfs_operator_sts_tls_ca_expiry_timestamp_seconds 1800000001.000000") + ); + } + #[test] fn unauthenticated_metric_series_have_zero_baselines() { let rejection_counters = initial_admission_rejection_counters(); diff --git a/src/sts/tls.rs b/src/sts/tls.rs index a77331f..8fbb075 100644 --- a/src/sts/tls.rs +++ b/src/sts/tls.rs @@ -28,8 +28,11 @@ use rcgen::{ }; use rustls::pki_types::CertificateDer; use snafu::{OptionExt, ResultExt, Snafu}; +use time::{Duration as TimeDuration, OffsetDateTime}; +use tokio::sync::watch; use tokio::time::sleep; use tracing::{info, warn}; +use x509_parser::parse_x509_certificate; const STS_TLS_SECRET_NAME: &str = "sts-tls"; const DEFAULT_STS_SERVICE_NAME: &str = "rustfs-operator-sts"; @@ -40,9 +43,15 @@ const TLS_CERT_KEY: &str = "tls.crt"; const TLS_KEY_KEY: &str = "tls.key"; const CA_CERT_KEY: &str = "ca.crt"; const MANAGED_LABEL: &str = "operator.rustfs.com/managed-sts-tls"; +const POLICY_VERSION_ANNOTATION: &str = "operator.rustfs.com/sts-tls-policy-version"; +const POLICY_VERSION: &str = "v1"; const KUBERNETES_TLS_SECRET_TYPE: &str = "kubernetes.io/tls"; const SECRET_WAIT_ATTEMPTS: usize = 30; const SECRET_WAIT_INTERVAL: Duration = Duration::from_secs(2); +const TLS_RELOAD_INTERVAL: Duration = Duration::from_secs(5 * 60); +const CERTIFICATE_VALIDITY: TimeDuration = TimeDuration::days(365); +const CERTIFICATE_RENEWAL_WINDOW: TimeDuration = TimeDuration::days(30); +const CERTIFICATE_CLOCK_SKEW: TimeDuration = TimeDuration::minutes(5); pub type TlsResult = Result; @@ -84,6 +93,9 @@ pub enum Error { #[snafu(display("failed to parse STS TLS certificate: {source}"))] ParseCertificate { source: std::io::Error }, + #[snafu(display("failed to inspect STS TLS {key}: {reason}"))] + InspectCertificate { key: &'static str, reason: String }, + #[snafu(display("STS TLS certificate bundle is empty"))] EmptyCertificateBundle, @@ -180,13 +192,68 @@ pub fn build_tls_server_config( .context(BuildServerConfigSnafu) } +pub async fn reload_sts_tls_config( + client: Client, + config: OperatorStsTlsConfig, + mut active_material: OperatorStsTlsMaterial, + sender: watch::Sender>, +) { + loop { + sleep(TLS_RELOAD_INTERVAL).await; + if sender.is_closed() { + return; + } + + match load_or_create_sts_tls_material(&client, &config).await { + Ok(material) if material == active_material => {} + Ok(material) => match build_tls_server_config(&material) { + Ok(server_config) => { + if sender.send(std::sync::Arc::new(server_config)).is_err() { + return; + } + active_material = material; + info!( + secret = STS_TLS_SECRET_NAME, + namespace = %config.namespace, + "reloaded operator STS TLS certificate" + ); + } + Err(error) => warn!( + secret = STS_TLS_SECRET_NAME, + namespace = %config.namespace, + %error, + "keeping last valid operator STS TLS configuration" + ), + }, + Err(error) => warn!( + secret = STS_TLS_SECRET_NAME, + namespace = %config.namespace, + %error, + "failed to refresh operator STS TLS Secret; keeping last valid configuration" + ), + } + } +} + async fn load_material_from_secret_or_regenerate( api: &Api, config: &OperatorStsTlsConfig, secret: corev1::Secret, ) -> TlsResult { - match material_from_secret(config, &secret) { - Ok(material) => Ok(material), + match validated_material_from_secret(config, &secret) { + Ok(material) => { + if should_rotate_managed_secret(config, &secret, &material, OffsetDateTime::now_utc())? + { + info!( + secret = STS_TLS_SECRET_NAME, + namespace = %config.namespace, + "rotating managed operator STS TLS certificate" + ); + replace_generated_secret(api, config, &secret).await + } else { + Ok(material) + } + } Err(error) if config.auto_generate && is_operator_managed(&secret) => { warn!( secret = STS_TLS_SECRET_NAME, @@ -211,7 +278,7 @@ async fn create_or_get_generated_secret( namespace = %config.namespace, "created operator STS TLS Secret" ); - material_from_secret(config, &secret) + validated_material_from_secret(config, &secret) } Err(kube::Error::Api(error)) if error.code == 409 => { wait_for_secret_material(api, config).await @@ -236,7 +303,28 @@ async fn replace_generated_secret( .replace(STS_TLS_SECRET_NAME, &PostParams::default(), &generated) .await { - Ok(secret) => material_from_secret(config, &secret), + Ok(secret) => validated_material_from_secret(config, &secret), + Err(source) if matches!(&source, kube::Error::Api(error) if error.code == 409) => { + let latest = api + .get(STS_TLS_SECRET_NAME) + .await + .map_err(|get_error| Error::Kube { + source: Box::new(get_error), + action: "load after replace conflict", + namespace: config.namespace.clone(), + secret: STS_TLS_SECRET_NAME.to_string(), + })?; + let material = validated_material_from_secret(config, &latest)?; + if managed_secret_needs_rotation(&latest, &material, OffsetDateTime::now_utc())? { + return Err(Error::Kube { + source: Box::new(source), + action: "replace managed", + namespace: config.namespace.clone(), + secret: STS_TLS_SECRET_NAME.to_string(), + }); + } + Ok(material) + } Err(source) => Err(Error::Kube { source: Box::new(source), action: "replace managed", @@ -252,7 +340,7 @@ async fn wait_for_secret_material( ) -> TlsResult { for _ in 0..SECRET_WAIT_ATTEMPTS { match api.get(STS_TLS_SECRET_NAME).await { - Ok(secret) => return material_from_secret(config, &secret), + Ok(secret) => return validated_material_from_secret(config, &secret), Err(kube::Error::Api(error)) if error.code == 404 => { sleep(SECRET_WAIT_INTERVAL).await; } @@ -295,12 +383,17 @@ fn generated_sts_tls_secret(config: &OperatorStsTlsConfig) -> TlsResult TlsResult { + generate_sts_tls_material_at( + namespace, + service_name, + cluster_domain, + OffsetDateTime::now_utc(), + ) +} + +fn generate_sts_tls_material_at( + namespace: &str, + service_name: &str, + cluster_domain: &str, + now: OffsetDateTime, ) -> TlsResult { let ca_key = KeyPair::generate().context(GenerateCertificateSnafu)?; let mut ca_params = CertificateParams::default(); + ca_params.not_before = now - CERTIFICATE_CLOCK_SKEW; + ca_params.not_after = now + CERTIFICATE_VALIDITY; ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); ca_params.key_usages = vec![ KeyUsagePurpose::KeyCertSign, @@ -332,6 +441,8 @@ fn generate_sts_tls_material( server_names.push(Ipv4Addr::LOCALHOST.to_string()); let mut server_params = CertificateParams::new(server_names).context(GenerateCertificateSnafu)?; + server_params.not_before = now - CERTIFICATE_CLOCK_SKEW; + server_params.not_after = now + CERTIFICATE_VALIDITY; server_params.is_ca = IsCa::NoCa; server_params.key_usages = vec![ KeyUsagePurpose::DigitalSignature, @@ -378,6 +489,106 @@ fn material_from_secret( }) } +fn validated_material_from_secret( + config: &OperatorStsTlsConfig, + secret: &corev1::Secret, +) -> TlsResult { + let material = material_from_secret(config, secret)?; + build_tls_server_config(&material)?; + validate_material_at(&material, OffsetDateTime::now_utc())?; + record_expiry_metrics(&material)?; + Ok(material) +} + +fn validate_material_at(material: &OperatorStsTlsMaterial, now: OffsetDateTime) -> TlsResult<()> { + let now = now.unix_timestamp(); + for (pem, key) in [ + (material.cert_pem.as_slice(), TLS_CERT_KEY), + (material.ca_pem.as_slice(), CA_CERT_KEY), + ] { + let (not_before, not_after) = certificate_validity_timestamps(pem, key)?; + if now < not_before || now > not_after { + return Err(Error::InspectCertificate { + key, + reason: format!( + "certificate is not valid at timestamp {now} (valid from {not_before} to {not_after})" + ), + }); + } + } + Ok(()) +} + +fn managed_secret_needs_rotation( + secret: &corev1::Secret, + material: &OperatorStsTlsMaterial, + now: OffsetDateTime, +) -> TlsResult { + let policy_is_current = secret + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(POLICY_VERSION_ANNOTATION)) + .is_some_and(|version| version == POLICY_VERSION); + if !policy_is_current { + return Ok(true); + } + + let (certificate_expiry, ca_expiry) = material_expiry_timestamps(material)?; + let renewal_deadline = (now + CERTIFICATE_RENEWAL_WINDOW).unix_timestamp(); + Ok(certificate_expiry <= renewal_deadline || ca_expiry <= renewal_deadline) +} + +fn should_rotate_managed_secret( + config: &OperatorStsTlsConfig, + secret: &corev1::Secret, + material: &OperatorStsTlsMaterial, + now: OffsetDateTime, +) -> TlsResult { + if !config.auto_generate || !is_operator_managed(secret) { + return Ok(false); + } + managed_secret_needs_rotation(secret, material, now) +} + +fn record_expiry_metrics(material: &OperatorStsTlsMaterial) -> TlsResult<()> { + let (certificate_expiry, ca_expiry) = material_expiry_timestamps(material)?; + crate::metrics::set_sts_tls_expiry_timestamps(certificate_expiry, ca_expiry); + Ok(()) +} + +fn material_expiry_timestamps(material: &OperatorStsTlsMaterial) -> TlsResult<(i64, i64)> { + Ok(( + certificate_expiry_timestamp(&material.cert_pem, TLS_CERT_KEY)?, + certificate_expiry_timestamp(&material.ca_pem, CA_CERT_KEY)?, + )) +} + +fn certificate_expiry_timestamp(pem: &[u8], key: &'static str) -> TlsResult { + certificate_validity_timestamps(pem, key).map(|(_, not_after)| not_after) +} + +fn certificate_validity_timestamps(pem: &[u8], key: &'static str) -> TlsResult<(i64, i64)> { + let certificate = rustls_pemfile::certs(&mut Cursor::new(pem)) + .next() + .transpose() + .context(ParseCertificateSnafu)? + .context(InspectCertificateSnafu { + key, + reason: "certificate bundle is empty".to_string(), + })?; + let (_, certificate) = parse_x509_certificate(certificate.as_ref()).map_err(|source| { + Error::InspectCertificate { + key, + reason: source.to_string(), + } + })?; + Ok(( + certificate.validity().not_before.timestamp(), + certificate.validity().not_after.timestamp(), + )) +} + fn secret_data( data: &BTreeMap, key: &'static str, @@ -490,15 +701,80 @@ mod tests { build_tls_server_config(&material).unwrap(); } + #[test] + fn generated_certificates_are_valid_for_one_year() { + let now = OffsetDateTime::from_unix_timestamp(1_800_000_000).unwrap(); + let material = generate_sts_tls_material_at( + "rustfs-system", + "rustfs-operator-sts", + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + now, + ) + .unwrap(); + + let (certificate_expiry, ca_expiry) = material_expiry_timestamps(&material).unwrap(); + let expected_expiry = (now + CERTIFICATE_VALIDITY).unix_timestamp(); + assert_eq!(certificate_expiry, expected_expiry); + assert_eq!(ca_expiry, expected_expiry); + } + + #[test] + fn expired_certificate_material_is_rejected() { + let now = OffsetDateTime::from_unix_timestamp(1_800_000_000).unwrap(); + let material = generate_sts_tls_material_at( + "rustfs-system", + "rustfs-operator-sts", + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + now - TimeDuration::days(366), + ) + .unwrap(); + + assert!(validate_material_at(&material, now).is_err()); + } + + #[test] + fn managed_certificate_rotation_migrates_legacy_policy_and_renews_early() { + let config = test_config(); + let now = OffsetDateTime::from_unix_timestamp(1_800_000_000).unwrap(); + let current_material = generate_sts_tls_material_at( + &config.namespace, + &config.service_name, + &config.cluster_domain, + now, + ) + .unwrap(); + let mut secret = generated_sts_tls_secret(&config).unwrap(); + set_secret_material(&mut secret, ¤t_material); + + assert!(!managed_secret_needs_rotation(&secret, ¤t_material, now).unwrap()); + + secret.metadata.annotations = None; + assert!(managed_secret_needs_rotation(&secret, ¤t_material, now).unwrap()); + let mut external_config = config.clone(); + external_config.auto_generate = false; + assert!( + !should_rotate_managed_secret(&external_config, &secret, ¤t_material, now) + .unwrap() + ); + + secret.metadata.annotations = Some(BTreeMap::from([( + POLICY_VERSION_ANNOTATION.to_string(), + POLICY_VERSION.to_string(), + )])); + let expiring_material = generate_sts_tls_material_at( + &config.namespace, + &config.service_name, + &config.cluster_domain, + now - TimeDuration::days(336), + ) + .unwrap(); + set_secret_material(&mut secret, &expiring_material); + assert!(managed_secret_needs_rotation(&secret, &expiring_material, now).unwrap()); + } + #[test] fn secret_material_uses_leaf_as_ca_fallback() { - let config = OperatorStsTlsConfig { - enabled: true, - auto_generate: true, - namespace: "rustfs-system".to_string(), - service_name: "rustfs-operator-sts".to_string(), - cluster_domain: cluster_dns::DEFAULT_CLUSTER_DOMAIN.to_string(), - }; + let config = test_config(); let generated = generate_sts_tls_material( &config.namespace, &config.service_name, @@ -521,4 +797,28 @@ mod tests { let material = material_from_secret(&config, &secret).unwrap(); assert_eq!(material.ca_pem, material.cert_pem); } + + fn test_config() -> OperatorStsTlsConfig { + OperatorStsTlsConfig { + enabled: true, + auto_generate: true, + namespace: "rustfs-system".to_string(), + service_name: "rustfs-operator-sts".to_string(), + cluster_domain: cluster_dns::DEFAULT_CLUSTER_DOMAIN.to_string(), + } + } + + fn set_secret_material(secret: &mut corev1::Secret, material: &OperatorStsTlsMaterial) { + secret.data = Some(BTreeMap::from([ + ( + TLS_CERT_KEY.to_string(), + ByteString(material.cert_pem.clone()), + ), + ( + TLS_KEY_KEY.to_string(), + ByteString(material.key_pem.clone()), + ), + (CA_CERT_KEY.to_string(), ByteString(material.ca_pem.clone())), + ])); + } }