diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 7087fd08a..2c9710d94 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -129,7 +129,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -140,7 +140,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1692,7 +1692,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2576,7 +2576,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3487,7 +3487,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.4", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -3817,7 +3817,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4676,7 +4676,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4933,7 +4933,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -5609,7 +5609,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.4", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -5648,7 +5648,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.4", + "socket2 0.5.10", "tracing", "windows-sys 0.60.2", ] @@ -6330,7 +6330,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -6343,7 +6343,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6411,7 +6411,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -7267,7 +7267,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -7584,7 +7584,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -8420,9 +8420,8 @@ dependencies = [ [[package]] name = "wavekv" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9b73bc556dfdb7ef33617a9d477b803198db43ea3df25463efaf43d4986fe8" +version = "2.0.0" +source = "git+https://github.com/Phala-Network/wavekv?branch=feat%2Fdelta-state-sync#a95014ae79c4f95f1e869125c777fe253b12a79d" dependencies = [ "anyhow", "bincode 2.0.1", @@ -8522,7 +8521,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -8695,15 +8694,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.60.2" diff --git a/dstack/Cargo.toml b/dstack/Cargo.toml index a702671e1..20b903f09 100644 --- a/dstack/Cargo.toml +++ b/dstack/Cargo.toml @@ -119,7 +119,8 @@ serde-duration = { path = "serde-duration" } dstack-mr = { path = "dstack-mr" } dstack-verifier = { path = "verifier", default-features = false } size-parser = { path = "size-parser" } -wavekv = "1.0.0" +# TODO: repoint to `wavekv = "2.0"` once Phala-Network/wavekv#3 is released to crates.io. +wavekv = { git = "https://github.com/Phala-Network/wavekv", branch = "feat/delta-state-sync" } # Core dependencies anyhow = { version = "1.0.97", default-features = false } diff --git a/dstack/gateway/rpc/proto/gateway_rpc.proto b/dstack/gateway/rpc/proto/gateway_rpc.proto index 68202032d..ab3262df6 100644 --- a/dstack/gateway/rpc/proto/gateway_rpc.proto +++ b/dstack/gateway/rpc/proto/gateway_rpc.proto @@ -339,9 +339,22 @@ message PeerSyncStatus { uint32 id = 1; uint64 local_ack = 2; uint64 peer_ack = 3; - uint64 buffered_logs = 4; + // Always 0 since wavekv 2.0, which replicates state instead of operation logs and + // keeps no per-peer log buffers. Retained so existing clients keep decoding. + uint64 buffered_logs = 4 [deprecated = true]; // Last seen timestamps: [(observer_node_id, timestamp), ...] repeated LastSeenEntry last_seen = 5; + // Whether this peer has ever reported an ack map. + bool heard_from = 6; + // Sync protocol last negotiated with this peer: "v1" or "v2". + string protocol = 7; + // Consecutive quiescent rounds whose state digests disagreed. Non-zero means the + // replicas have silently diverged; wavekv 1.x could not detect this at all. + uint32 digest_mismatches = 8; + // Consecutive sync rounds that failed outright. Only a definitive 404/405 demotes a + // peer to "v1"; a 5xx or a timeout leaves `protocol` untouched by design, so this is + // the only field that moves when a peer is failing every round. + uint32 consecutive_failures = 9; } message LastSeenEntry { @@ -358,6 +371,13 @@ message StoreSyncStatus { bool dirty = 5; bool wal_enabled = 6; repeated PeerSyncStatus peers = 7; + // Hex SHA-256 over the replicated state. Two converged replicas produce equal + // digests by construction, so comparing this across the cluster is the promotion + // gate for the wavekv v2 rollout and the standing divergence check afterwards. + string digest = 8; + uint64 entries_merged = 9; + // Entries refused by the admission policy or the ingest quotas. + uint64 entries_rejected = 10; } // WaveKV sync status response diff --git a/dstack/gateway/src/admin_service.rs b/dstack/gateway/src/admin_service.rs index c1e988d19..078113e9b 100644 --- a/dstack/gateway/src/admin_service.rs +++ b/dstack/gateway/src/admin_service.rs @@ -208,16 +208,33 @@ impl AdminRpc for AdminRpcHandler { .collect() }; + // Per-peer protocol/digest telemetry lives on the sync manager, not the store. + let links = self + .state + .wavekv_sync + .as_ref() + .map(|s| s.link_status()) + .unwrap_or_default(); + let links_for = |name: &str| -> Vec { + links + .iter() + .find(|(store, _)| *store == name) + .map(|(_, l)| l.clone()) + .unwrap_or_default() + }; + Ok(WaveKvStatusResponse { enabled: self.state.config.sync.enabled, persistent: Some(build_store_status( "persistent", persistent_status, + &links_for("persistent"), &get_peer_last_seen, )), ephemeral: Some(build_store_status( "ephemeral", ephemeral_status, + &links_for("ephemeral"), &get_peer_last_seen, )), }) @@ -718,6 +735,7 @@ fn port_policy_view_to_proto(view: PortPolicyView) -> GetInstancePortPolicyRespo fn build_store_status( name: &str, status: WaveKvNodeStatus, + links: &[wavekv::sync::PeerLinkStatus], get_peer_last_seen: &impl Fn(u32) -> Vec<(u32, u64)>, ) -> StoreSyncStatus { StoreSyncStatus { @@ -727,6 +745,9 @@ fn build_store_status( next_seq: status.next_seq, dirty: status.dirty, wal_enabled: status.wal, + digest: status.digest, + entries_merged: status.entries_merged, + entries_rejected: status.entries_rejected, peers: status .peers .into_iter() @@ -735,12 +756,19 @@ fn build_store_status( .into_iter() .map(|(node_id, timestamp)| LastSeenEntry { node_id, timestamp }) .collect(); + let link = links.iter().find(|l| l.id == p.id); + #[allow(deprecated)] ProtoPeerSyncStatus { id: p.id, local_ack: p.ack, - peer_ack: p.pack, - buffered_logs: p.logs as u64, + peer_ack: p.peer_ack, + // wavekv 2.0 keeps no per-peer log buffers. + buffered_logs: 0, last_seen, + heard_from: p.heard_from, + protocol: link.map(|l| l.protocol).unwrap_or_default().to_string(), + digest_mismatches: link.map(|l| l.digest_mismatches).unwrap_or(0), + consecutive_failures: link.map(|l| l.consecutive_failures).unwrap_or(0), } }) .collect(), diff --git a/dstack/gateway/src/kv/https_client.rs b/dstack/gateway/src/kv/https_client.rs index d0d034a9b..bf0e15afc 100644 --- a/dstack/gateway/src/kv/https_client.rs +++ b/dstack/gateway/src/kv/https_client.rs @@ -5,12 +5,12 @@ //! HTTPS client with mTLS and custom certificate verification during TLS handshake. use std::fmt::Debug; -use std::io::{Read, Write}; +use std::io::Write; use std::sync::Arc; use anyhow::{Context, Result}; -use flate2::{read::GzDecoder, write::GzEncoder, Compression}; -use http_body_util::{BodyExt, Full}; +use flate2::{write::GzEncoder, Compression}; +use http_body_util::{BodyExt, Full, Limited}; use hyper::body::Bytes; use hyper_rustls::HttpsConnectorBuilder; use hyper_util::{ @@ -25,6 +25,24 @@ use serde::{de::DeserializeOwned, Serialize}; use super::{decode, encode}; +/// Read a peer's response body, refusing one larger than the routes accept on a request. +/// +/// `Body::collect` reads to completion, so without this a peer could stream an unbounded +/// response and the decompression limit downstream would never be reached — the memory +/// is already gone by then. +async fn read_body_bounded(body: hyper::body::Incoming) -> Result { + Limited::new(body, super::MAX_COMPRESSED_SYNC_BYTES) + .collect() + .await + .map(|collected| collected.to_bytes()) + .map_err(|err| { + anyhow::anyhow!( + "failed to read response body (limit {} bytes): {err}", + super::MAX_COMPRESSED_SYNC_BYTES + ) + }) +} + /// Custom certificate validator trait for TLS handshake verification. /// /// Implementations can perform additional validation on the peer certificate @@ -218,16 +236,55 @@ impl HttpsClient { anyhow::bail!("request failed: {}", response.status()); } - let body = response - .into_body() - .collect() - .await - .context("failed to read response body")? - .to_bytes(); + // Bounded like every other response: this is the bootnode GetPeers path, and + // the threat model does not assume a bootnode is honest. + let body = read_body_bounded(response.into_body()).await?; serde_json::from_slice(&body).context("failed to parse response") } + /// Send an already-encoded body and return the raw response bytes, or `None` when + /// the peer does not expose the route. + /// + /// `None` (rather than an error) is what lets the caller distinguish "this peer has + /// not been upgraded yet" from "the request failed", which is the basis of the + /// wavekv v1/v2 protocol negotiation. + pub async fn post_bytes_probe(&self, url: &str, body: Vec) -> Result>> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::fast()); + encoder + .write_all(&body) + .context("failed to compress request")?; + let compressed = encoder.finish().context("failed to finish compression")?; + + let request = hyper::Request::builder() + .method(hyper::Method::POST) + .uri(url) + .header("content-type", "application/x-msgpack-gz") + .body(Full::new(Bytes::from(compressed))) + .context("failed to build request")?; + + let response = self + .client + .request(request) + .await + .with_context(|| format!("failed to send request to {url}"))?; + + let status = response.status(); + if status == hyper::StatusCode::NOT_FOUND || status == hyper::StatusCode::METHOD_NOT_ALLOWED + { + return Ok(None); + } + if !status.is_success() { + anyhow::bail!("request failed: {status}"); + } + + let body = read_body_bounded(response.into_body()).await?; + Ok(Some(crate::kv::gunzip_bounded( + &body, + crate::kv::MAX_DECOMPRESSED_SYNC_BYTES, + )?)) + } + /// Send a POST request with msgpack + gzip encoded body and receive msgpack + gzip response pub async fn post_compressed_msg( &self, @@ -260,20 +317,9 @@ impl HttpsClient { anyhow::bail!("request failed: {}", response.status()); } - let body = response - .into_body() - .collect() - .await - .context("failed to read response body")? - .to_bytes(); - - // Decompress - let mut decoder = GzDecoder::new(body.as_ref()); - let mut decompressed = Vec::new(); - decoder - .read_to_end(&mut decompressed) - .context("failed to decompress response")?; - + let body = read_body_bounded(response.into_body()).await?; + let decompressed = + crate::kv::gunzip_bounded(&body, crate::kv::MAX_DECOMPRESSED_SYNC_BYTES)?; decode(&decompressed).context("failed to decode response") } } @@ -320,3 +366,365 @@ impl CertValidator for AppIdValidator { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use ra_tls::cert::CertRequest; + use ra_tls::rcgen::KeyPair; + + /// A certificate carrying `PHALA_RATLS_APP_ID`, minted in process. + /// + /// No TEE is involved: `CertRequest` writes the extension unconditionally, and the + /// validator below never looks at a quote — it parses DER and compares bytes. + fn cert_with_app_id(app_id: &[u8]) -> Vec { + let key = KeyPair::generate().expect("key"); + CertRequest::builder() + .key(&key) + .subject("peer.test") + .app_id(app_id) + .build() + .self_signed() + .expect("self-signed cert") + .der() + .to_vec() + } + + fn cert_without_app_id() -> Vec { + let key = KeyPair::generate().expect("key"); + CertRequest::builder() + .key(&key) + .subject("peer.test") + .build() + .self_signed() + .expect("self-signed cert") + .der() + .to_vec() + } + + /// The client half of the same rule the sync routes enforce on inbound requests. + /// + /// This runs during the TLS handshake, so a validator that always returns `Ok(())` + /// means this gateway will complete a mutually-authenticated connection to any peer + /// presenting any certificate our CA signed — and then send it our state. Replacing + /// the whole body with `Ok(())`, or inverting the comparison, left the suite green. + #[test] + fn a_peer_certificate_is_accepted_only_when_its_app_id_matches() { + let ours = b"app-id-of-this-cluster".to_vec(); + let validator = AppIdValidator::new(ours.clone()); + + assert_eq!(validator.validate(&cert_with_app_id(&ours)), Ok(())); + assert!( + validator + .validate(&cert_with_app_id(b"a-different-app")) + .is_err(), + "a certificate from another app must not complete the handshake" + ); + } + + /// A certificate that says nothing about which app holds it proves nothing, and must + /// be refused rather than treated as unconstrained. + #[test] + fn a_peer_certificate_without_an_app_id_is_refused() { + let validator = AppIdValidator::new(b"app-id-of-this-cluster".to_vec()); + let err = validator + .validate(&cert_without_app_id()) + .expect_err("a certificate with no app identity must be refused"); + assert!(err.contains("app_id"), "{err}"); + } + + /// Anything that is not a certificate is a parse failure, not a pass. + #[test] + fn a_malformed_certificate_is_refused() { + let validator = AppIdValidator::new(b"whatever".to_vec()); + assert!(validator.validate(b"not a certificate at all").is_err()); + } +} + +/// Negotiation and response-bounding tested against a real TLS peer. +/// +/// `post_bytes_probe`'s mapping of 404/405 to `Ok(None)` *is* the v1/v2 protocol +/// negotiation: a gateway that has not been upgraded has no `/wavekv/sync2` route, and +/// that status is the only signal its peers get. Every mutation of that condition +/// survived, because nothing exercised the function at all — it needs a peer that speaks +/// TLS, and `https_only()` means a plain HTTP stub will not do. +/// +/// No container and no TEE: a local listener with a certificate minted in process. +#[cfg(test)] +mod transport_tests { + use super::*; + use hyper::service::service_fn; + use hyper::{Response, StatusCode}; + use hyper_util::rt::TokioIo; + use std::convert::Infallible; + use tokio::net::TcpListener; + use tokio_rustls::TlsAcceptor; + + /// A CA plus a leaf valid for 127.0.0.1, written where `HttpsClient::new` expects. + fn tls_material(dir: &std::path::Path) -> (HttpsClientConfig, Vec, Vec) { + use ra_tls::rcgen::{BasicConstraints, CertificateParams, IsCa, KeyPair}; + + let ca_key = KeyPair::generate().expect("ca key"); + let mut ca_params = CertificateParams::new(vec![]).expect("ca params"); + ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + let ca_cert = ca_params.self_signed(&ca_key).expect("ca cert"); + + let leaf_key = KeyPair::generate().expect("leaf key"); + let leaf_params = + CertificateParams::new(vec!["127.0.0.1".to_string()]).expect("leaf params"); + let leaf_cert = leaf_params + .signed_by(&leaf_key, &ca_cert, &ca_key) + .expect("leaf cert"); + + let cert_path = dir.join("node.crt"); + let key_path = dir.join("node.key"); + let ca_path = dir.join("ca.crt"); + std::fs::write(&cert_path, leaf_cert.pem()).expect("write cert"); + std::fs::write(&key_path, leaf_key.serialize_pem()).expect("write key"); + std::fs::write(&ca_path, ca_cert.pem()).expect("write ca"); + + ( + HttpsClientConfig { + cert_path: cert_path.to_string_lossy().into_owned(), + key_path: key_path.to_string_lossy().into_owned(), + ca_cert_path: ca_path.to_string_lossy().into_owned(), + cert_validator: None, + }, + leaf_cert.der().to_vec(), + leaf_key.serialize_der(), + ) + } + + /// Serve one fixed response over TLS and return the URL to reach it. + async fn serve(status: StatusCode, body: Vec, cert: Vec, key: Vec) -> String { + let certs = vec![rustls::pki_types::CertificateDer::from(cert)]; + let key = rustls::pki_types::PrivateKeyDer::try_from(key).expect("server key"); + let config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certs, key) + .expect("server config"); + let acceptor = TlsAcceptor::from(Arc::new(config)); + + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("addr"); + + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + let acceptor = acceptor.clone(); + let body = body.clone(); + tokio::spawn(async move { + let Ok(tls) = acceptor.accept(stream).await else { + return; + }; + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection( + TokioIo::new(tls), + service_fn(move |_req| { + let body = body.clone(); + async move { + Ok::<_, Infallible>( + Response::builder() + .status(status) + .body(Full::new(Bytes::from(body))) + .expect("response"), + ) + } + }), + ) + .await; + }); + } + }); + + format!("https://127.0.0.1:{}/wavekv/sync2/persistent", addr.port()) + } + + fn gzip(bytes: &[u8]) -> Vec { + let mut encoder = GzEncoder::new(Vec::new(), Compression::fast()); + encoder.write_all(bytes).expect("gzip"); + encoder.finish().expect("gzip finish") + } + + async fn probe(status: StatusCode, body: Vec) -> Result>> { + let _ = rustls::crypto::ring::default_provider().install_default(); + let dir = tempfile::tempdir().expect("tempdir"); + let (config, cert, key) = tls_material(dir.path()); + let url = serve(status, body, cert, key).await; + HttpsClient::new(&config) + .expect("client") + .post_bytes_probe(&url, b"request".to_vec()) + .await + } + + /// A peer still on v1 has no `/wavekv/sync2` route. Both statuses a router can give + /// for that must read as "not upgraded", not as a failure — a failure would be + /// retried forever instead of falling back to the v1 route. + #[tokio::test] + async fn a_missing_route_reads_as_not_upgraded() { + assert_eq!( + probe(StatusCode::NOT_FOUND, Vec::new()).await.unwrap(), + None + ); + assert_eq!( + probe(StatusCode::METHOD_NOT_ALLOWED, Vec::new()) + .await + .unwrap(), + None + ); + } + + /// ...and any other failure must stay a failure. Reading a 500 as "not upgraded" + /// would demote a healthy v2 peer to the v1 path for a whole reprobe window. + #[tokio::test] + async fn a_server_error_is_not_mistaken_for_a_missing_route() { + assert!(probe(StatusCode::INTERNAL_SERVER_ERROR, Vec::new()) + .await + .is_err()); + assert!(probe(StatusCode::BAD_REQUEST, Vec::new()).await.is_err()); + } + + /// A peer that answers gets its body decompressed and returned. + #[tokio::test] + async fn an_upgraded_peer_returns_its_decoded_body() { + let payload = b"the-envelope-bytes".to_vec(); + let got = probe(StatusCode::OK, gzip(&payload)).await.unwrap(); + assert_eq!(got, Some(payload)); + } + + /// A server certificate carrying an app id, signed by the same test CA. + fn app_id_server_cert( + dir: &std::path::Path, + app_id: &[u8], + ) -> (HttpsClientConfig, Vec, Vec) { + use ra_tls::cert::CertRequest; + use ra_tls::rcgen::{BasicConstraints, CertificateParams, IsCa, KeyPair}; + + let ca_key = KeyPair::generate().expect("ca key"); + let mut ca_params = CertificateParams::new(vec![]).expect("ca params"); + ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + let ca_cert = ca_params.self_signed(&ca_key).expect("ca cert"); + + let leaf_key = KeyPair::generate().expect("leaf key"); + let alt_names = vec!["127.0.0.1".to_string()]; + let leaf_cert = CertRequest::builder() + .key(&leaf_key) + .subject("peer.test") + .alt_names(&alt_names) + .app_id(app_id) + .usage_server_auth(true) + .build() + .signed_by(&ca_cert, &ca_key) + .expect("leaf cert"); + + let cert_path = dir.join("node.crt"); + let key_path = dir.join("node.key"); + let ca_path = dir.join("ca.crt"); + std::fs::write(&cert_path, leaf_cert.pem()).expect("write cert"); + std::fs::write(&key_path, leaf_key.serialize_pem()).expect("write key"); + std::fs::write(&ca_path, ca_cert.pem()).expect("write ca"); + + ( + HttpsClientConfig { + cert_path: cert_path.to_string_lossy().into_owned(), + key_path: key_path.to_string_lossy().into_owned(), + ca_cert_path: ca_path.to_string_lossy().into_owned(), + cert_validator: None, + }, + leaf_cert.der().to_vec(), + leaf_key.serialize_der(), + ) + } + + /// The client-side identity check, over a real handshake rather than a direct call. + /// + /// `AppIdValidator` runs inside `CustomCertVerifier`, which rustls only reaches once + /// standard chain verification passes — so unit-testing the validator alone leaves + /// the wiring untested. A peer from another app must fail to connect at all, before + /// any application bytes move. + #[tokio::test] + async fn a_peer_from_another_app_cannot_complete_the_handshake() { + let _ = rustls::crypto::ring::default_provider().install_default(); + let ours = b"app-id-of-this-cluster".to_vec(); + + for (server_app_id, expect_ok) in + [(ours.clone(), true), (b"a-different-app".to_vec(), false)] + { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut config, cert, key) = app_id_server_cert(dir.path(), &server_app_id); + config.cert_validator = Some(Arc::new(AppIdValidator::new(ours.clone()))); + let url = serve(StatusCode::NOT_FOUND, Vec::new(), cert, key).await; + + let got = HttpsClient::new(&config) + .expect("client") + .post_bytes_probe(&url, b"x".to_vec()) + .await; + + if expect_ok { + assert_eq!( + got.expect("a peer from our own app must connect"), + None, + "the 404 should still read as not-upgraded" + ); + } else { + assert!( + got.is_err(), + "a peer from another app completed the handshake" + ); + } + } + } + + /// `post_compressed_msg` is the v1 sync path — how a v2 gateway talks to one that + /// has not been upgraded. Its status check was as untested as the negotiation's, so + /// a v1 peer answering 500 could have been decoded as a successful round. + #[tokio::test] + async fn a_failed_v1_sync_is_not_decoded_as_a_response() { + let _ = rustls::crypto::ring::default_provider().install_default(); + let dir = tempfile::tempdir().expect("tempdir"); + let (config, cert, key) = tls_material(dir.path()); + let url = serve(StatusCode::INTERNAL_SERVER_ERROR, Vec::new(), cert, key).await; + let client = HttpsClient::new(&config).expect("client"); + let out: Result = client.post_compressed_msg(&url, &1u32).await; + assert!(out.is_err(), "a 500 from a v1 peer must not decode"); + } + + /// `post_json` is the bootnode GetPeers path, and the threat model does not assume a + /// bootnode is honest — so a failure status must not be parsed as a peer list. + #[tokio::test] + async fn a_failed_bootnode_fetch_is_not_parsed_as_peers() { + let _ = rustls::crypto::ring::default_provider().install_default(); + let dir = tempfile::tempdir().expect("tempdir"); + let (config, cert, key) = tls_material(dir.path()); + let url = serve(StatusCode::FORBIDDEN, b"null".to_vec(), cert, key).await; + let client = HttpsClient::new(&config).expect("client"); + let out: Result> = client.post_json(&url, &()).await; + assert!( + out.is_err(), + "a 403 from a bootnode must not parse as a body" + ); + } + + /// The response body is bounded before it is decompressed, so a peer cannot spend + /// our memory ahead of any decoding limit. + /// + /// The body must be *valid* gzip that merely exceeds the compressed ceiling. A + /// malformed one is rejected by `gunzip_bounded` whatever the ceiling says, so it + /// would pass this test with the bound removed entirely — which is exactly what the + /// first version of it did. Stored-mode gzip keeps the encoded size at roughly the + /// input size, so the payload clears the ceiling while decompressing well inside it. + #[tokio::test] + async fn an_oversized_response_body_is_refused() { + let stored = { + let mut encoder = GzEncoder::new(Vec::new(), Compression::none()); + encoder + .write_all(&vec![0u8; super::super::MAX_COMPRESSED_SYNC_BYTES + 1]) + .expect("gzip"); + encoder.finish().expect("gzip finish") + }; + assert!( + stored.len() > super::super::MAX_COMPRESSED_SYNC_BYTES, + "the fixture depends on the compressed body clearing the ceiling" + ); + assert!(probe(StatusCode::OK, stored).await.is_err()); + } +} diff --git a/dstack/gateway/src/kv/mod.rs b/dstack/gateway/src/kv/mod.rs index 09392798c..2a8b4e341 100644 --- a/dstack/gateway/src/kv/mod.rs +++ b/dstack/gateway/src/kv/mod.rs @@ -29,6 +29,7 @@ //! - `last_seen/node/{node_id}/{seen_by_node_id}` → u64 (timestamp) mod https_client; +mod schema; mod sync_service; pub use https_client::{AppIdValidator, HttpsClientConfig}; @@ -374,6 +375,56 @@ pub mod keys { /// in `#[serde(default)]` fields it does not receive, so the value types below /// can gain fields without breaking gateways running an older build. Decoding /// accepts both forms, so values written by older releases stay readable. +/// wavekv configuration shared by both stores. +/// +/// The admission policy is the important part: it confines a peer to the key shapes +/// this gateway actually defines, so a compromised or buggy node in the cluster cannot +/// plant arbitrary keys that every other node would then replicate and persist forever. +fn store_config(store: schema::Store) -> wavekv::NodeConfig { + wavekv::NodeConfig { + admission: Some(std::sync::Arc::new(schema::GatewaySchema::new(store))), + ..Default::default() + } +} + +/// Ceiling on a decompressed sync payload. +/// +/// The wire is gzipped, and gzip expands by three orders of magnitude on +/// attacker-chosen input: the 16 MiB cap on a request body is a cap on the *compressed* +/// size, which bounds nothing useful on its own. Every gateway in the cluster shares one +/// app_id, so mTLS proves only that a peer is *some* gateway of this deployment — the +/// same reason the key schema exists (see `schema.rs`). +/// +/// The value is far above any legitimate payload. A v2 delta is capped by +/// `max_delta_bytes` (4 MiB by default) and the v1 shim answers with the whole live +/// state, which is bounded by the gateway's own key set — instances, certificates and +/// node records — not by anything a peer controls. +pub const MAX_DECOMPRESSED_SYNC_BYTES: usize = 128 * 1024 * 1024; + +/// Ceiling on a compressed sync response, mirroring the 16 MiB the routes accept on a +/// request. Without it a peer's response body is read to completion before any decoding +/// bound applies. +pub const MAX_COMPRESSED_SYNC_BYTES: usize = 16 * 1024 * 1024; + +/// Decompress gzip, refusing anything that expands past `limit`. +/// +/// Reads one byte past the limit so a payload landing exactly on it is still accepted +/// and a larger one is rejected rather than silently truncated — `Read::take` alone +/// would hand back a short buffer that then fails to decode, reporting the wrong fault. +pub fn gunzip_bounded(data: &[u8], limit: usize) -> Result> { + use std::io::Read; + + let mut out = Vec::new(); + flate2::read::GzDecoder::new(data) + .take(limit as u64 + 1) + .read_to_end(&mut out) + .context("failed to decompress payload")?; + if out.len() > limit { + anyhow::bail!("decompressed payload exceeds {limit} bytes"); + } + Ok(out) +} + pub fn encode(value: &T) -> Result> { rmp_serde::encode::to_vec_named(value).context("failed to encode value") } @@ -467,9 +518,13 @@ impl KvStore { peer_ids: Vec, data_dir: impl AsRef, ) -> Result { - let persistent = - Node::new_with_persistence(my_node_id, peer_ids.clone(), data_dir.as_ref()) - .context("failed to create persistent wavekv node")?; + let persistent = Node::with_persistence_and_config( + my_node_id, + peer_ids.clone(), + data_dir.as_ref(), + store_config(schema::Store::Persistent), + ) + .context("failed to create persistent wavekv node")?; // Get peers from persistent store (may have been restored from WAL) // and include them when creating ephemeral store @@ -481,7 +536,11 @@ impl KvStore { } } - let ephemeral = Node::new(my_node_id, all_peer_ids); + let ephemeral = Node::with_config( + my_node_id, + all_peer_ids, + store_config(schema::Store::Ephemeral), + ); Ok(Self { persistent, @@ -1355,6 +1414,165 @@ mod value_encoding_tests { } } +/// The gateway speaks two wavekv protocols during a rolling upgrade: the frozen v1 +/// `SyncMessage`/`SyncResponse` pair on `/wavekv/sync`, and the v2 `SyncEnvelope` on +/// `/wavekv/sync2`. These tests pin the wire behaviour of both at the gateway layer. +#[cfg(test)] +mod sync_wire_tests { + use super::*; + use wavekv::sync::{SyncEnvelope, SyncMessage, SyncResponse}; + + fn store(dir: &std::path::Path, id: NodeId, peers: Vec) -> KvStore { + KvStore::new(id, peers, dir).expect("failed to create kv store") + } + + /// A gateway still on wavekv 1.x encodes `SyncMessage` positionally. The v1 route + /// must keep accepting that after this upgrade. + #[test] + fn a_positionally_encoded_v1_request_is_still_accepted() { + let msg = SyncMessage { + sender_id: 2, + sender_uuid: b"uuid".to_vec(), + sender_ack: [(1u32, 5u64)].into_iter().collect(), + entries: Vec::new(), + }; + let legacy = rmp_serde::encode::to_vec(&msg).expect("legacy encode"); + assert_eq!( + legacy[0] & 0xf0, + 0x90, + "fixture must be positional to exercise the legacy path" + ); + + let decoded: SyncMessage = decode(&legacy).expect("the v1 wire format must still decode"); + assert_eq!(decoded.sender_id, 2); + assert_eq!(decoded.sender_ack.get(&1), Some(&5)); + } + + /// ...and the response this gateway sends back must decode on that older peer, + /// which uses a reader built before the named-map switch. + #[test] + fn a_v1_peer_can_decode_our_sync_response() { + let response = SyncResponse { + peer_id: 1, + entries: Vec::new(), + progress: [(1u32, 7u64)].into_iter().collect(), + is_snapshot: true, + }; + let encoded = encode(&response).expect("encode"); + let decoded: SyncResponse = + rmp_serde::decode::from_slice(&encoded).expect("a v1 peer must decode this"); + assert!(decoded.is_snapshot); + assert_eq!(decoded.progress.get(&1), Some(&7)); + } + + #[test] + fn a_v2_envelope_survives_the_transport_framing() { + use flate2::{read::GzDecoder, write::GzEncoder, Compression}; + use std::io::{Read, Write}; + + let dir = tempfile::tempdir().expect("tempdir"); + let kv = store(dir.path(), 1, vec![2]); + kv.persistent() + .write() + .put(keys::peer_addr(1), b"https://a.example".to_vec()) + .expect("put"); + + // Requests deliberately carry no digest: sending it would let any responder + // echo it back and forge agreement forever. So frame a *response*, which is + // the direction the digest actually travels. + assert!(kv + .persistent() + .read() + .prepare_sync(2, Vec::new()) + .digest + .is_none()); + let env = kv + .persistent() + .write() + .handle_envelope(SyncEnvelope::new(2, Vec::new()), Vec::new()) + .expect("respond"); + assert!(!env.entries.is_empty()); + + let mut encoder = GzEncoder::new(Vec::new(), Compression::fast()); + encoder.write_all(&env.encode().expect("encode")).unwrap(); + let wire = encoder.finish().unwrap(); + + let mut plain = Vec::new(); + GzDecoder::new(&wire[..]).read_to_end(&mut plain).unwrap(); + let decoded = SyncEnvelope::decode(&plain).expect("decode"); + + assert_eq!(decoded.sender_id, 1); + assert_eq!(decoded.entries.len(), env.entries.len()); + assert!( + decoded.digest.is_some(), + "the digest drives divergence detection" + ); + } + + /// End-to-end through the shim: a v1-shaped exchange against this gateway's store + /// converges it with the requester's view. + #[test] + fn the_v1_shim_serves_a_complete_delta() { + let dir = tempfile::tempdir().expect("tempdir"); + let kv = store(dir.path(), 1, vec![2]); + for id in 1..=3 { + kv.persistent() + .write() + .put(keys::peer_addr(id), format!("https://n{id}").into_bytes()) + .expect("put"); + } + + let request = SyncMessage { + sender_id: 2, + sender_uuid: Vec::new(), + sender_ack: Default::default(), + entries: Vec::new(), + }; + let response = kv + .persistent() + .write() + .handle_sync_v1(request) + .expect("shim response"); + + assert_eq!(response.entries.len(), 3); + assert!( + response.is_snapshot, + "the flag is what makes a v1 client adopt our coverage before merging" + ); + assert_eq!(response.progress.get(&1), Some(&3)); + } + + /// A peer cannot plant keys outside the schema, in either store. + #[test] + fn merged_entries_outside_the_schema_are_refused() { + use wavekv::types::{Entry, Metadata}; + + let dir = tempfile::tempdir().expect("tempdir"); + let kv = store(dir.path(), 1, vec![2]); + + let mut env = SyncEnvelope::new(2, Vec::new()); + env.entries.push(Entry::new( + "not-a-gateway-key".to_string(), + Some(b"x".to_vec()), + Metadata::new(2, 1, 1), + )); + env.acks.insert(2, 1); + + let outcome = kv + .persistent() + .write() + .apply_envelope(env) + .expect("apply should not fail the whole round"); + + assert_eq!(outcome.rejected, 1); + assert!( + !outcome.acks_adopted, + "a rejection must park the round's acks so the peer keeps re-offering" + ); + assert!(kv.persistent().read().get("not-a-gateway-key").is_none()); + } +} + #[cfg(test)] mod peer_url_tests { use super::validate_peer_url; @@ -1376,3 +1594,67 @@ mod peer_url_tests { } } } + +/// The key namespace is the on-disk contract between releases. +/// +/// Every builder and parser here survived mutation: `handshake_prefix` could return +/// `""`, `parse_inst_key` could return `Some("xyzzy")`, and nothing noticed. That is not +/// a cosmetic gap — these strings are what a gateway uses to find its own state after an +/// upgrade. Changing one silently orphans every existing record: the data is still +/// replicated, still in the digest, and no longer reachable by any reader. +#[cfg(test)] +mod key_schema_tests { + use super::keys; + + /// A prefix must actually be a prefix of the keys it is used to iterate, or a range + /// scan silently returns nothing and the caller reads an empty collection as "none". + #[test] + fn every_iteration_prefix_matches_the_keys_it_must_find() { + assert!(keys::handshake("inst-a", 7).starts_with(&keys::handshake_prefix("inst-a"))); + assert!(keys::last_seen_node(3, 7).starts_with(&keys::last_seen_node_prefix(3))); + assert!(keys::cert_attestation_latest("a.example") + .starts_with(&keys::cert_attestation_prefix("a.example"))); + assert!(keys::cert_attestation_history("a.example", 1234) + .starts_with(&keys::cert_attestation_prefix("a.example"))); + } + + /// A prefix must not be so short that it also matches a neighbour's keys, which + /// would make an iteration return another instance's or node's records. + #[test] + fn an_iteration_prefix_does_not_capture_a_neighbour() { + assert!(!keys::handshake("inst-b", 7).starts_with(&keys::handshake_prefix("inst-a"))); + assert!(!keys::last_seen_node(4, 7).starts_with(&keys::last_seen_node_prefix(3))); + assert!(!keys::cert_attestation_latest("b.example") + .starts_with(&keys::cert_attestation_prefix("a.example"))); + // `inst-a` must not swallow `inst-ab`. + assert!(!keys::handshake("inst-ab", 7).starts_with(&keys::handshake_prefix("inst-a"))); + } + + /// Builders and parsers must agree, or a record written by one release is invisible + /// to the next. + #[test] + fn every_key_parses_back_to_what_built_it() { + assert_eq!(keys::parse_inst_key(&keys::inst("inst-a")), Some("inst-a")); + assert_eq!(keys::parse_node_info_key(&keys::node_info(42)), Some(42)); + assert_eq!( + keys::parse_cert_domain(&keys::cert_attestation_latest("a.example")), + Some("a.example") + ); + assert_eq!( + keys::parse_cert_domain(&keys::cert_lock("a.example")), + Some("a.example") + ); + } + + /// A parser must reject a key from another namespace rather than returning a value + /// derived from it, which would cross-wire two record types. + #[test] + fn a_parser_refuses_a_key_from_another_namespace() { + assert_eq!(keys::parse_inst_key(&keys::node_info(1)), None); + assert_eq!(keys::parse_cert_domain(&keys::inst("inst-a")), None); + assert_eq!(keys::parse_node_info_key(&keys::node_status(1)), None); + assert_eq!(keys::parse_node_info_key(&keys::inst("inst-a")), None); + // `node/info/` and `node/status/` share a stem; neither may claim the other. + assert_eq!(keys::parse_node_info_key("node/info/not-a-number"), None); + } +} diff --git a/dstack/gateway/src/kv/schema.rs b/dstack/gateway/src/kv/schema.rs new file mode 100644 index 000000000..958c2b68d --- /dev/null +++ b/dstack/gateway/src/kv/schema.rs @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Key-prefix admission policy for the replicated stores. +//! +//! Every gateway in a cluster shares one app_id, so mTLS proves only that a peer is +//! *some* gateway of this deployment — not that it is well-behaved. A peer that has +//! been compromised, or that is simply running buggy code, can otherwise write any key +//! it likes into the replicated namespace, and every other node will accept and persist +//! it forever (the data map is never truncated). +//! +//! wavekv 2.0 enforces admission inside `merge`, which covers both sync directions; +//! a check on the HTTP handler would only see inbound requests, not the entries that +//! arrive in a response. Rejected entries also park the round's ack adoption (rule R1), +//! so a peer sending inadmissible data keeps re-offering it rather than having it +//! silently dropped. +//! +//! # Adding a key: this schema must be widened one release before it is used +//! +//! Ack parking makes the schema *forward-incompatible in one direction*. Values may gain +//! fields freely — they are named-map encoded, so an older gateway skips what it does not +//! know. Adding a **key** is different: an older gateway rejects it, which sets +//! `complete = false` for the whole round, which parks ack adoption for that pair +//! entirely. The two nodes then re-exchange the same batch forever and their digests stay +//! unequal. Nothing errors; the pair simply stops making progress, and the symptom is +//! indistinguishable from an unrelated stall such as a peer with a runaway clock. +//! +//! So a new key ships in two releases, never one: +//! +//! 1. Widen the schema to **accept** the new prefix. Do not write it yet. Roll this out +//! to every node. +//! 2. Only then start **writing** it. +//! +//! The same applies in reverse when retiring a key: stop writing it, roll that out, and +//! only afterwards narrow the schema. + +use wavekv::{types::Entry, Admission, AdmissionPolicy}; + +use super::keys; + +/// Which store a policy guards. The two stores have disjoint schemas. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Store { + Persistent, + Ephemeral, +} + +/// Accepts only the key shapes this gateway actually defines. +#[derive(Debug, Clone, Copy)] +pub struct GatewaySchema { + store: Store, +} + +impl GatewaySchema { + pub fn new(store: Store) -> Self { + Self { store } + } + + fn permits(&self, key: &str) -> bool { + match self.store { + Store::Persistent => { + key.starts_with(keys::INST_PREFIX) + || key.starts_with(keys::NODE_PREFIX) + || key.starts_with(keys::CERT_PREFIX) + || key.starts_with(keys::DNS_CRED_PREFIX) + || key.starts_with(keys::PEER_ADDR_PREFIX) + || key == keys::DNS_CRED_DEFAULT + || key == keys::GLOBAL_CERTBOT_CONFIG + || key == keys::GLOBAL_ACME_CREDENTIALS + || key == keys::GLOBAL_ACME_ATTESTATION + || key == keys::GLOBAL_ACME_ROTATION_LOCK + } + Store::Ephemeral => { + key.starts_with(keys::CONN_PREFIX) + || key.starts_with(keys::HANDSHAKE_PREFIX) + || key.starts_with(keys::LAST_SEEN_NODE_PREFIX) + || key.starts_with(keys::PEER_ADDR_PREFIX) + } + } + } +} + +impl AdmissionPolicy for GatewaySchema { + fn admit(&self, entry: &Entry) -> Admission { + if self.permits(&entry.key) { + Admission::Accept + } else { + Admission::Reject { + reason: "key is outside the gateway schema for this store", + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use wavekv::types::Metadata; + + fn entry(key: &str) -> Entry { + Entry::new(key.to_string(), Some(b"v".to_vec()), Metadata::new(1, 1, 0)) + } + + fn admits(store: Store, key: &str) -> bool { + GatewaySchema::new(store).admit(&entry(key)) == Admission::Accept + } + + #[test] + fn every_key_the_gateway_writes_is_admissible() { + for key in [ + keys::inst("abc"), + keys::node_info(1), + keys::node_status(1), + keys::zt_domain_config("example.com"), + keys::cert_data("example.com"), + keys::cert_lock("example.com"), + keys::cert_attestation_latest("example.com"), + keys::cert_attestation_history("example.com", 42), + keys::dns_cred("cred"), + keys::peer_addr(1), + keys::DNS_CRED_DEFAULT.to_string(), + keys::GLOBAL_CERTBOT_CONFIG.to_string(), + keys::GLOBAL_ACME_CREDENTIALS.to_string(), + keys::GLOBAL_ACME_ATTESTATION.to_string(), + keys::GLOBAL_ACME_ROTATION_LOCK.to_string(), + ] { + assert!( + admits(Store::Persistent, &key), + "the persistent schema must admit a key the gateway itself writes: {key}" + ); + } + + for key in [ + keys::conn("inst", 1), + keys::handshake("inst", 1), + keys::last_seen_node(1, 2), + keys::peer_addr(1), + ] { + assert!( + admits(Store::Ephemeral, &key), + "the ephemeral schema must admit a key the gateway itself writes: {key}" + ); + } + } + + #[test] + fn keys_outside_the_schema_are_refused() { + for key in ["", "random", "../escape", "global/", "certificate/x"] { + assert!(!admits(Store::Persistent, key), "accepted {key}"); + assert!(!admits(Store::Ephemeral, key), "accepted {key}"); + } + } + + #[test] + fn the_two_stores_do_not_accept_each_others_keys() { + assert!(!admits(Store::Ephemeral, &keys::inst("abc"))); + assert!(!admits(Store::Ephemeral, &keys::cert_data("example.com"))); + assert!(!admits(Store::Persistent, &keys::conn("inst", 1))); + assert!(!admits(Store::Persistent, &keys::last_seen_node(1, 2))); + } +} diff --git a/dstack/gateway/src/kv/sync_service.rs b/dstack/gateway/src/kv/sync_service.rs index f691595a1..7d3a48ae4 100644 --- a/dstack/gateway/src/kv/sync_service.rs +++ b/dstack/gateway/src/kv/sync_service.rs @@ -13,7 +13,10 @@ use anyhow::{Context, Result}; use dstack_gateway_rpc::GetPeersResponse; use tracing::{info, warn}; use wavekv::{ - sync::{ExchangeInterface, SyncConfig as KvSyncConfig, SyncManager, SyncMessage, SyncResponse}, + sync::{ + ExchangeInterface, PeerLinkStatus, SyncConfig as KvSyncConfig, SyncEnvelope, SyncManager, + SyncMessage, SyncResponse, + }, types::NodeId, Node, }; @@ -37,15 +40,21 @@ pub struct HttpSyncNetwork { } impl HttpSyncNetwork { + /// `my_uuid` is passed in rather than read back out of the store. + /// + /// Our own uuid is local configuration, not replicated state, and sourcing it from + /// the store forced this node's `node/info` record to be written before the service + /// could be built — which is to say before `bootstrap` had rebuilt the sequence + /// counter. After a data-directory loss that made the record spend a sequence number + /// the peers already consider seen, so the one record they check us against was the + /// one guaranteed to be dropped. pub fn new( kv_store: KvStore, store_path: &'static str, tls_config: &HttpsClientConfig, + my_uuid: Vec, ) -> Result { let client = HttpsClient::new(tls_config)?; - let my_uuid = kv_store - .get_peer_uuid(kv_store.my_node_id) - .context("failed to get my UUID")?; Ok(Self { client, kv_store, @@ -70,15 +79,7 @@ impl ExchangeInterface for HttpSyncNetwork { } async fn sync_to(&self, _node: &Node, peer: NodeId, msg: SyncMessage) -> Result { - let url = self - .get_peer_url(peer) - .ok_or_else(|| anyhow::anyhow!("peer {} address not found in DB", peer))?; - - let sync_url = format!( - "{}/wavekv/sync/{}", - url.trim_end_matches('/'), - self.store_path - ); + let sync_url = self.route_for(peer, "sync")?; // Send request with msgpack + gzip encoding // app_id verification happens during TLS handshake via AppIdVerifier @@ -93,6 +94,65 @@ impl ExchangeInterface for HttpSyncNetwork { Ok(sync_response) } + + /// Native v2 exchange. + /// + /// A peer still running a v1 gateway has no `/wavekv/sync2` route and answers 404, + /// which surfaces here as `Ok(None)`; the sync manager then records the peer as + /// v1-only, falls back to `/wavekv/sync`, and re-probes periodically so an upgraded + /// peer is picked up without a restart. + async fn sync_v2_to( + &self, + _node: &Node, + peer: NodeId, + env: SyncEnvelope, + ) -> Result> { + let sync_url = self.route_for(peer, "sync2")?; + + let Some(body) = self + .client + .post_bytes_probe(&sync_url, env.encode()?) + .await + .with_context(|| format!("failed to sync to peer {peer} at {sync_url}"))? + else { + return Ok(None); + }; + + self.kv_store.update_peer_last_seen(peer); + Ok(Some(SyncEnvelope::decode(&body)?)) + } + + /// Opportunistic push. Best-effort by design: the periodic round remains the + /// anti-entropy backstop and the only ack authority. + async fn push_to(&self, _node: &Node, peer: NodeId, env: SyncEnvelope) -> Result<()> { + let push_url = self.route_for(peer, "push")?; + let delivered = self + .client + .post_bytes_probe(&push_url, env.encode()?) + .await + .with_context(|| format!("failed to push to peer {peer} at {push_url}"))?; + // `post_bytes_probe` maps 404/405 to `Ok(None)` so the v2 probe can read it as + // "not upgraded yet". Discarding that here would report a mistyped URL, or a + // peer with no push route, as a delivered push — and pushes are best-effort and + // debug-logged, so nothing else would ever contradict it. + if delivered.is_none() { + anyhow::bail!("peer {peer} has no push route at {push_url}"); + } + Ok(()) + } +} + +impl HttpSyncNetwork { + fn route_for(&self, peer: NodeId, verb: &str) -> Result { + let url = self + .get_peer_url(peer) + .ok_or_else(|| anyhow::anyhow!("peer {peer} address not found in DB"))?; + Ok(format!( + "{}/wavekv/{verb}/{}", + url.trim_end_matches('/'), + self.store_path + )) + } } /// WaveKV sync service that manages synchronization for both persistent and ephemeral stores @@ -112,15 +172,19 @@ impl WaveKvSyncService { kv_store: &KvStore, sync_config: &GwSyncConfig, tls_config: HttpsClientConfig, + my_uuid: Vec, ) -> Result { let sync_config = KvSyncConfig { interval: sync_config.interval, timeout: sync_config.timeout, + ..Default::default() }; // Both networks use the same persistent node for URL lookup, but different paths - let persistent_network = HttpSyncNetwork::new(kv_store.clone(), "persistent", &tls_config)?; - let ephemeral_network = HttpSyncNetwork::new(kv_store.clone(), "ephemeral", &tls_config)?; + let persistent_network = + HttpSyncNetwork::new(kv_store.clone(), "persistent", &tls_config, my_uuid.clone())?; + let ephemeral_network = + HttpSyncNetwork::new(kv_store.clone(), "ephemeral", &tls_config, my_uuid)?; let persistent_manager = Arc::new(SyncManager::with_config( kv_store.persistent().clone(), @@ -173,6 +237,32 @@ impl WaveKvSyncService { pub fn handle_ephemeral_sync(&self, msg: SyncMessage) -> Result { self.ephemeral_manager.handle_sync(msg) } + + fn manager_for(&self, store: &str) -> Option<&Arc>> { + match store { + "persistent" => Some(&self.persistent_manager), + "ephemeral" => Some(&self.ephemeral_manager), + _ => None, + } + } + + /// Handle an inbound v2 sync envelope. + pub fn handle_envelope(&self, store: &str, env: SyncEnvelope) -> Option> { + Some(self.manager_for(store)?.handle_envelope(env)) + } + + /// Handle an inbound opportunistic push (merges data only; never moves acks). + pub fn handle_push(&self, store: &str, env: SyncEnvelope) -> Option> { + Some(self.manager_for(store)?.handle_push(env)) + } + + /// Per-peer protocol and digest telemetry for both stores. + pub fn link_status(&self) -> Vec<(&'static str, Vec)> { + vec![ + ("persistent", self.persistent_manager.link_status()), + ("ephemeral", self.ephemeral_manager.link_status()), + ] + } } /// Fetch peer list from bootnode and register them in KvStore. diff --git a/dstack/gateway/src/main_service.rs b/dstack/gateway/src/main_service.rs index 6167eaf1c..7580698af 100644 --- a/dstack/gateway/src/main_service.rs +++ b/dstack/gateway/src/main_service.rs @@ -182,7 +182,13 @@ impl ProxyInner { ); let state = build_state_from_kv_store(instances); - // Sync this node to KvStore + // This node's own records are written *after* the bootstrap below, not here. + // A local write allocates a sequence number, and after a data-directory loss + // this node has no record of which numbers it already spent — only its peers + // do. `bootstrap` rebuilds the counter from their coverage, so anything written + // before it reuses numbers the peers already treat as seen and is silently + // dropped cluster-wide. That would strand exactly the records recovery depends + // on: the fresh uuid peers check us against, and our sync address. let node_data = NodeData { uuid: config.uuid(), url: config.sync.my_url.clone(), @@ -190,17 +196,6 @@ impl ProxyInner { wg_endpoint: config.wg.endpoint.clone(), wg_ip: config.wg.ip.to_string(), }; - if let Err(err) = kv_store.sync_node(config.sync.node_id, &node_data) { - error!("Failed to sync this node to KvStore: {err:?}"); - } - // Set this node's status to Online - if let Err(err) = kv_store.set_node_status(config.sync.node_id, NodeStatus::Up) { - error!("Failed to set node status: {err:?}"); - } - // Register this node's sync URL in DB (for peer discovery) - if let Err(err) = kv_store.register_peer_url(config.sync.node_id, &config.sync.my_url) { - error!("Failed to register peer URL: {err:?}"); - } // Build HttpsClientConfig for mTLS communication let https_config = { @@ -232,7 +227,12 @@ impl ProxyInner { // Create WaveKV sync service (only if sync is enabled) let wavekv_sync = if config.sync.enabled { - match WaveKvSyncService::new(&kv_store, &config.sync, https_config.clone()) { + match WaveKvSyncService::new( + &kv_store, + &config.sync, + https_config.clone(), + node_data.uuid.clone(), + ) { Ok(sync_service) => Some(Arc::new(sync_service)), Err(err) => { error!("Failed to create WaveKV sync service: {err:?}"); @@ -267,6 +267,20 @@ impl ProxyInner { } } + // Publish this node's own records now that the sequence counter reflects + // whatever the peers already know we have spent (see the note above). + if let Err(err) = kv_store.sync_node(config.sync.node_id, &node_data) { + error!("Failed to sync this node to KvStore: {err:?}"); + } + // Set this node's status to Online + if let Err(err) = kv_store.set_node_status(config.sync.node_id, NodeStatus::Up) { + error!("Failed to set node status: {err:?}"); + } + // Register this node's sync URL in DB (for peer discovery) + if let Err(err) = kv_store.register_peer_url(config.sync.node_id, &config.sync.my_url) { + error!("Failed to register peer URL: {err:?}"); + } + // Create CertResolver and load certificates from KvStore let cert_resolver = Arc::new(CertResolver::new()); let all_cert_data = kv_store.load_all_cert_data(); diff --git a/dstack/gateway/src/pp.rs b/dstack/gateway/src/pp.rs index f6c6e09f2..893e3f844 100644 --- a/dstack/gateway/src/pp.rs +++ b/dstack/gateway/src/pp.rs @@ -251,7 +251,7 @@ mod tests { // PROXY prefix matched but no \r\n terminator within V1_MAX_LENGTH bytes. let bytes = vec![b'P'; V1_MAX_LENGTH + 8]; // all 'P' — never closes let mut head = b"PROXY".to_vec(); - head.extend(std::iter::repeat(b'A').take(V1_MAX_LENGTH)); + head.extend(std::iter::repeat_n(b'A', V1_MAX_LENGTH)); let err = read_proxy_header(&head[..]).await.unwrap_err(); let msg = format!("{err:#}"); assert!( diff --git a/dstack/gateway/src/web_routes.rs b/dstack/gateway/src/web_routes.rs index 5f72735db..55b04f7ff 100644 --- a/dstack/gateway/src/web_routes.rs +++ b/dstack/gateway/src/web_routes.rs @@ -30,5 +30,37 @@ pub fn health_routes() -> Vec { /// WaveKV sync endpoint (for main server, requires mTLS gateway auth) pub fn wavekv_sync_routes() -> Vec { - routes![wavekv_sync::sync_store] + routes![ + wavekv_sync::sync_store, + wavekv_sync::sync_store_v2, + wavekv_sync::push_store + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The v1/v2 negotiation is driven entirely by whether a peer answers 404 on the v2 + /// route. A typo in any of these paths would therefore not fail — every peer would + /// simply 404 forever and the whole cluster would stay silently on v1. + #[test] + fn the_sync_routes_are_mounted_where_peers_look_for_them() { + let mounted: Vec = wavekv_sync_routes() + .iter() + .map(|route| route.uri.to_string()) + .collect(); + + for expected in [ + "/wavekv/sync/", + "/wavekv/sync2/", + "/wavekv/push/", + ] { + assert!( + mounted.iter().any(|uri| uri == expected), + "{expected} is not mounted; peers would 404 and never negotiate v2. \ + mounted: {mounted:?}" + ); + } + } } diff --git a/dstack/gateway/src/web_routes/wavekv_sync.rs b/dstack/gateway/src/web_routes/wavekv_sync.rs index 406c45698..337120291 100644 --- a/dstack/gateway/src/web_routes/wavekv_sync.rs +++ b/dstack/gateway/src/web_routes/wavekv_sync.rs @@ -7,28 +7,32 @@ //! Sync data is encoded using msgpack + gzip compression for efficiency. use crate::{ - kv::{decode, encode}, + kv::{decode, encode, gunzip_bounded, MAX_DECOMPRESSED_SYNC_BYTES}, main_service::Proxy, }; -use flate2::{read::GzDecoder, write::GzEncoder, Compression}; +use flate2::{write::GzEncoder, Compression}; use ra_tls::traits::CertExt; use rocket::{ data::{Data, ToByteUnit}, http::{ContentType, Status}, - mtls::{oid::Oid, Certificate}, + mtls::{oid::Oid, x509::X509Extension, Certificate}, post, State, }; -use std::io::{Read, Write}; +use std::io::Write; use tracing::warn; -use wavekv::sync::{SyncMessage, SyncResponse}; +use wavekv::sync::{SyncEnvelope, SyncMessage, SyncResponse}; -/// Wrapper to implement CertExt for Rocket's Certificate -struct RocketCert<'a>(&'a Certificate<'a>); +/// Adapter implementing `CertExt` over a parsed certificate's extension list. +/// +/// It holds the extensions rather than the `Certificate` so that a test can build one: +/// `rocket::mtls::Certificate` has no public constructor — it can only be produced by a +/// real mTLS handshake — while an extension list comes straight out of `X509Certificate`. +struct RocketCert<'a, 'b>(&'b [X509Extension<'a>]); -impl CertExt for RocketCert<'_> { +impl CertExt for RocketCert<'_, '_> { fn get_extension_der(&self, oid: &[u64]) -> anyhow::Result>> { let oid = Oid::from(oid).map_err(|_| anyhow::anyhow!("failed to create OID from slice"))?; - let Some(ext) = self.0.extensions().iter().find(|ext| ext.oid == oid) else { + let Some(ext) = self.0.iter().find(|ext| ext.oid == oid) else { return Ok(None); }; Ok(Some(ext.value.to_vec())) @@ -37,11 +41,8 @@ impl CertExt for RocketCert<'_> { /// Decode compressed msgpack data fn decode_sync_message(data: &[u8]) -> Result { - // Decompress - let mut decoder = GzDecoder::new(data); - let mut decompressed = Vec::new(); - decoder.read_to_end(&mut decompressed).map_err(|e| { - warn!("failed to decompress sync message: {e}"); + let decompressed = gunzip_bounded(data, MAX_DECOMPRESSED_SYNC_BYTES).map_err(|e| { + warn!("failed to decompress sync message: {e:#}"); Status::BadRequest })?; @@ -57,10 +58,12 @@ fn encode_sync_response(response: &SyncResponse) -> Result, Status> { warn!("failed to encode sync response: {e}"); Status::InternalServerError })?; + gzip(&encoded) +} - // Compress +fn gzip(bytes: &[u8]) -> Result, Status> { let mut encoder = GzEncoder::new(Vec::new(), Compression::fast()); - encoder.write_all(&encoded).map_err(|e| { + encoder.write_all(bytes).map_err(|e| { warn!("failed to compress sync response: {e}"); Status::InternalServerError })?; @@ -70,6 +73,29 @@ fn encode_sync_response(response: &SyncResponse) -> Result, Status> { }) } +fn gunzip(data: &[u8]) -> Result, Status> { + gunzip_bounded(data, MAX_DECOMPRESSED_SYNC_BYTES).map_err(|e| { + warn!("failed to decompress sync payload: {e:#}"); + Status::BadRequest + }) +} + +/// Read a v2 envelope from a request body, applying the same size cap as the v1 route. +async fn read_envelope(data: Data<'_>) -> Result { + let bytes = data + .open(16.mebibytes()) + .into_bytes() + .await + .map_err(|_| Status::BadRequest)?; + let decompressed = gunzip(&bytes)?; + // `SyncEnvelope::decode` enforces the schema version and rejects trailing bytes; + // it is deliberately not the generic `decode` used for KV values. + SyncEnvelope::decode(&decompressed).map_err(|e| { + warn!("failed to decode sync envelope: {e:#}"); + Status::BadRequest + }) +} + /// Verify that the request is from a gateway with the same app_id (mTLS verification) fn verify_gateway_peer(state: &Proxy, cert: Option>) -> Result<(), Status> { // Skip verification if not running in dstack (test mode) @@ -82,7 +108,15 @@ fn verify_gateway_peer(state: &Proxy, cert: Option>) -> Result<( return Err(Status::Unauthorized); }; - let cert = RocketCert(&cert); + authorize_peer(&RocketCert(cert.extensions()), state.my_app_id()) +} + +/// Decide whether a certificate's app identity is one we accept. +/// +/// Split out from `verify_gateway_peer` because that function's other half — the +/// attestation bypass and Rocket's certificate guard — cannot be exercised from a test, +/// which left this decision, the actual authorization rule, uncovered. +fn authorize_peer(cert: &impl CertExt, my_app_id: Option<&[u8]>) -> Result<(), Status> { let remote_app_id = match cert.get_app_id().map_err(|e| { warn!("WaveKV sync: failed to extract app_id from certificate: {e}"); Status::Unauthorized @@ -102,12 +136,8 @@ fn verify_gateway_peer(state: &Proxy, cert: Option>) -> Result<( return Err(Status::Unauthorized); }; - if state.my_app_id() != Some(remote_app_id.as_slice()) { - warn!( - "WaveKV sync: app_id mismatch, expected {:?}, got {:?}", - state.my_app_id(), - remote_app_id - ); + if my_app_id != Some(remote_app_id.as_slice()) { + warn!("WaveKV sync: app_id mismatch, expected {my_app_id:?}, got {remote_app_id:?}"); return Err(Status::Forbidden); } @@ -158,3 +188,637 @@ pub async fn sync_store( Ok((ContentType::new("application", "x-msgpack-gz"), encoded)) } + +/// Native v2 sync endpoint. +/// +/// A gateway still running wavekv 1.x has no route here and answers 404, which is +/// exactly the signal its peers use to fall back to `/wavekv/sync`. Mounting this route +/// is therefore the whole of the server-side protocol negotiation. +#[post("/wavekv/sync2/", data = "")] +pub async fn sync_store_v2( + state: &State, + cert: Option>, + store: &str, + data: Data<'_>, +) -> Result<(ContentType, Vec), Status> { + verify_gateway_peer(state, cert)?; + + let Some(ref wavekv_sync) = state.wavekv_sync else { + return Err(Status::ServiceUnavailable); + }; + + let env = read_envelope(data).await?; + if env.sender_id == 0 { + warn!("rejected v2 sync from invalid node_id 0"); + return Err(Status::BadRequest); + } + + let Some(result) = wavekv_sync.handle_envelope(store, env) else { + return Err(Status::NotFound); + }; + let response = result.map_err(|e| { + tracing::error!("{store} v2 sync failed: {e:#}"); + Status::InternalServerError + })?; + + let encoded = response.encode().map_err(|e| { + warn!("failed to encode sync envelope: {e:#}"); + Status::InternalServerError + })?; + Ok(( + ContentType::new("application", "x-msgpack-gz"), + gzip(&encoded)?, + )) +} + +/// Opportunistic push endpoint (wavekv RFC 0001 section 3.9). +/// +/// Entries only: the receiver merges data but never moves its ack coverage from this +/// channel, so loss, duplication and reordering here are all harmless and the periodic +/// round remains the anti-entropy backstop. +#[post("/wavekv/push/", data = "")] +pub async fn push_store( + state: &State, + cert: Option>, + store: &str, + data: Data<'_>, +) -> Result { + verify_gateway_peer(state, cert)?; + + let Some(ref wavekv_sync) = state.wavekv_sync else { + return Err(Status::ServiceUnavailable); + }; + + let env = read_envelope(data).await?; + if env.sender_id == 0 { + warn!("rejected push from invalid node_id 0"); + return Err(Status::BadRequest); + } + + let Some(result) = wavekv_sync.handle_push(store, env) else { + return Err(Status::NotFound); + }; + result.map_err(|e| { + tracing::error!("{store} push failed: {e:#}"); + Status::InternalServerError + })?; + Ok(Status::Ok) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{load_config_figment, Config, MutualConfig, TlsConfig}; + use crate::kv::NodeData; + use crate::main_service::{Proxy, ProxyOptions}; + use rocket::local::asynchronous::Client; + use tempfile::TempDir; + use wavekv::types::{Entry, Metadata}; + + const ME: u32 = 1; + const PEER: u32 = 2; + + fn peer_uuid() -> Vec { + b"the-real-peer-2".to_vec() + } + + /// A self-signed CA plus a leaf it signs. `HttpSyncNetwork::new` loads all three + /// from disk to build its rustls client config, and the root store only accepts a + /// trust anchor with `CA:TRUE` — so a lone self-signed leaf is not enough. + fn write_tls_material(dir: &std::path::Path) -> TlsConfig { + use ra_tls::rcgen::{BasicConstraints, CertificateParams, IsCa, KeyPair}; + + let ca_key = KeyPair::generate().expect("ca key"); + let mut ca_params = CertificateParams::new(vec![]).expect("ca params"); + ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + let ca_cert = ca_params.self_signed(&ca_key).expect("ca cert"); + + let leaf_key = KeyPair::generate().expect("leaf key"); + let leaf_params = + CertificateParams::new(vec!["gateway.test".to_string()]).expect("leaf params"); + let leaf_cert = leaf_params + .signed_by(&leaf_key, &ca_cert, &ca_key) + .expect("leaf cert"); + + let cert_path = dir.join("node.crt"); + let key_path = dir.join("node.key"); + let ca_path = dir.join("ca.crt"); + std::fs::write(&cert_path, leaf_cert.pem()).expect("write cert"); + std::fs::write(&key_path, leaf_key.serialize_pem()).expect("write key"); + std::fs::write(&ca_path, ca_cert.pem()).expect("write ca"); + + TlsConfig { + certs: cert_path.to_string_lossy().into_owned(), + key: key_path.to_string_lossy().into_owned(), + mutual: MutualConfig { + ca_certs: ca_path.to_string_lossy().into_owned(), + }, + } + } + + /// A gateway serving the real sync routes over Rocket's local client. + /// + /// `insecure_skip_attestation` is on, which makes `verify_gateway_peer` return + /// immediately: these tests are about everything below it — route dispatch, the gzip + /// framing, the store split, the uuid check. `enforcing_gateway` covers the gate + /// itself, which this fixture cannot, because Rocket's local client speaks no TLS + /// and so can never present a certificate. + async fn serving_gateway(sync_enabled: bool) -> (Client, Proxy, TempDir) { + serving_gateway_with(sync_enabled, true).await + } + + /// The same gateway with the attestation bypass switched off, so the peer check runs + /// for real. + async fn enforcing_gateway() -> (Client, Proxy, TempDir) { + serving_gateway_with(true, false).await + } + + async fn serving_gateway_with( + sync_enabled: bool, + skip_attestation: bool, + ) -> (Client, Proxy, TempDir) { + // `main` installs this once at startup; the sync client builds a rustls config, + // so a test that skips it panics inside rustls rather than failing an assertion. + let _ = rustls::crypto::ring::default_provider().install_default(); + + let figment = load_config_figment(None); + let mut config = figment.focus("core").extract::().unwrap(); + let temp_dir = TempDir::new().expect("temp dir"); + + config.sync.enabled = sync_enabled; + config.sync.node_id = ME; + config.sync.bootnode = String::new(); + config.sync.data_dir = temp_dir.path().to_string_lossy().into_owned(); + config.wg.config_path = temp_dir + .path() + .join("wg.conf") + .to_string_lossy() + .into_owned(); + config.debug.insecure_skip_attestation = skip_attestation; + + let tls_config = write_tls_material(temp_dir.path()); + let proxy = Proxy::new(ProxyOptions { + config, + my_app_id: None, + tls_config, + }) + .await + .expect("failed to build gateway"); + + let rocket = rocket::build() + .manage(proxy.clone()) + .mount("/", crate::web_routes::wavekv_sync_routes()); + let client = Client::tracked(rocket).await.expect("rocket client"); + (client, proxy, temp_dir) + } + + /// The sync routes are the cluster's write surface: anything that reaches them can + /// insert entries that replicate to every gateway. `verify_gateway_peer` is the only + /// thing standing in front of them, and with `insecure_skip_attestation` set — which + /// every other test here sets — its first statement returns `Ok(())`, so the gate + /// itself was never executed by any test. Replacing the whole function body with + /// `Ok(())` did not turn the suite red. + /// + /// Rocket's local client speaks no TLS and so presents no certificate, which is + /// exactly the case that must be refused. + #[tokio::test] + async fn every_sync_route_refuses_a_peer_it_cannot_identify() { + let (client, _proxy, _tmp) = enforcing_gateway().await; + + for route in [ + "/wavekv/sync/persistent", + "/wavekv/sync2/persistent", + "/wavekv/push/persistent", + ] { + let response = client.post(route).body(Vec::new()).dispatch().await; + assert_eq!( + response.status(), + Status::Unauthorized, + "{route} served a request from an unauthenticated caller" + ); + } + } + + /// A real certificate carrying `PHALA_RATLS_APP_ID`, minted locally. + /// + /// Nothing here needs a TEE: the extension is an ordinary X.509 extension that + /// `CertRequest` adds unconditionally, and the check under test never looks at a + /// quote — it reads two extensions and compares bytes. + fn cert_with_app_id(app_id: &[u8]) -> Vec { + use ra_tls::cert::CertRequest; + use ra_tls::rcgen::KeyPair; + + let key = KeyPair::generate().expect("key"); + let cert = CertRequest::builder() + .key(&key) + .subject("peer.test") + .app_id(app_id) + .build() + .self_signed() + .expect("self-signed cert"); + cert.der().to_vec() + } + + /// A certificate with no app identity at all. + fn cert_without_app_id() -> Vec { + use ra_tls::cert::CertRequest; + use ra_tls::rcgen::KeyPair; + + let key = KeyPair::generate().expect("key"); + let cert = CertRequest::builder() + .key(&key) + .subject("peer.test") + .build() + .self_signed() + .expect("self-signed cert"); + cert.der().to_vec() + } + + fn authorize(der: &[u8], my_app_id: Option<&[u8]>) -> Result<(), Status> { + use rocket::mtls::x509::{FromDer, X509Certificate}; + let (_, parsed) = X509Certificate::from_der(der).expect("parse cert"); + authorize_peer(&RocketCert(parsed.extensions()), my_app_id) + } + + /// The rule the sync routes are defended by: same app id or nothing. + /// + /// Every case below was previously unreachable, because the only tests that touched + /// this code set `insecure_skip_attestation` and returned before it. Inverting the + /// comparison to `==` left the suite green. + #[test] + fn a_peer_is_authorized_only_when_its_app_id_matches_ours() { + let ours = b"app-id-of-this-cluster".to_vec(); + + assert_eq!(authorize(&cert_with_app_id(&ours), Some(&ours)), Ok(())); + + assert_eq!( + authorize(&cert_with_app_id(b"a-different-app"), Some(&ours)), + Err(Status::Forbidden), + "a valid certificate from another app must not reach the sync routes" + ); + } + + /// A certificate that proves nothing about which app presented it is refused, rather + /// than falling through to a comparison against `None`. + #[test] + fn a_certificate_without_an_app_id_is_refused() { + assert_eq!( + authorize(&cert_without_app_id(), Some(b"app-id-of-this-cluster")), + Err(Status::Unauthorized) + ); + } + + /// A gateway that does not know its own app id cannot authorize anyone. Comparing + /// `None` against a present remote id must reject, never match. + #[test] + fn a_gateway_without_an_app_id_authorizes_nobody() { + assert_eq!( + authorize(&cert_with_app_id(b"anything"), None), + Err(Status::Forbidden) + ); + } + + /// The adapter must match the app-id extension by OID and no other. Returning some + /// other extension's bytes would hand `authorize_peer` a value it would happily + /// compare. + #[test] + fn the_adapter_reads_the_app_id_extension_and_not_a_neighbour() { + use ra_tls::traits::CertExt; + use rocket::mtls::x509::{FromDer, X509Certificate}; + + let der = cert_with_app_id(b"the-app-id"); + let (_, parsed) = X509Certificate::from_der(&der).expect("parse cert"); + let adapter = RocketCert(parsed.extensions()); + + assert_eq!( + adapter.get_app_id().expect("read app id"), + Some(b"the-app-id".to_vec()) + ); + assert_eq!( + adapter.get_special_usage().expect("read special usage"), + None, + "an extension that was never set must read back as absent" + ); + } + + /// Register the peer so `query_uuid` returns something: the uuid check is opt-in and + /// an unknown sender bypasses it entirely. + fn register_peer(proxy: &Proxy) { + proxy + .kv_store() + .sync_node( + PEER, + &NodeData { + uuid: peer_uuid(), + url: "https://peer.test:8011".to_string(), + wg_public_key: String::new(), + wg_endpoint: String::new(), + wg_ip: String::new(), + }, + ) + .expect("register peer"); + } + + fn push_envelope(uuid: Vec, key: &str) -> SyncEnvelope { + let mut env = SyncEnvelope::new(PEER, uuid); + env.push_only = true; + env.entries.push(Entry::new( + key.to_string(), + Some(b"v".to_vec()), + Metadata::new(PEER, 1, 1), + )); + env + } + + fn body(env: &SyncEnvelope) -> Vec { + gzip(&env.encode().expect("encode envelope")).expect("gzip") + } + + #[tokio::test] + async fn a_stamped_push_is_accepted_and_lands_in_the_store() { + let (client, proxy, _tmp) = serving_gateway(true).await; + register_peer(&proxy); + + let response = client + .post("/wavekv/push/persistent") + .body(body(&push_envelope(peer_uuid(), "node/9"))) + .dispatch() + .await; + + assert_eq!(response.status(), Status::Ok); + assert!( + proxy.kv_store().persistent().read().get("node/9").is_some(), + "a well-formed push must reach the store" + ); + } + + /// The route-level view of the bug that made every opportunistic push fail: the + /// sender built its envelope without stamping `sender_uuid`, and the receiver's + /// `check_uuid` — which only the manager runs, not `merge_push` — rejected it. + #[tokio::test] + async fn an_unstamped_push_is_refused_at_the_route() { + let (client, proxy, _tmp) = serving_gateway(true).await; + register_peer(&proxy); + + let response = client + .post("/wavekv/push/persistent") + .body(body(&push_envelope(Vec::new(), "node/9"))) + .dispatch() + .await; + + assert_eq!(response.status(), Status::InternalServerError); + assert!( + proxy.kv_store().persistent().read().get("node/9").is_none(), + "a push that fails the identity check must not write anything" + ); + } + + #[tokio::test] + async fn a_v2_round_trip_returns_a_decodable_envelope() { + let (client, proxy, _tmp) = serving_gateway(true).await; + register_peer(&proxy); + proxy + .kv_store() + .persistent() + .write() + .put("node/7".to_string(), b"v".to_vec()) + .expect("seed"); + + let request = SyncEnvelope::new(PEER, peer_uuid()); + let response = client + .post("/wavekv/sync2/persistent") + .body(body(&request)) + .dispatch() + .await; + + assert_eq!(response.status(), Status::Ok); + let bytes = response.into_bytes().await.expect("body"); + let decoded = SyncEnvelope::decode(&gunzip(&bytes).expect("gunzip")).expect("decode"); + assert_eq!(decoded.sender_id, ME); + assert!( + decoded.entries.iter().any(|e| e.key == "node/7"), + "an empty ack map must draw the whole live state" + ); + } + + /// 404 is the negotiation signal: it is what tells a peer "this node has no v2 + /// route, fall back to v1". Nothing else on these routes may produce it by accident. + #[tokio::test] + async fn an_unknown_store_is_a_404_because_that_is_the_v1_signal() { + let (client, proxy, _tmp) = serving_gateway(true).await; + register_peer(&proxy); + + let response = client + .post("/wavekv/sync2/bogus") + .body(body(&SyncEnvelope::new(PEER, peer_uuid()))) + .dispatch() + .await; + + assert_eq!(response.status(), Status::NotFound); + } + + fn v1_body(msg: &SyncMessage) -> Vec { + gzip(&encode(msg).expect("encode v1 message")).expect("gzip") + } + + fn v1_request() -> SyncMessage { + SyncMessage { + sender_id: PEER, + sender_uuid: peer_uuid(), + // Empty coverage, so the shim answers with everything it holds. + sender_ack: Default::default(), + entries: Vec::new(), + } + } + + /// The v1 shim is how a gateway that has not been upgraded still receives state, and + /// nothing exercised it at the route level: the store dispatch could be deleted, the + /// node-id-zero guard inverted, and the response body replaced with three bytes, + /// all without turning the suite red. + /// + /// Deleting the `"persistent"` arm is the sharpest of those. It falls through to + /// `_ => 404`, and a 404 on a sync route is precisely the signal a v2 peer reads as + /// "this node does not speak that protocol" — so the failure would not look like an + /// error, it would look like a successful protocol downgrade. + #[tokio::test] + async fn a_v1_round_trip_serves_the_state_this_node_holds() { + let (client, proxy, _tmp) = serving_gateway(true).await; + register_peer(&proxy); + proxy + .kv_store() + .persistent() + .write() + .put("node/7".to_string(), b"v".to_vec()) + .expect("seed"); + + let response = client + .post("/wavekv/sync/persistent") + .body(v1_body(&v1_request())) + .dispatch() + .await; + + assert_eq!(response.status(), Status::Ok); + let bytes = response.into_bytes().await.expect("body"); + let decoded: SyncResponse = + decode(&gunzip(&bytes).expect("gunzip")).expect("decode v1 response"); + + assert_eq!(decoded.peer_id, ME); + assert!( + decoded.entries.iter().any(|e| e.key == "node/7"), + "a peer with no coverage must receive the state this node holds" + ); + } + + /// Both stores are reachable over the v1 route. The ephemeral arm carries the + /// liveness data a stale peer needs most, and losing it would read as a downgrade + /// rather than a fault, exactly as above. + #[tokio::test] + async fn the_v1_route_serves_the_ephemeral_store_as_well() { + let (client, proxy, _tmp) = serving_gateway(true).await; + register_peer(&proxy); + + let response = client + .post("/wavekv/sync/ephemeral") + .body(v1_body(&v1_request())) + .dispatch() + .await; + + assert_eq!( + response.status(), + Status::Ok, + "a 404 here would demote this node to no-such-route in the caller's cache" + ); + } + + /// Node id 0 is the unset value, so an entry authored by it collides with every + /// other unset sender. The v1 route rejects it, as the push and v2 routes do. + #[tokio::test] + async fn a_v1_sync_from_node_id_zero_is_refused() { + let (client, proxy, _tmp) = serving_gateway(true).await; + register_peer(&proxy); + + let mut msg = v1_request(); + msg.sender_id = 0; + let response = client + .post("/wavekv/sync/persistent") + .body(v1_body(&msg)) + .dispatch() + .await; + + assert_eq!(response.status(), Status::BadRequest); + } + + /// ...which is why a node with sync switched off must answer 503 and not 404. A 404 + /// here would demote this node to v1 in every peer's cache for a whole reprobe + /// window — silently, and without sync being on to fix it. + #[tokio::test] + async fn a_sync_disabled_node_answers_503_rather_than_404() { + let (client, _proxy, _tmp) = serving_gateway(false).await; + + for path in [ + "/wavekv/sync/persistent", + "/wavekv/sync2/persistent", + "/wavekv/push/persistent", + ] { + let response = client + .post(path) + .body(body(&SyncEnvelope::new(PEER, peer_uuid()))) + .dispatch() + .await; + assert_eq!( + response.status(), + Status::ServiceUnavailable, + "{path} must not look like a missing v2 route" + ); + } + } + + /// gzip expands by three orders of magnitude on attacker-chosen input, so the + /// 16 MiB cap on the request body bounds the *compressed* size and nothing else. + /// mTLS proves only that the sender is some gateway of this deployment, which is + /// the same trust level the key schema already assumes is insufficient. + #[tokio::test] + async fn a_compression_bomb_is_refused_before_it_is_decompressed() { + let (client, _proxy, _tmp) = serving_gateway(true).await; + + // ~130 MiB of zeroes compresses to well under the request cap. + let bomb = gzip(&vec![0u8; MAX_DECOMPRESSED_SYNC_BYTES + 1]).expect("gzip"); + assert!( + bomb.len() < 16 * 1024 * 1024, + "the fixture has to fit through the body cap to be testing anything: {} bytes", + bomb.len() + ); + + for path in [ + "/wavekv/sync/persistent", + "/wavekv/sync2/persistent", + "/wavekv/push/persistent", + ] { + let response = client.post(path).body(bomb.clone()).dispatch().await; + assert_eq!( + response.status(), + Status::BadRequest, + "{path} must refuse an over-sized expansion" + ); + } + } + + /// The limits must leave room for the largest legitimate message. + /// + /// The boundary test below asserts a payload of exactly `MAX_DECOMPRESSED_SYNC_BYTES` + /// is accepted — but it builds that payload *from the same constant*, so it holds + /// whatever the constant says. Shrinking the limit to a few kilobytes keeps it green + /// while rejecting every real delta. Pin the values against what production sends, + /// which is the property that actually matters. + // Deliberately runtime assertions rather than `const { assert!(..) }`: a const block + // would fail the build, which mutation testing scores as "unviable" rather than + // "caught", and would lose the message explaining what the number is for. + #[allow(clippy::assertions_on_constants)] + #[test] + fn the_sync_limits_admit_the_largest_message_the_protocol_can_produce() { + // A v2 delta is capped by wavekv's `max_delta_bytes` (4 MiB by default), and the + // v1 shim answers with the whole live state. + const MAX_DELTA_BYTES: usize = 4 * 1024 * 1024; + assert!( + MAX_DECOMPRESSED_SYNC_BYTES >= 8 * MAX_DELTA_BYTES, + "a decompression limit of {MAX_DECOMPRESSED_SYNC_BYTES} bytes would reject \ + ordinary sync traffic, not just a bomb" + ); + + // The compressed ceiling mirrors what the routes accept on a request, so a peer + // cannot answer with more than it would have been allowed to ask. + assert_eq!( + crate::kv::MAX_COMPRESSED_SYNC_BYTES, + 16 * 1024 * 1024, + "this must stay equal to the 16 MiB the routes accept on a request body" + ); + } + + /// The limit is inclusive, so a payload landing exactly on it still decodes. Without + /// this the bound could tighten by a byte and only the bomb test would still pass. + #[test] + fn a_payload_exactly_on_the_limit_still_decompresses() { + let exact = gzip(&vec![7u8; MAX_DECOMPRESSED_SYNC_BYTES]).expect("gzip"); + let out = gunzip_bounded(&exact, MAX_DECOMPRESSED_SYNC_BYTES).expect("must be accepted"); + assert_eq!(out.len(), MAX_DECOMPRESSED_SYNC_BYTES); + + let one_over = gzip(&vec![7u8; MAX_DECOMPRESSED_SYNC_BYTES + 1]).expect("gzip"); + assert!(gunzip_bounded(&one_over, MAX_DECOMPRESSED_SYNC_BYTES).is_err()); + } + + #[tokio::test] + async fn a_push_from_node_id_zero_is_refused() { + let (client, proxy, _tmp) = serving_gateway(true).await; + register_peer(&proxy); + + let mut env = push_envelope(peer_uuid(), "node/9"); + env.sender_id = 0; + let response = client + .post("/wavekv/push/persistent") + .body(body(&env)) + .dispatch() + .await; + + assert_eq!(response.status(), Status::BadRequest); + } +}