diff --git a/members/nullnet-client/src/commands/dnat.rs b/members/nullnet-client/src/commands/dnat.rs index cb76ca8..422f859 100644 --- a/members/nullnet-client/src/commands/dnat.rs +++ b/members/nullnet-client/src/commands/dnat.rs @@ -28,23 +28,38 @@ pub(crate) fn init() { /// Install a DNAT for `port → overlay_ip:port`. When `container_ip` is a /// real address, the rule is scoped to that source via `-s` so co-located -/// replicas hit independent chains. `Ipv4Addr::UNSPECIFIED` (0.0.0.0) means -/// "no source filter" — used by legacy callers that don't know the source. +/// replicas hit independent chains. When `dest_ip` is a real address, the +/// rule is additionally scoped to that destination via `-d` — so two +/// backend-trigger dependencies sharing a port from the same initiator (each +/// with its own placeholder address, see `placeholder.rs`) get independent +/// rules instead of one clobbering the other. `Ipv4Addr::UNSPECIFIED` +/// (0.0.0.0) means "no filter" for either — used by legacy callers that +/// don't know the source/destination. /// Returns `false` if any of the per-proto `iptables` rules failed to apply. -pub(crate) fn install(port: u16, overlay_ip: Ipv4Addr, container_ip: Ipv4Addr) -> bool { +pub(crate) fn install( + port: u16, + overlay_ip: Ipv4Addr, + container_ip: Ipv4Addr, + dest_ip: Ipv4Addr, +) -> bool { let mut ok = true; for proto in PROTOS { - ok &= run_iptables("-A", proto, port, overlay_ip, container_ip); + ok &= run_iptables("-A", proto, port, overlay_ip, container_ip, dest_ip); } flush_conntrack(port); ok } /// Returns `false` if any of the per-proto `iptables` rules failed to delete. -pub(crate) fn remove(port: u16, overlay_ip: Ipv4Addr, container_ip: Ipv4Addr) -> bool { +pub(crate) fn remove( + port: u16, + overlay_ip: Ipv4Addr, + container_ip: Ipv4Addr, + dest_ip: Ipv4Addr, +) -> bool { let mut ok = true; for proto in PROTOS { - ok &= run_iptables("-D", proto, port, overlay_ip, container_ip); + ok &= run_iptables("-D", proto, port, overlay_ip, container_ip, dest_ip); } flush_conntrack(port); ok @@ -57,14 +72,19 @@ fn run_iptables( port: u16, overlay_ip: Ipv4Addr, container_ip: Ipv4Addr, + dest_ip: Ipv4Addr, ) -> bool { let port_s = port.to_string(); let target = format!("{overlay_ip}:{port}"); let container_ip_s = container_ip.to_string(); + let dest_ip_s = dest_ip.to_string(); let mut args: Vec<&str> = vec!["iptables", "-t", "nat", action, CHAIN, "-p", proto]; if !container_ip.is_unspecified() { args.extend_from_slice(&["-s", &container_ip_s]); } + if !dest_ip.is_unspecified() { + args.extend_from_slice(&["-d", &dest_ip_s]); + } args.extend_from_slice(&[ "--dport", &port_s, @@ -79,19 +99,28 @@ fn run_iptables( } else { container_ip_s.clone() }; + let dst = if dest_ip.is_unspecified() { + "any".to_string() + } else { + dest_ip_s.clone() + }; match status { Ok(s) if s.success() => { - println!("[dnat] iptables {action} {CHAIN} {proto}/{port} -s {src} -> {target}"); + println!( + "[dnat] iptables {action} {CHAIN} {proto}/{port} -s {src} -d {dst} -> {target}" + ); true } Ok(s) => { eprintln!( - "[dnat] iptables {action} {CHAIN} {proto}/{port} -s {src} -> {target} exited {s}" + "[dnat] iptables {action} {CHAIN} {proto}/{port} -s {src} -d {dst} -> {target} exited {s}" ); false } Err(e) => { - eprintln!("[dnat] iptables {action} {CHAIN} {proto}/{port} -s {src} -> {target}: {e}"); + eprintln!( + "[dnat] iptables {action} {CHAIN} {proto}/{port} -s {src} -d {dst} -> {target}: {e}" + ); false } } diff --git a/members/nullnet-client/src/commands/egress.rs b/members/nullnet-client/src/commands/egress.rs index df88b1e..92b4567 100644 --- a/members/nullnet-client/src/commands/egress.rs +++ b/members/nullnet-client/src/commands/egress.rs @@ -95,6 +95,19 @@ pub(crate) fn init() { &["ipset", "add", "-exist", INTERNAL_SET, range], ); } + // The backend-trigger placeholder block (see `placeholder::placeholder_cidr`) + // is synthetic — never real internet traffic — but isn't one of the + // static private/special ranges above. Without this, a trigger dial to + // a port `nullnet_watched_ports` hasn't caught up on yet (a startup race + // against the gRPC round trip that populates it, see `nfqueue::apply_ports_diff`) + // falls through to this rule instead and gets misclassified as an egress + // candidate — held for the egress-trigger timeout, then dropped, well + // past most callers' own request timeout. + let placeholder_cidr = crate::placeholder::placeholder_cidr(); + sudo_ok( + "ipset add internal placeholder", + &["ipset", "add", "-exist", INTERNAL_SET, &placeholder_cidr], + ); // NFQUEUE trigger rule: NEW flows to non-internal destinations → queue 1. // The listener filters by container (registered services only). --queue-bypass @@ -213,6 +226,27 @@ pub(crate) fn install_steer( ], ); } + // Same bypass for the backend-trigger placeholder block — see the + // matching comment in `init()`. Lands at base+9, still clear of the + // catch-all at base+15. + let placeholder_cidr = crate::placeholder::placeholder_cidr(); + let placeholder_prio = (base + INTERNAL_RANGES.len() as u32).to_string(); + ok &= sudo_ok( + "ip rule add placeholder bypass", + &[ + "ip", + "rule", + "add", + "from", + &cip, + "to", + &placeholder_cidr, + "lookup", + "main", + "priority", + &placeholder_prio, + ], + ); // Catch-all: everything else from this container → the egress table. let catch_all = (base + 15).to_string(); ok &= sudo_ok( diff --git a/members/nullnet-client/src/control_channel.rs b/members/nullnet-client/src/control_channel.rs index 67e02db..7ada815 100644 --- a/members/nullnet-client/src/control_channel.rs +++ b/members/nullnet-client/src/control_channel.rs @@ -2,7 +2,7 @@ use crate::commands::{RtNetLinkHandle, configure_access_port, dnat, egress, remo use crate::ebpf::{FirewallPeers, FirewallVxlanPorts, NetId}; use crate::egress_policy::{PolicyVerdicts, flush_container_conntrack}; use crate::egress_state::{EgressRecord, EgressState}; -use crate::host_mappings::{HOSTS_MARKER, HostMappingsState, hosts_file_lock}; +use crate::host_mappings::{self, HOSTS_MARKER, HostMappingsState, hosts_file_lock}; use crate::nfqueue::BridgeIpCache; use crate::peers::peer::{Peers, VethKey}; use crate::triggers::TriggersState; @@ -493,6 +493,7 @@ async fn handle_vxlan_setup( if let Some(container) = message.docker_container.as_deref() { triggers_state.mark_active( container, + crate::triggers::EGRESS_DST_IP, crate::triggers::EGRESS_TRIGGER_PORT, vxlan_id, gw, @@ -552,7 +553,7 @@ async fn handle_vxlan_setup( // packet traverses `nat PREROUTING`. The DNAT rule MUST already be // installed by then, so we: // 1. peek the initiator's bridge IP (stashed at `mark_pending`) - // 2. install DNAT with `-s ` + // 2. install DNAT with `-s -d ` // 3. mark_active → wakes the waiter, packet released into the new // rule if let Some(dnat_port) = message.dnat_port @@ -560,14 +561,21 @@ async fn handle_vxlan_setup( && let Ok(overlay_ip) = host_mapping.ip.parse::() { let container_key = message.docker_container.as_deref().unwrap_or(""); - let container_ip = triggers_state.peek_container_ip(container_key, dnat_port); + // `host_mapping.name` is chain[0] for this trigger — the same + // literal name the NFQUEUE listener read from its port→target + // map when it observed the original packet — so recomputing the + // placeholder here independently lands on the exact same + // address it used, without needing it on the wire separately. + let dest_ip = crate::placeholder::ip_for(&host_mapping.name); + let container_ip = triggers_state.peek_container_ip(container_key, dest_ip, dnat_port); // Only promote to Active if the DNAT rule is actually live. Waking // the held packet without it would release the SYN into a missing // rule (→ misroute to the original dest); instead leave it Pending // so the listener drops at ACTIVE_TIMEOUT. - if dnat::install(dnat_port, overlay_ip, container_ip) { + if dnat::install(dnat_port, overlay_ip, container_ip, dest_ip) { triggers_state.mark_active( container_key, + dest_ip, dnat_port, vxlan_id, overlay_ip, @@ -654,30 +662,52 @@ fn handle_vxlan_teardown( firewall_vxlan_ports.remove(message.vxlan_id); // remove DNAT before tearing the tunnel down so existing flows reset - // cleanly. The `container_ip` matches the `-s` we used at install time. - // `remove_by_vxlan` also matches the egress steer's sentinel-port entry - // (EGRESS_TRIGGER_PORT = 0). That path installs a policy-route steer, not a - // DNAT — its teardown runs via EgressState below — so skip DNAT removal for - // it. Only real backend DNAT ports (>0) go through `dnat::remove`; otherwise - // we'd fire a bogus `iptables -D --dport 0` and a false removal-failed event. - if let Some((_container, port, overlay_ip, container_ip)) = + // cleanly. The `container_ip`/`dst_ip` match the `-s`/`-d` we used at + // install time. `remove_by_vxlan` also matches the egress steer's + // sentinel-port entry (EGRESS_TRIGGER_PORT = 0). That path installs a + // policy-route steer, not a DNAT — its teardown runs via EgressState + // below — so skip DNAT removal for it. Only real backend DNAT ports (>0) + // go through `dnat::remove`; otherwise we'd fire a bogus + // `iptables -D --dport 0` and a false removal-failed event. Whether this + // was a backend-trigger entry (vs. an egress steer) also decides how the + // host mapping below gets torn down. + let mut is_backend_trigger_entry = false; + if let Some((_container, dst_ip, port, overlay_ip, container_ip)) = triggers_state.remove_by_vxlan(message.vxlan_id) && port != crate::triggers::EGRESS_TRIGGER_PORT - && !dnat::remove(port, overlay_ip, container_ip) { - fire_event( - &grpc, - AgentEventKind::DnatRemovalFailed(AgentDnatRemovalFailed { - port: u32::from(port), - overlay_ip: overlay_ip.to_string(), - }), - ); + is_backend_trigger_entry = true; + if !dnat::remove(port, overlay_ip, container_ip, dst_ip) { + fire_event( + &grpc, + AgentEventKind::DnatRemovalFailed(AgentDnatRemovalFailed { + port: u32::from(port), + overlay_ip: overlay_ip.to_string(), + }), + ); + } } - // remove host mapping if one was installed at setup + // Remove the host mapping installed at setup — except for a + // backend-trigger entry, where we re-seed the placeholder instead of + // deleting the line outright. Deleting would leave the initiator unable + // to resolve the name at all: the next `connect(name, ...)` would + // dead-end exactly like an un-seeded bare name does (no packet, no + // re-trigger, chain never rebuilds). Proxy-dependency mappings are + // unaffected — a fresh proxy request rebuilds the chain (and writes the + // real mapping) before forwarding, so there's no gap to cover there. if let Some((host_mapping, docker_container)) = host_mappings_state.take_vxlan(message.vxlan_id) { - let _ = remove_host_mapping(&host_mapping, docker_container.as_deref()); + if is_backend_trigger_entry && let Some(container) = docker_container.as_deref() { + if let Err(e) = crate::placeholder::seed_placeholder(container, &host_mapping.name) { + eprintln!( + "[control_channel] failed to re-seed placeholder for '{}' in {container}: {e:?}", + host_mapping.name + ); + } + } else { + let _ = remove_host_mapping(&host_mapping, docker_container.as_deref()); + } } // teardown VXLAN on this machine @@ -842,147 +872,49 @@ fn container_running(container: &str) -> bool { } fn add_host_mapping(hm: &HostMapping, docker_container: Option<&str>) -> Result<(), Error> { - let path = "/etc/hosts"; - // Marked so a restarted process can sweep its predecessor's entries without - // touching the operator's (see `host_mappings::purge_stale_mappings`). - let entry = format!("{} {} {HOSTS_MARKER}", hm.ip, hm.name); - - // Serialize against every other mapping change on this same file; the - // read-modify-write below is only atomic while this is held. - let lock = hosts_file_lock(docker_container); - let _guard = lock.lock().unwrap_or_else(PoisonError::into_inner); - if let Some(container) = docker_container { // container-targeted: the resolver that needs this name lives inside // the container, so write only there and leave the host's file alone. - let cat = std::process::Command::new("docker") - .args(["exec", container, "cat", path]) - .output() - .handle_err(location!())?; - // Bail rather than write on a failed read: `output()` is Ok even when - // the exec itself failed (container gone, docker hiccup), and treating - // the empty stdout as the file's contents would truncate a live - // `/etc/hosts` down to this one entry. - if !cat.status.success() { - Err(format!( - "reading {path} in '{container}' failed: {}", - String::from_utf8_lossy(&cat.stderr).trim() - )) - .handle_err(location!())?; - } - let content = upsert_hosts_entry(&String::from_utf8_lossy(&cat.stdout), &hm.name, &entry); - let mut child = std::process::Command::new("docker") - .args([ - "exec", - "-i", - container, - "sh", - "-c", - &format!("cat > {path}"), - ]) - .stdin(std::process::Stdio::piped()) - .spawn() - .handle_err(location!())?; - if let Some(mut stdin) = child.stdin.take() { - use std::io::Write; - stdin - .write_all(content.as_bytes()) - .handle_err(location!())?; - } - let _ = child.wait(); + // Locking, the crash-recovery marker, and the failed-read guard all + // live inside this shared helper now — see `host_mappings.rs`. + host_mappings::upsert_container_host_entry(container, &hm.name, &hm.ip) } else { // host-targeted: upsert into the host's /etc/hosts + let path = "/etc/hosts"; + // Serialize against every other mapping change on this same file; + // the read-modify-write below is only atomic while this is held. + let lock = hosts_file_lock(None); + let _guard = lock.lock().unwrap_or_else(PoisonError::into_inner); + // Marked so a restarted process can sweep its predecessor's entries + // without touching the operator's (see `host_mappings::purge_stale_mappings`). + let entry = format!("{} {} {HOSTS_MARKER}", hm.ip, hm.name); let content = std::fs::read_to_string(path).handle_err(location!())?; - std::fs::write(path, upsert_hosts_entry(&content, &hm.name, &entry)) - .handle_err(location!())?; - } - - Ok(()) -} - -fn upsert_hosts_entry(content: &str, name: &str, entry: &str) -> String { - let mut lines: Vec = content.lines().map(ToString::to_string).collect(); - let mut found = false; - for line in &mut lines { - if line.split_whitespace().skip(1).any(|tok| tok == name) { - *line = entry.to_string(); - found = true; - } - } - if !found { - lines.push(entry.to_string()); + std::fs::write( + path, + host_mappings::upsert_hosts_entry(&content, &hm.name, &entry), + ) + .handle_err(location!()) } - lines.join("\n") + "\n" } fn remove_host_mapping(hm: &HostMapping, docker_container: Option<&str>) -> Result<(), Error> { - let path = "/etc/hosts"; - - let lock = hosts_file_lock(docker_container); - let _guard = lock.lock().unwrap_or_else(PoisonError::into_inner); - if let Some(container) = docker_container { // container-targeted: setup only wrote inside the container, so the - // matching removal is container-only too. - let cat = std::process::Command::new("docker") - .args(["exec", container, "cat", path]) - .output() - .handle_err(location!())?; - if !cat.status.success() { - Err(format!( - "reading {path} in '{container}' failed: {}", - String::from_utf8_lossy(&cat.stderr).trim() - )) - .handle_err(location!())?; - } - let content = remove_hosts_entry(&String::from_utf8_lossy(&cat.stdout), &hm.name, &hm.ip); - let mut child = std::process::Command::new("docker") - .args([ - "exec", - "-i", - container, - "sh", - "-c", - &format!("cat > {path}"), - ]) - .stdin(std::process::Stdio::piped()) - .spawn() - .handle_err(location!())?; - if let Some(mut stdin) = child.stdin.take() { - use std::io::Write; - stdin - .write_all(content.as_bytes()) - .handle_err(location!())?; - } - let _ = child.wait(); + // matching removal is container-only too. Locking and the IP-scoped + // match both live inside this shared helper now — see `host_mappings.rs`. + host_mappings::remove_container_host_entry(container, &hm.name, &hm.ip) } else { // host-targeted: drop this net's line from the host file + let path = "/etc/hosts"; + let lock = hosts_file_lock(None); + let _guard = lock.lock().unwrap_or_else(PoisonError::into_inner); let content = std::fs::read_to_string(path).handle_err(location!())?; - std::fs::write(path, remove_hosts_entry(&content, &hm.name, &hm.ip)) - .handle_err(location!())?; + std::fs::write( + path, + host_mappings::remove_hosts_entry(&content, &hm.name, &hm.ip), + ) + .handle_err(location!()) } - - Ok(()) -} - -/// Drop the line mapping `name`, but only while it still points at `ip`. -/// -/// NET IDs are recycled, so a teardown can land after a *newer* net has already -/// re-installed the same name at a different overlay IP (`upsert_hosts_entry` -/// keys on the name alone, which is what makes the replacement correct). -/// Matching the IP too makes the late teardown a no-op instead of deleting a -/// mapping that belongs to a live tunnel. -fn remove_hosts_entry(content: &str, name: &str, ip: &str) -> String { - let lines: Vec = content - .lines() - .filter(|line| { - let mut tokens = line.split_whitespace(); - let line_ip = tokens.next(); - !(line_ip == Some(ip) && tokens.any(|tok| tok == name)) - }) - .map(ToString::to_string) - .collect(); - lines.join("\n") + "\n" } /// Lowercase hex encoding, used to pass the tunnel's AES key to diff --git a/members/nullnet-client/src/host_mappings.rs b/members/nullnet-client/src/host_mappings.rs index 7c9e909..52cd57c 100644 --- a/members/nullnet-client/src/host_mappings.rs +++ b/members/nullnet-client/src/host_mappings.rs @@ -1,4 +1,5 @@ use nullnet_grpc_lib::nullnet_grpc::HostMapping; +use nullnet_liberror::{Error, ErrorHandler, Location, location}; use std::collections::HashMap; use std::process::Command; use std::sync::{Arc, LazyLock, Mutex, PoisonError}; @@ -100,7 +101,7 @@ pub fn purge_stale_mappings() { continue; }; let cleaned = strip_marked(&content); - if cleaned != content && write_container_hosts(&container, &cleaned) { + if cleaned != content && write_container_hosts(&container, &cleaned).is_ok() { swept += 1; } } @@ -136,7 +137,8 @@ fn running_containers() -> Vec { } /// `None` when the read failed — never an empty string, which would truncate -/// the file on write-back (the same trap `add_host_mapping` guards against). +/// the file on write-back (the same trap `upsert_container_host_entry` guards +/// against). fn container_hosts(container: &str) -> Option { let out = Command::new("docker") .args(["exec", container, "cat", HOSTS_PATH]) @@ -147,9 +149,68 @@ fn container_hosts(container: &str) -> Option { .then(|| String::from_utf8_lossy(&out.stdout).into_owned()) } -fn write_container_hosts(container: &str, content: &str) -> bool { - use std::io::Write; - let Ok(mut child) = Command::new("docker") +/// Upsert `name -> ip` into `container`'s own `/etc/hosts` via `docker exec`, +/// tagged with [`HOSTS_MARKER`] so a restarted process can sweep it later. +/// Shared by two callers: the reactive real-mapping write once a tunnel is up +/// (`control_channel`'s `add_host_mapping`) and the proactive placeholder seed +/// written before any packet exists (`placeholder::seed_placeholder`) — both +/// need identical docker-exec read/upsert/write mechanics, a different `ip` +/// value, and the same crash-safety (locking, tagging). +pub(crate) fn upsert_container_host_entry( + container: &str, + name: &str, + ip: &str, +) -> Result<(), Error> { + let lock = hosts_file_lock(Some(container)); + let _guard = lock.lock().unwrap_or_else(PoisonError::into_inner); + + let entry = format!("{ip} {name} {HOSTS_MARKER}"); + let cat = Command::new("docker") + .args(["exec", container, "cat", HOSTS_PATH]) + .output() + .handle_err(location!())?; + // Bail rather than write on a failed read: `output()` is Ok even when the + // exec itself failed (container gone, docker hiccup), and treating the + // empty stdout as the file's contents would truncate a live `/etc/hosts` + // down to this one entry. + if !cat.status.success() { + return Err(format!( + "reading {HOSTS_PATH} in '{container}' failed: {}", + String::from_utf8_lossy(&cat.stderr).trim() + )) + .handle_err(location!()); + } + let content = upsert_hosts_entry(&String::from_utf8_lossy(&cat.stdout), name, &entry); + write_container_hosts(container, &content) +} + +/// Remove `name`'s line from `container`'s own `/etc/hosts` via `docker exec`, +/// but only while it still points at `ip` — see `remove_hosts_entry`. +pub(crate) fn remove_container_host_entry( + container: &str, + name: &str, + ip: &str, +) -> Result<(), Error> { + let lock = hosts_file_lock(Some(container)); + let _guard = lock.lock().unwrap_or_else(PoisonError::into_inner); + + let cat = Command::new("docker") + .args(["exec", container, "cat", HOSTS_PATH]) + .output() + .handle_err(location!())?; + if !cat.status.success() { + return Err(format!( + "reading {HOSTS_PATH} in '{container}' failed: {}", + String::from_utf8_lossy(&cat.stderr).trim() + )) + .handle_err(location!()); + } + let content = remove_hosts_entry(&String::from_utf8_lossy(&cat.stdout), name, ip); + write_container_hosts(container, &content) +} + +fn write_container_hosts(container: &str, content: &str) -> Result<(), Error> { + let mut child = Command::new("docker") .args([ "exec", "-i", @@ -160,15 +221,50 @@ fn write_container_hosts(container: &str, content: &str) -> bool { ]) .stdin(std::process::Stdio::piped()) .spawn() - else { - return false; - }; - if let Some(mut stdin) = child.stdin.take() - && stdin.write_all(content.as_bytes()).is_err() - { - return false; + .handle_err(location!())?; + if let Some(mut stdin) = child.stdin.take() { + use std::io::Write; + stdin + .write_all(content.as_bytes()) + .handle_err(location!())?; + } + let _ = child.wait(); + Ok(()) +} + +pub(crate) fn upsert_hosts_entry(content: &str, name: &str, entry: &str) -> String { + let mut lines: Vec = content.lines().map(ToString::to_string).collect(); + let mut found = false; + for line in &mut lines { + if line.split_whitespace().skip(1).any(|tok| tok == name) { + *line = entry.to_string(); + found = true; + } + } + if !found { + lines.push(entry.to_string()); } - child.wait().map(|s| s.success()).unwrap_or(false) + lines.join("\n") + "\n" +} + +/// Drop the line mapping `name`, but only while it still points at `ip`. +/// +/// NET IDs are recycled, so a teardown can land after a *newer* net has +/// already re-installed the same name at a different overlay IP +/// (`upsert_hosts_entry` keys on the name alone, which is what makes the +/// replacement correct). Matching the IP too makes the late teardown a no-op +/// instead of deleting a mapping that belongs to a live tunnel. +pub(crate) fn remove_hosts_entry(content: &str, name: &str, ip: &str) -> String { + let lines: Vec = content + .lines() + .filter(|line| { + let mut tokens = line.split_whitespace(); + let line_ip = tokens.next(); + !(line_ip == Some(ip) && tokens.any(|tok| tok == name)) + }) + .map(ToString::to_string) + .collect(); + lines.join("\n") + "\n" } #[cfg(test)] @@ -196,3 +292,42 @@ mod tests { assert_eq!(strip_marked(content), content); } } + +#[cfg(test)] +mod hosts_entry_tests { + use super::*; + + #[test] + fn upsert_appends_when_absent() { + let out = upsert_hosts_entry("127.0.0.1 localhost\n", "redis", "203.0.113.7 redis"); + assert_eq!(out, "127.0.0.1 localhost\n203.0.113.7 redis\n"); + } + + #[test] + fn upsert_replaces_existing_line() { + let out = upsert_hosts_entry( + "127.0.0.1 localhost\n203.0.113.7 redis\n", + "redis", + "10.0.0.5 redis", + ); + assert_eq!(out, "127.0.0.1 localhost\n10.0.0.5 redis\n"); + } + + #[test] + fn remove_drops_matching_line_only() { + let out = remove_hosts_entry( + "127.0.0.1 localhost\n10.0.0.5 redis\n10.0.0.6 billing\n", + "redis", + "10.0.0.5", + ); + assert_eq!(out, "127.0.0.1 localhost\n10.0.0.6 billing\n"); + } + + #[test] + fn remove_is_a_noop_when_ip_no_longer_matches() { + // A late teardown for a torn-down tunnel landing after a newer tunnel + // re-mapped the same name at a different IP must not delete it. + let out = remove_hosts_entry("127.0.0.1 localhost\n10.0.0.9 redis\n", "redis", "10.0.0.5"); + assert_eq!(out, "127.0.0.1 localhost\n10.0.0.9 redis\n"); + } +} diff --git a/members/nullnet-client/src/main.rs b/members/nullnet-client/src/main.rs index 05af4ee..4db09fe 100644 --- a/members/nullnet-client/src/main.rs +++ b/members/nullnet-client/src/main.rs @@ -41,6 +41,7 @@ mod host_mappings; mod local_endpoints; mod nfqueue; mod peers; +mod placeholder; mod triggers; pub const FORWARD_PORT: u16 = 9999; @@ -161,7 +162,8 @@ async fn main() -> Result<(), Error> { // packet of each new watched-port flow, listener fires backend_trigger // with the resolved initiator container, waits for VxlanSetup to install // DNAT, then verdicts ACCEPT so the original packet hits the new chain. - let (config_tx, config_rx) = tokio::sync::mpsc::unbounded_channel::>(); + let (config_tx, config_rx) = + tokio::sync::mpsc::unbounded_channel::>>(); // Poked by the cache's docker-events watcher after every container // start/die so `declare_services` re-runs immediately instead of @@ -181,7 +183,7 @@ async fn main() -> Result<(), Error> { policy_verdicts, ); - // declare services + push the port→service map to the NFQUEUE listener + // declare services + push the port→target map to the NFQUEUE listener // on each refresh. tokio::spawn(async move { declare_services(grpc_server, config_tx, docker_changed) @@ -291,7 +293,7 @@ async fn grpc_init() -> Result { async fn declare_services( grpc_server: NullnetGrpcInterface, - config_tx: UnboundedSender>, + config_tx: UnboundedSender>>, docker_changed: Arc, ) -> Result<(), Error> { let mut last_snapshot: Vec = Vec::new(); @@ -370,17 +372,45 @@ async fn declare_services( }); } - let mut port_to_service: HashMap = HashMap::new(); + // More than one target can share a port (two dependencies + // reached on the same real port, e.g. two plain-HTTPS deps + // both 443) — group by port so the listener can disambiguate + // by destination address at trigger time. + let mut port_to_target: HashMap> = HashMap::new(); for st in response.service_triggers { - for port in st.ports { - let Ok(port) = u16::try_from(port) else { - eprintln!("server returned invalid trigger port {port}; skipping"); + let initiator_container = (!st.initiator_container.is_empty()) + .then_some(st.initiator_container.as_str()); + for tp in st.trigger_ports { + let Ok(port) = u16::try_from(tp.port) else { + eprintln!("server returned invalid trigger port {}; skipping", tp.port); continue; }; - port_to_service.insert(port, st.service_name.clone()); + // Pre-seed a placeholder /etc/hosts entry for the + // dependency's literal name *before* any packet is + // observed, so a bare container name (not just a + // pre-provisioned DNS alias) produces a real first + // packet for NFQUEUE to catch. Idempotent — safe on + // every reconcile pass; self-heals across container + // restarts (Docker wipes /etc/hosts on every start). + if let Some(container) = initiator_container + && let Err(e) = + placeholder::seed_placeholder(container, &tp.target_name) + { + eprintln!( + "failed to seed placeholder for '{}' in {container}: {e:?}", + tp.target_name + ); + } + port_to_target + .entry(port) + .or_default() + .push(nfqueue::TriggerTarget { + service_name: st.service_name.clone(), + target_name: tp.target_name.clone(), + }); } } - if config_tx.send(port_to_service).is_err() { + if config_tx.send(port_to_target).is_err() { // observer task gone; nothing more to do here return Ok(()); } diff --git a/members/nullnet-client/src/nfqueue/cache.rs b/members/nullnet-client/src/nfqueue/cache.rs index 5fed8e2..d18de1a 100644 --- a/members/nullnet-client/src/nfqueue/cache.rs +++ b/members/nullnet-client/src/nfqueue/cache.rs @@ -1,3 +1,4 @@ +use crate::triggers::TriggersState; use std::collections::{HashMap, HashSet}; use std::net::Ipv4Addr; use std::process::Stdio; @@ -280,15 +281,22 @@ fn strip_cidr_to_ipv4(ip_with_mask: &str) -> Option { } /// Spawn the long-running `docker events` watcher. Triggers a refresh after -/// every container start/die and pokes `docker_changed` so the -/// declare-services loop in `main` re-runs immediately. If docker isn't -/// installed or the subprocess can't be spawned, the task logs and exits — -/// listener falls back to the initial cache snapshot. Restarts the -/// subprocess on unexpected exit. -pub fn spawn_events_watcher(cache: BridgeIpCache, docker_changed: Arc) { +/// every container start/die, purges any stale trigger state for that +/// specific container (see `forget_container`'s doc comment — a restart or +/// recreate keeps the container's name but gets a new sandbox, so a stale +/// `TriggersState` entry would otherwise wrongly look `Active` forever), and +/// pokes `docker_changed` so the declare-services loop in `main` re-runs +/// immediately. If docker isn't installed or the subprocess can't be +/// spawned, the task logs and exits — listener falls back to the initial +/// cache snapshot. Restarts the subprocess on unexpected exit. +pub fn spawn_events_watcher( + cache: BridgeIpCache, + docker_changed: Arc, + triggers_state: Arc, +) { tokio::spawn(async move { loop { - if let Err(e) = run_events_loop(&cache, &docker_changed).await { + if let Err(e) = run_events_loop(&cache, &docker_changed, &triggers_state).await { eprintln!("[nfqueue/cache] events watcher: {e}; restarting in 5s"); tokio::time::sleep(std::time::Duration::from_secs(5)).await; } @@ -296,7 +304,11 @@ pub fn spawn_events_watcher(cache: BridgeIpCache, docker_changed: Arc) { }); } -async fn run_events_loop(cache: &BridgeIpCache, docker_changed: &Notify) -> Result<(), String> { +async fn run_events_loop( + cache: &BridgeIpCache, + docker_changed: &Notify, + triggers_state: &TriggersState, +) -> Result<(), String> { let mut child = tokio::process::Command::new("docker") .args([ "events", @@ -308,9 +320,12 @@ async fn run_events_loop(cache: &BridgeIpCache, docker_changed: &Notify) -> Resu "event=die", // `.Action` (start/die) is the modern field; the legacy // `.Status` was removed in newer daemons (template eval errors - // "can't evaluate field Status in type *events.Message"). + // "can't evaluate field Status in type *events.Message"). The + // container name is needed (not just the action) so a + // restart/recreate can purge that one container's stale + // TriggersState entries instead of guessing which one changed. "--format", - "{{.Action}}", + "{{.Action}} {{.Actor.Attributes.name}}", ]) .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -342,10 +357,16 @@ async fn run_events_loop(cache: &BridgeIpCache, docker_changed: &Notify) -> Resu .await .map_err(|e| format!("read docker events: {e}"))? { - // We don't parse the line — any container start/die warrants a - // full refresh. Cheap enough: a few processes per event. + // Beyond action+name we don't otherwise parse the line — any + // container start/die warrants a full cache refresh. Cheap enough: + // a few processes per event. println!("[nfqueue/cache] docker event: {line} — refreshing"); cache.refresh().await; + if let Some((_, name)) = line.split_once(' ') + && !name.is_empty() + { + triggers_state.forget_container(name); + } // Cache now reflects the new task; kick the declare-services // loop so the ipset catches up before the new task dials. docker_changed.notify_one(); diff --git a/members/nullnet-client/src/nfqueue/egress_listener.rs b/members/nullnet-client/src/nfqueue/egress_listener.rs index cf98db4..1071ef1 100644 --- a/members/nullnet-client/src/nfqueue/egress_listener.rs +++ b/members/nullnet-client/src/nfqueue/egress_listener.rs @@ -19,7 +19,7 @@ use crate::egress_policy::PolicyVerdicts; use crate::nfqueue::cache::BridgeIpCache; use crate::nfqueue::parse::ipv4_flow; use crate::nfqueue::recv_loop::spawn_queue_loop; -use crate::triggers::{EGRESS_TRIGGER_PORT, TriggerState, TriggersState}; +use crate::triggers::{EGRESS_DST_IP, EGRESS_TRIGGER_PORT, TriggerState, TriggersState}; use nfq::{Message, Verdict}; use nullnet_grpc_lib::NullnetGrpcInterface; use nullnet_grpc_lib::nullnet_grpc::{ @@ -157,13 +157,19 @@ async fn decide_verdict(ctx: &EgressCtx, flow: Option<(Ipv4Addr, Ipv4Addr, u16)> return Verdict::Drop; } - match ctx.triggers_state.state(&container, EGRESS_TRIGGER_PORT) { + match ctx + .triggers_state + .state(&container, EGRESS_DST_IP, EGRESS_TRIGGER_PORT) + { TriggerState::Active => Verdict::Accept, TriggerState::Pending(notify) => wait_for_steer(ctx, &container, notify).await, TriggerState::Fresh => { - let notify = ctx - .triggers_state - .mark_pending(&container, EGRESS_TRIGGER_PORT, src_ip); + let notify = ctx.triggers_state.mark_pending( + &container, + EGRESS_DST_IP, + EGRESS_TRIGGER_PORT, + src_ip, + ); // Register the waiter BEFORE the gRPC round-trip: the server can // dispatch the egress `VxlanSetup` (→ `mark_active`) faster than its // reply to `egress_trigger` returns, and `notify_waiters` only wakes @@ -172,7 +178,8 @@ async fn decide_verdict(ctx: &EgressCtx, flow: Option<(Ipv4Addr, Ipv4Addr, u16)> tokio::pin!(notified); if notified.as_mut().enable() || matches!( - ctx.triggers_state.state(&container, EGRESS_TRIGGER_PORT), + ctx.triggers_state + .state(&container, EGRESS_DST_IP, EGRESS_TRIGGER_PORT), TriggerState::Active ) { @@ -202,7 +209,8 @@ async fn decide_verdict(ctx: &EgressCtx, flow: Option<(Ipv4Addr, Ipv4Addr, u16)> Ok(Err(e)) => { eprintln!("[egress-nfq] egress_trigger {container}: {e}"); report_trigger_send_failed(&ctx.grpc, &container, dst_ip, dst_port, e); - ctx.triggers_state.forget(&container, EGRESS_TRIGGER_PORT); + ctx.triggers_state + .forget(&container, EGRESS_DST_IP, EGRESS_TRIGGER_PORT); Verdict::Drop } Err(_) => { @@ -214,7 +222,8 @@ async fn decide_verdict(ctx: &EgressCtx, flow: Option<(Ipv4Addr, Ipv4Addr, u16)> dst_port, format!("egress_trigger timed out after {TRIGGER_TIMEOUT:?}"), ); - ctx.triggers_state.forget(&container, EGRESS_TRIGGER_PORT); + ctx.triggers_state + .forget(&container, EGRESS_DST_IP, EGRESS_TRIGGER_PORT); Verdict::Drop } } @@ -261,7 +270,8 @@ async fn wait_for_steer(ctx: &EgressCtx, container: &str, notify: Arc) - tokio::pin!(notified); if notified.as_mut().enable() || matches!( - ctx.triggers_state.state(container, EGRESS_TRIGGER_PORT), + ctx.triggers_state + .state(container, EGRESS_DST_IP, EGRESS_TRIGGER_PORT), TriggerState::Active ) { diff --git a/members/nullnet-client/src/nfqueue/listener.rs b/members/nullnet-client/src/nfqueue/listener.rs index 56694ed..a3718f1 100644 --- a/members/nullnet-client/src/nfqueue/listener.rs +++ b/members/nullnet-client/src/nfqueue/listener.rs @@ -1,5 +1,5 @@ use crate::nfqueue::cache::BridgeIpCache; -use crate::nfqueue::parse::ipv4_src_and_dst_port; +use crate::nfqueue::parse::ipv4_flow; use crate::nfqueue::recv_loop::spawn_queue_loop; use crate::triggers::{TriggerState, TriggersState}; use nfq::{Message, Verdict}; @@ -8,6 +8,7 @@ use nullnet_grpc_lib::nullnet_grpc::{ AgentBackendTriggerSendFailed, AgentEvent, agent_event::Event as AgentEventKind, }; use std::collections::HashMap; +use std::net::Ipv4Addr; use std::sync::mpsc::Sender; use std::sync::{Arc, RwLock}; use std::time::Duration; @@ -32,16 +33,49 @@ const TRIGGER_TIMEOUT: Duration = Duration::from_secs(5); /// giving up on the held packet. const ACTIVE_TIMEOUT: Duration = Duration::from_secs(5); +/// What a watched trigger port is associated with: the declaring (initiator) +/// service, for reporting, and the literal name its chain[0] resolves to — +/// what `placeholder::seed_placeholder` pre-seeds into the initiator +/// container's `/etc/hosts`, and what disambiguates two dependencies that +/// happen to share a port (via their distinct placeholder addresses). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TriggerTarget { + pub service_name: String, + pub target_name: String, +} + /// State shared by every per-packet handler. Cloned freely across tokio tasks. #[derive(Clone)] pub struct ListenerCtx { pub grpc: NullnetGrpcInterface, pub cache: BridgeIpCache, - pub port_to_service: Arc>>, + /// More than one target can share a port (two dependencies reached on + /// the same real port); `resolve_target` picks the right one by + /// matching the packet's observed destination against each candidate's + /// deterministic placeholder address. + pub port_to_target: Arc>>>, pub triggers_state: Arc, pub semaphore: Arc, } +/// Pick the target this packet's destination actually belongs to. +/// +/// A single candidate is used unconditionally — this is the common case, +/// and also covers a destination that doesn't (yet) match the computed +/// placeholder (e.g. the very first packet on a freshly re-seeded name, or a +/// hardcoded IP). With more than one candidate sharing this port, an exact +/// match against each candidate's own placeholder address is required — +/// there is no safe default to fall back on, since guessing wrong would +/// route the packet into the wrong tunnel. +fn resolve_target(targets: &[TriggerTarget], dst_ip: Ipv4Addr) -> Option<&TriggerTarget> { + match targets { + [only] => Some(only), + many => many + .iter() + .find(|t| crate::placeholder::ip_for(&t.target_name) == dst_ip), + } +} + /// Spawn the backend-trigger recv loop (queue 0). Each packet is held until /// `handle_packet` resolves a verdict — see `recv_loop::spawn_queue_loop`. pub fn spawn_recv_thread(ctx: ListenerCtx) { @@ -70,7 +104,7 @@ async fn handle_packet(mut msg: Message, ctx: ListenerCtx, verdict_tx: Sender Verdict { - match ctx.triggers_state.state(container, dst_port) { + let service = target.service_name.as_str(); + match ctx.triggers_state.state(container, dst_ip, dst_port) { TriggerState::Active => Verdict::Accept, TriggerState::Pending(notify) => { // `mark_active` wakes us with `Notify::notify_waiters()`, which @@ -123,7 +172,7 @@ async fn decide_verdict( tokio::pin!(notified); if notified.as_mut().enable() || matches!( - ctx.triggers_state.state(container, dst_port), + ctx.triggers_state.state(container, dst_ip, dst_port), TriggerState::Active ) { @@ -140,7 +189,9 @@ async fn decide_verdict( } } TriggerState::Fresh => { - let notify = ctx.triggers_state.mark_pending(container, dst_port, src_ip); + let notify = ctx + .triggers_state + .mark_pending(container, dst_ip, dst_port, src_ip); // Register BEFORE the gRPC round-trip: the server can dispatch // `VxlanSetup` (→ `mark_active` here) faster than its reply to // `backend_trigger` arrives back, especially on multi-edge @@ -151,7 +202,7 @@ async fn decide_verdict( tokio::pin!(notified); if notified.as_mut().enable() || matches!( - ctx.triggers_state.state(container, dst_port), + ctx.triggers_state.state(container, dst_ip, dst_port), TriggerState::Active ) { @@ -163,6 +214,7 @@ async fn decide_verdict( service.to_string(), u32::from(dst_port), container.to_string(), + target.target_name.clone(), ), ) .await; @@ -186,7 +238,7 @@ async fn decide_verdict( "[nfqueue] backend_trigger '{service}' port {dst_port} container {container}: {e}" ); report_trigger_send_failed(&ctx.grpc, service, dst_port, e); - ctx.triggers_state.forget(container, dst_port); + ctx.triggers_state.forget(container, dst_ip, dst_port); Verdict::Drop } Err(_) => { @@ -199,7 +251,7 @@ async fn decide_verdict( dst_port, format!("backend_trigger timed out after {TRIGGER_TIMEOUT:?}"), ); - ctx.triggers_state.forget(container, dst_port); + ctx.triggers_state.forget(container, dst_ip, dst_port); Verdict::Drop } } diff --git a/members/nullnet-client/src/nfqueue/mod.rs b/members/nullnet-client/src/nfqueue/mod.rs index f7a7cdd..2991efe 100644 --- a/members/nullnet-client/src/nfqueue/mod.rs +++ b/members/nullnet-client/src/nfqueue/mod.rs @@ -5,6 +5,7 @@ mod parse; mod recv_loop; pub use cache::BridgeIpCache; +pub use listener::TriggerTarget; use crate::commands::nfqueue as rules; use crate::egress_policy::PolicyVerdicts; @@ -23,8 +24,8 @@ use tokio::sync::mpsc::UnboundedReceiver; /// - Populates the bridge-IP → container-name cache from `docker inspect` /// and keeps it fresh via a `docker events` watcher. /// - Consumes `config_rx` (driven by the services-list refresh in `main`) to -/// keep the kernel ipset in sync and to maintain a port → service lookup -/// for the per-packet handler. +/// keep the kernel ipset in sync and to maintain a port → trigger-target +/// lookup for the per-packet handler. /// - Spawns the recv thread that owns the netfilter queue. Each packet is /// handed off to a tokio task; the recv thread drains verdicts in lockstep /// so packets release back into the netfilter pipeline. @@ -34,12 +35,13 @@ use tokio::sync::mpsc::UnboundedReceiver; pub fn spawn_listener( grpc: NullnetGrpcInterface, triggers_state: Arc, - config_rx: UnboundedReceiver>, + config_rx: UnboundedReceiver>>, docker_changed: Arc, cache: BridgeIpCache, verdicts: Arc, ) { - let port_to_service: Arc>> = Arc::new(RwLock::new(HashMap::new())); + let port_to_target: Arc>>> = + Arc::new(RwLock::new(HashMap::new())); // Initial cache populate + long-running docker-events watcher. The // watcher pings `docker_changed` after every refresh so the @@ -48,20 +50,21 @@ pub fn spawn_listener( // might fire a SYN before its trigger port is being watched. { let bridge_cache = cache.clone(); + let triggers_state_for_events = triggers_state.clone(); tokio::spawn(async move { bridge_cache.refresh().await; - cache::spawn_events_watcher(bridge_cache, docker_changed); + cache::spawn_events_watcher(bridge_cache, docker_changed, triggers_state_for_events); }); } - // Config consumer: each services-list refresh produces a port→service + // Config consumer: each services-list refresh produces a port→target // map. We diff vs the previous, push the diff to the ipset (so the // kernel knows which ports to queue), then atomically replace our - // userspace port→service lookup that the handler reads. + // userspace port→target lookup that the handler reads. { - let port_to_service = port_to_service.clone(); + let port_to_target = port_to_target.clone(); tokio::spawn(async move { - consume_config(config_rx, port_to_service).await; + consume_config(config_rx, port_to_target).await; }); } @@ -77,7 +80,7 @@ pub fn spawn_listener( let ctx = ListenerCtx { grpc, cache, - port_to_service, + port_to_target, triggers_state, semaphore: Arc::new(Semaphore::new(HANDLER_CONCURRENCY)), }; @@ -85,8 +88,8 @@ pub fn spawn_listener( } async fn consume_config( - mut config_rx: UnboundedReceiver>, - port_to_service: Arc>>, + mut config_rx: UnboundedReceiver>>, + port_to_target: Arc>>>, ) { let mut current_ports: HashSet = HashSet::new(); while let Some(new_map) = config_rx.recv().await { @@ -94,7 +97,7 @@ async fn consume_config( rules::apply_ports_diff(¤t_ports, &new_ports); // Swap the lookup. Sync RwLock; write is brief, never held across // an `.await`. - *port_to_service.write().unwrap() = new_map; + *port_to_target.write().unwrap() = new_map; current_ports = new_ports; } } diff --git a/members/nullnet-client/src/nfqueue/parse.rs b/members/nullnet-client/src/nfqueue/parse.rs index f618d5f..cd9aedf 100644 --- a/members/nullnet-client/src/nfqueue/parse.rs +++ b/members/nullnet-client/src/nfqueue/parse.rs @@ -2,25 +2,12 @@ use etherparse::{LaxPacketHeaders, NetHeaders, TransportHeader}; use std::net::Ipv4Addr; /// NFQUEUE delivers L3 (no Ethernet) IPv4 packets to userspace. Extract the -/// source IP and the L4 destination port for TCP and UDP. Returns `None` for -/// non-IPv4, non-TCP/UDP, fragmented, or malformed packets. -pub fn ipv4_src_and_dst_port(packet: &[u8]) -> Option<(Ipv4Addr, u16)> { - let headers = LaxPacketHeaders::from_ip(packet).ok()?; - let src_octets = match headers.net? { - NetHeaders::Ipv4(ipv4, _) => ipv4.source, - _ => return None, - }; - let dst_port = match headers.transport? { - TransportHeader::Tcp(tcp) => tcp.destination_port, - TransportHeader::Udp(udp) => udp.destination_port, - _ => return None, - }; - Some((Ipv4Addr::from(src_octets), dst_port)) -} - -/// Like [`ipv4_src_and_dst_port`] but also returns the destination IP. Used by -/// the egress listener, which classifies flows by destination (external vs -/// internal) rather than by port. +/// source IP, destination IP, and L4 destination port for TCP and UDP. +/// Returns `None` for non-IPv4, non-TCP/UDP, fragmented, or malformed +/// packets. The destination IP is what lets the backend-trigger listener +/// disambiguate two dependencies sharing a port (via their distinct +/// placeholder addresses, see `placeholder.rs`) and what the egress listener +/// uses to classify flows (external vs internal). pub fn ipv4_flow(packet: &[u8]) -> Option<(Ipv4Addr, Ipv4Addr, u16)> { let headers = LaxPacketHeaders::from_ip(packet).ok()?; let (src_octets, dst_octets) = match headers.net? { @@ -83,8 +70,8 @@ mod tests { 80, ); assert_eq!( - ipv4_src_and_dst_port(&pkt), - Some((Ipv4Addr::new(172, 17, 0, 5), 80)) + ipv4_flow(&pkt), + Some((Ipv4Addr::new(172, 17, 0, 5), Ipv4Addr::new(10, 0, 0, 1), 80)) ); } @@ -97,14 +84,14 @@ mod tests { 53, ); assert_eq!( - ipv4_src_and_dst_port(&pkt), - Some((Ipv4Addr::new(172, 17, 0, 6), 53)) + ipv4_flow(&pkt), + Some((Ipv4Addr::new(172, 17, 0, 6), Ipv4Addr::new(10, 0, 0, 2), 53)) ); } #[test] fn rejects_too_short() { - assert!(ipv4_src_and_dst_port(&[0x45; 10]).is_none()); + assert!(ipv4_flow(&[0x45; 10]).is_none()); } #[test] @@ -114,7 +101,7 @@ mod tests { let mut buf = vec![0u8; 40]; buf[0] = 0x60; buf[6] = 59; // No-Next-Header so etherparse stops cleanly - assert!(ipv4_src_and_dst_port(&buf).is_none()); + assert!(ipv4_flow(&buf).is_none()); } #[test] @@ -124,6 +111,6 @@ mod tests { buf[0] = 0x45; buf[2..4].copy_from_slice(&total_len); buf[9] = 1; // ICMP — not TCP/UDP - assert!(ipv4_src_and_dst_port(&buf).is_none()); + assert!(ipv4_flow(&buf).is_none()); } } diff --git a/members/nullnet-client/src/placeholder.rs b/members/nullnet-client/src/placeholder.rs new file mode 100644 index 0000000..667c7db --- /dev/null +++ b/members/nullnet-client/src/placeholder.rs @@ -0,0 +1,192 @@ +use crate::host_mappings; +use nullnet_liberror::Error; +use std::net::Ipv4Addr; + +/// Default placeholder range: 203.0.113.0/24 (RFC 5737 TEST-NET-3) — +/// reserved for documentation, never assigned to a real host, never on-link +/// for a container's own subnet. A container's routing table has no direct +/// route for it, so any address in this range falls through to the +/// container's default gateway — the path NFQUEUE's `mangle PREROUTING` +/// hook sits on — instead of being resolved on-link. +const DEFAULT_CIDR_BASE: [u8; 3] = [203, 0, 113]; + +/// Env override for the placeholder range, in case an operator's environment +/// does something unusual with the default block. Only the /24's network +/// address matters here; the last octet is always derived per-name. +const CIDR_ENV_VAR: &str = "TRIGGER_PLACEHOLDER_CIDR"; + +/// Deterministic placeholder address for `name`, distinct per name so two +/// backend-trigger dependencies sharing a port (see `triggers::TriggersState`'s +/// `dst_ip`-widened key) still disambiguate by destination address alone. +/// Stable across restarts — this is a pure function of `name`, not a stored +/// allocation, so the client and (once it knows the same name) the +/// control-channel setup/teardown paths always agree on it independently. +/// The last octet comes from `nullnet_grpc_lib::last_octet_for`, shared with +/// nullnet-server so its config-validation can detect a collision before +/// this ever runs. +pub(crate) fn ip_for(name: &str) -> Ipv4Addr { + let [a, b, c] = cidr_base(); + Ipv4Addr::new(a, b, c, nullnet_grpc_lib::last_octet_for(name)) +} + +/// The placeholder block as a CIDR string (e.g. `"203.0.113.0/24"`), for +/// callers that need to treat it as a single internal-ish destination range +/// rather than resolve individual names — see `commands::egress`'s +/// `INTERNAL_RANGES`: a backend-trigger placeholder address is synthetic and +/// never real internet traffic, so it must never be classified as an egress +/// candidate, regardless of which CIDR base is configured. +pub(crate) fn placeholder_cidr() -> String { + let [a, b, c] = cidr_base(); + format!("{a}.{b}.{c}.0/24") +} + +fn cidr_base() -> [u8; 3] { + match std::env::var(CIDR_ENV_VAR) { + Ok(cidr) => parse_cidr_base(&cidr).unwrap_or_else(|| { + eprintln!( + "[placeholder] invalid {CIDR_ENV_VAR} '{cidr}'; falling back to default {DEFAULT_CIDR_BASE:?}" + ); + DEFAULT_CIDR_BASE + }), + Err(_) => DEFAULT_CIDR_BASE, + } +} + +fn parse_cidr_base(cidr: &str) -> Option<[u8; 3]> { + let ip_part = cidr.split('/').next()?; + let octets: Vec = ip_part.split('.').filter_map(|o| o.parse().ok()).collect(); + match octets.as_slice() { + [a, b, c, ..] => Some([*a, *b, *c]), + _ => None, + } +} + +/// Write `name -> ip_for(name)` into `container`'s own `/etc/hosts`, before +/// any packet has been observed on the trigger port it's associated with. +/// Idempotent — safe to call on every declare-services reconcile pass. +pub(crate) fn seed_placeholder(container: &str, name: &str) -> Result<(), Error> { + warn_if_no_default_route(container); + let ip = ip_for(name); + host_mappings::upsert_container_host_entry(container, name, &ip.to_string()) +} + +/// Containers already warned about missing a default route, so the warning +/// prints once per bad spell rather than on every declare-services reconcile +/// pass — and clears once the container recovers, so a later regression (a +/// recreate that drops the primary interface again) warns again instead of +/// staying silent forever. +static WARNED_NO_DEFAULT_ROUTE: std::sync::LazyLock< + std::sync::Mutex>, +> = std::sync::LazyLock::new(|| std::sync::Mutex::new(std::collections::HashSet::new())); + +/// Best-effort check: a container with no default route can never reach an +/// off-link placeholder address (see this module's doc comment on why the +/// placeholder block is deliberately never on-link) — every backend-trigger +/// dial from it is doomed before nullnet is even involved, and the only +/// visible symptom would otherwise be a bare `ENETUNREACH`/`EHOSTUNREACH` in +/// the initiator's own app three layers away, indistinguishable from any +/// other network hiccup. Log loudly instead. A `docker exec` failure here +/// (container gone, docker hiccup, `ip` missing in a minimal image) is not +/// itself something to report — this is diagnostic, not load-bearing. +fn warn_if_no_default_route(container: &str) { + let Ok(out) = std::process::Command::new("docker") + .args(["exec", container, "ip", "route", "show", "default"]) + .output() + else { + return; + }; + if !out.status.success() { + return; + } + let mut warned = WARNED_NO_DEFAULT_ROUTE + .lock() + .unwrap_or_else(|e| e.into_inner()); + if out.stdout.is_empty() { + if warned.insert(container.to_string()) { + eprintln!( + "[placeholder] container '{container}' has no default route — off-link \ + trigger traffic will fail; check its Docker network attachment \ + (missing eth0, exhausted address pool, etc.) before retrying" + ); + } + } else { + warned.remove(container); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deterministic_across_calls() { + assert_eq!(ip_for("redis"), ip_for("redis")); + } + + #[test] + fn distinct_names_get_distinct_addresses() { + assert_ne!(ip_for("auth"), ip_for("billing")); + } + + #[test] + fn stays_within_default_range() { + let ip = ip_for("redis"); + let [a, b, c, d] = ip.octets(); + assert_eq!([a, b, c], DEFAULT_CIDR_BASE); + assert!((1..=254).contains(&d)); + } + + #[test] + fn placeholder_cidr_matches_default_base() { + assert_eq!(placeholder_cidr(), "203.0.113.0/24"); + } + + #[test] + fn parses_cidr_base_from_env_value() { + assert_eq!(parse_cidr_base("198.51.100.0/24"), Some([198, 51, 100])); + assert_eq!(parse_cidr_base("198.51.100.5"), Some([198, 51, 100])); + assert_eq!(parse_cidr_base("not-a-cidr"), None); + } + + #[test] + fn no_default_route_warning_dedups_then_clears_on_recovery() { + let mut warned = WARNED_NO_DEFAULT_ROUTE + .lock() + .unwrap_or_else(|e| e.into_inner()); + warned.clear(); + drop(warned); + + // First sighting of a bad container: newly inserted, warns. + assert!( + WARNED_NO_DEFAULT_ROUTE + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert("flaky".to_string()) + ); + // Same container again while still bad: already present, would not + // re-warn — this is what keeps every reconcile pass from spamming. + assert!( + !WARNED_NO_DEFAULT_ROUTE + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert("flaky".to_string()) + ); + // Recovery clears it, so a later regression warns again instead of + // staying silent forever. + WARNED_NO_DEFAULT_ROUTE + .lock() + .unwrap_or_else(|e| e.into_inner()) + .remove("flaky"); + assert!( + WARNED_NO_DEFAULT_ROUTE + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert("flaky".to_string()) + ); + + WARNED_NO_DEFAULT_ROUTE + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clear(); + } +} diff --git a/members/nullnet-client/src/triggers.rs b/members/nullnet-client/src/triggers.rs index bf28c83..e558ee0 100644 --- a/members/nullnet-client/src/triggers.rs +++ b/members/nullnet-client/src/triggers.rs @@ -16,11 +16,21 @@ const PENDING_TIMEOUT: Duration = Duration::from_secs(10); /// sentinel never collides with them in the shared `TriggersState`. pub const EGRESS_TRIGGER_PORT: u16 = 0; -/// Per-(initiator_container, port) lifecycle. `container_ip` is the bridge IP -/// the NFQUEUE listener observed when the trigger fired; it is carried through -/// to `Active` so DNAT install/remove can match the right `-s` source. Legacy -/// callers that don't know the container IP pass `Ipv4Addr::UNSPECIFIED`, -/// which the dnat module treats as "no source filter". +/// `dst_ip` sentinel for egress triggers, which aren't destination-scoped at +/// all (steering matches every destination for the initiator container). +/// Mirrors `Ipv4Addr::UNSPECIFIED`'s existing "no filter" meaning elsewhere +/// in this codebase (e.g. `dnat`'s `container_ip`). +pub const EGRESS_DST_IP: Ipv4Addr = Ipv4Addr::UNSPECIFIED; + +/// Per-(initiator_container, dst_ip, port) lifecycle. `dst_ip` disambiguates +/// two backend-trigger dependencies that share a port (e.g. two plain-HTTPS +/// deps, both 443) — each target name gets its own deterministic placeholder +/// address (see `placeholder.rs`), so the destination the packet was +/// actually addressed to is enough to tell them apart. `container_ip` is the +/// bridge IP the NFQUEUE listener observed when the trigger fired; it is +/// carried through to `Active` so DNAT install/remove can match the right +/// `-s` source. Legacy callers that don't know the container IP pass +/// `Ipv4Addr::UNSPECIFIED`, which the dnat module treats as "no source filter". pub enum Lifecycle { Pending { since: Instant, @@ -45,18 +55,23 @@ pub enum TriggerState { Active, } +/// Key: `(initiator_container, dst_ip, port)`. `dst_ip` is `EGRESS_DST_IP` +/// for egress entries (not destination-scoped) or the observed/placeholder +/// destination for backend-trigger entries. +type Key = (String, Ipv4Addr, u16); + #[derive(Default)] pub struct TriggersState { - by_key: Mutex>, + by_key: Mutex>, } impl TriggersState { - /// Snapshot the state for `(container, port)`. The lock is dropped before - /// returning so callers can `.await` on the returned `Notify` without - /// holding it. - pub fn state(&self, container: &str, port: u16) -> TriggerState { + /// Snapshot the state for `(container, dst_ip, port)`. The lock is dropped + /// before returning so callers can `.await` on the returned `Notify` + /// without holding it. + pub fn state(&self, container: &str, dst_ip: Ipv4Addr, port: u16) -> TriggerState { let by_key = self.by_key.lock().unwrap(); - match by_key.get(&(container.to_string(), port)) { + match by_key.get(&(container.to_string(), dst_ip, port)) { Some(Lifecycle::Active { .. }) => TriggerState::Active, Some(Lifecycle::Pending { since, notify, .. }) if since.elapsed() < PENDING_TIMEOUT => { TriggerState::Pending(notify.clone()) @@ -68,9 +83,15 @@ impl TriggersState { /// Insert (or refresh) a `Pending` entry and return the `Notify` the /// caller awaits. If an in-flight `Pending` already exists, its `Notify` /// is returned so concurrent fires share one wake-up. - pub fn mark_pending(&self, container: &str, port: u16, container_ip: Ipv4Addr) -> Arc { + pub fn mark_pending( + &self, + container: &str, + dst_ip: Ipv4Addr, + port: u16, + container_ip: Ipv4Addr, + ) -> Arc { let mut by_key = self.by_key.lock().unwrap(); - let key = (container.to_string(), port); + let key = (container.to_string(), dst_ip, port); if let Some(Lifecycle::Pending { since, notify, .. }) = by_key.get(&key) && since.elapsed() < PENDING_TIMEOUT { @@ -94,9 +115,9 @@ impl TriggersState { /// (which wakes on `mark_active`'s `notify_waiters`) finds the DNAT rule /// live by the time it traverses `nat PREROUTING`. Returns /// `Ipv4Addr::UNSPECIFIED` if no entry exists. - pub fn peek_container_ip(&self, container: &str, port: u16) -> Ipv4Addr { + pub fn peek_container_ip(&self, container: &str, dst_ip: Ipv4Addr, port: u16) -> Ipv4Addr { let by_key = self.by_key.lock().unwrap(); - match by_key.get(&(container.to_string(), port)) { + match by_key.get(&(container.to_string(), dst_ip, port)) { Some(Lifecycle::Pending { container_ip, .. }) | Some(Lifecycle::Active { container_ip, .. }) => *container_ip, None => Ipv4Addr::UNSPECIFIED, @@ -110,13 +131,14 @@ impl TriggersState { pub fn mark_active( &self, container: &str, + dst_ip: Ipv4Addr, port: u16, vxlan_id: u32, overlay_ip: Ipv4Addr, container_ip: Ipv4Addr, ) { let mut by_key = self.by_key.lock().unwrap(); - let key = (container.to_string(), port); + let key = (container.to_string(), dst_ip, port); let notify = match by_key.remove(&key) { Some(Lifecycle::Pending { notify, .. } | Lifecycle::Active { notify, .. }) => notify, None => Arc::new(Notify::new()), @@ -136,21 +158,43 @@ impl TriggersState { notify.notify_waiters(); } - /// Drop the entry so the next observed packet on this `(container, port)` retriggers. - pub fn forget(&self, container: &str, port: u16) { + /// Drop the entry so the next observed packet on this + /// `(container, dst_ip, port)` retriggers. + pub fn forget(&self, container: &str, dst_ip: Ipv4Addr, port: u16) { self.by_key .lock() .unwrap() - .remove(&(container.to_string(), port)); + .remove(&(container.to_string(), dst_ip, port)); + } + + /// Drop every entry for `container`, regardless of `dst_ip`/`port`. + /// Called when Docker reports the container restarted or was recreated: + /// its name is stable across a recreate, but the sandbox (and therefore + /// the source IP any installed DNAT's `-s` matched on) is not. A stale + /// `Active` entry would otherwise short-circuit `decide_verdict` straight + /// to `Accept` for the new instance's packets — skipping re-trigger + /// entirely — even though the tunnel it points at no longer matches + /// anything real. This is a coarser hammer than `forget`/`remove_by_vxlan` + /// (no attempt to also tear down whatever DNAT the stale entry implied; + /// the container-side network is being rebuilt anyway), but it's what + /// restores self-healing without a manual `nullnet-client` restart. + pub fn forget_container(&self, container: &str) { + self.by_key + .lock() + .unwrap() + .retain(|(c, ..), _| c != container); } /// Find the `Active` entry for `vxlan_id`, remove it, and return - /// `(container, port, overlay_ip, container_ip)` so the caller can tear - /// down DNAT with the matching `-s`. - pub fn remove_by_vxlan(&self, vxlan_id: u32) -> Option<(String, u16, Ipv4Addr, Ipv4Addr)> { + /// `(container, dst_ip, port, overlay_ip, container_ip)` so the caller + /// can tear down DNAT with the matching `-s`/`-d`. + pub fn remove_by_vxlan( + &self, + vxlan_id: u32, + ) -> Option<(String, Ipv4Addr, u16, Ipv4Addr, Ipv4Addr)> { let mut by_key = self.by_key.lock().unwrap(); - let key = by_key.iter().find_map(|((c, p), lc)| match lc { - Lifecycle::Active { vxlan_id: v, .. } if *v == vxlan_id => Some((c.clone(), *p)), + let key = by_key.iter().find_map(|((c, d, p), lc)| match lc { + Lifecycle::Active { vxlan_id: v, .. } if *v == vxlan_id => Some((c.clone(), *d, *p)), _ => None, })?; // The lock is held across the `iter().find_map` and the `remove` @@ -162,7 +206,7 @@ impl TriggersState { overlay_ip, container_ip, .. - }) => Some((key.0, key.1, overlay_ip, container_ip)), + }) => Some((key.0, key.1, key.2, overlay_ip, container_ip)), Some(Lifecycle::Pending { .. }) | None => { unreachable!("find_map matched Active for {key:?}; lock held across remove") } @@ -179,23 +223,27 @@ mod tests { const IP: Ipv4Addr = Ipv4Addr::new(172, 17, 0, 5); const OVERLAY: Ipv4Addr = Ipv4Addr::new(10, 0, 0, 1); + /// Stand-in placeholder/destination address — same role a real deployment + /// gets from `placeholder::ip_for(target_name)`. + const DST: Ipv4Addr = Ipv4Addr::new(203, 0, 113, 7); + const DST2: Ipv4Addr = Ipv4Addr::new(203, 0, 113, 42); #[tokio::test] async fn pending_then_active_wakes_waiter() { let state = Arc::new(TriggersState::default()); - let notify = state.mark_pending("c1", 80, IP); - assert_eq!(state.peek_container_ip("c1", 80), IP); + let notify = state.mark_pending("c1", DST, 80, IP); + assert_eq!(state.peek_container_ip("c1", DST, 80), IP); let state_clone = state.clone(); let waiter = tokio::spawn(async move { notify.notified().await; - matches!(state_clone.state("c1", 80), TriggerState::Active) + matches!(state_clone.state("c1", DST, 80), TriggerState::Active) }); tokio::time::sleep(Duration::from_millis(20)).await; // The control-channel pattern: peek for install, install DNAT, then // mark_active to wake. peek above already returned IP for the install. - state.mark_active("c1", 80, 42, OVERLAY, IP); + state.mark_active("c1", DST, 80, 42, OVERLAY, IP); let ok = timeout(Duration::from_secs(1), waiter) .await @@ -213,17 +261,17 @@ mod tests { // synchronous state recheck after `.enable()` — it must observe // the Active state set by mark_active and short-circuit the await. let state = Arc::new(TriggersState::default()); - let notify = state.mark_pending("c1", 80, IP); + let notify = state.mark_pending("c1", DST, 80, IP); // mark_active fires BEFORE the listener registers a waiter — this // is exactly the lost-wake race. - state.mark_active("c1", 80, 42, OVERLAY, IP); + state.mark_active("c1", DST, 80, 42, OVERLAY, IP); // The listener's race-protected pattern. let notified = notify.notified(); tokio::pin!(notified); let trip_via_enable = notified.as_mut().enable(); - let trip_via_state = matches!(state.state("c1", 80), TriggerState::Active); + let trip_via_state = matches!(state.state("c1", DST, 80), TriggerState::Active); assert!( trip_via_enable || trip_via_state, "race-fix must observe Active even when mark_active fired before enable" @@ -249,15 +297,18 @@ mod tests { // during `backend_trigger`'s round-trip — after enable, before the // listener resumes awaiting. let state = Arc::new(TriggersState::default()); - let notify = state.mark_pending("c1", 80, IP); + let notify = state.mark_pending("c1", DST, 80, IP); let notified = notify.notified(); tokio::pin!(notified); notified.as_mut().enable(); - assert!(matches!(state.state("c1", 80), TriggerState::Pending(_))); + assert!(matches!( + state.state("c1", DST, 80), + TriggerState::Pending(_) + )); // mark_active fires AFTER enable but before await. - state.mark_active("c1", 80, 42, OVERLAY, IP); + state.mark_active("c1", DST, 80, 42, OVERLAY, IP); let result = timeout(Duration::from_millis(50), notified).await; assert!( @@ -269,44 +320,98 @@ mod tests { #[tokio::test] async fn concurrent_pending_share_notify() { let state = Arc::new(TriggersState::default()); - let n1 = state.mark_pending("c1", 80, IP); - let n2 = state.mark_pending("c1", 80, IP); + let n1 = state.mark_pending("c1", DST, 80, IP); + let n2 = state.mark_pending("c1", DST, 80, IP); assert!(Arc::ptr_eq(&n1, &n2), "same key must reuse the Notify"); } #[tokio::test] async fn distinct_containers_are_independent() { let state = TriggersState::default(); - let _ = state.mark_pending("c1", 80, IP); - let _ = state.mark_pending("c2", 80, Ipv4Addr::new(172, 17, 0, 6)); - assert!(matches!(state.state("c1", 80), TriggerState::Pending(_))); - assert!(matches!(state.state("c2", 80), TriggerState::Pending(_))); - state.mark_active("c1", 80, 7, OVERLAY, IP); - assert!(matches!(state.state("c1", 80), TriggerState::Active)); - assert!(matches!(state.state("c2", 80), TriggerState::Pending(_))); + let _ = state.mark_pending("c1", DST, 80, IP); + let _ = state.mark_pending("c2", DST, 80, Ipv4Addr::new(172, 17, 0, 6)); + assert!(matches!( + state.state("c1", DST, 80), + TriggerState::Pending(_) + )); + assert!(matches!( + state.state("c2", DST, 80), + TriggerState::Pending(_) + )); + state.mark_active("c1", DST, 80, 7, OVERLAY, IP); + assert!(matches!(state.state("c1", DST, 80), TriggerState::Active)); + assert!(matches!( + state.state("c2", DST, 80), + TriggerState::Pending(_) + )); + } + + #[tokio::test] + async fn same_port_different_dst_ip_are_independent() { + // Two backend-trigger dependencies sharing a port (e.g. two + // plain-HTTPS deps, both 443) from the same initiator container — + // disambiguated purely by their distinct placeholder/destination IPs. + let state = TriggersState::default(); + let _ = state.mark_pending("c1", DST, 443, IP); + let _ = state.mark_pending("c1", DST2, 443, IP); + assert!(matches!( + state.state("c1", DST, 443), + TriggerState::Pending(_) + )); + assert!(matches!( + state.state("c1", DST2, 443), + TriggerState::Pending(_) + )); + state.mark_active("c1", DST, 443, 7, OVERLAY, IP); + assert!(matches!(state.state("c1", DST, 443), TriggerState::Active)); + assert!(matches!( + state.state("c1", DST2, 443), + TriggerState::Pending(_) + )); } #[tokio::test] - async fn remove_by_vxlan_returns_container_port_and_ip() { + async fn remove_by_vxlan_returns_container_dst_ip_port_and_ip() { let state = TriggersState::default(); - let _ = state.mark_pending("c1", 80, IP); - state.mark_active("c1", 80, 42, OVERLAY, IP); + let _ = state.mark_pending("c1", DST, 80, IP); + state.mark_active("c1", DST, 80, 42, OVERLAY, IP); let removed = state.remove_by_vxlan(42).expect("entry should exist"); - assert_eq!(removed, ("c1".to_string(), 80, OVERLAY, IP)); - assert!(matches!(state.state("c1", 80), TriggerState::Fresh)); + assert_eq!(removed, ("c1".to_string(), DST, 80, OVERLAY, IP)); + assert!(matches!(state.state("c1", DST, 80), TriggerState::Fresh)); } #[tokio::test] async fn peek_returns_unspecified_when_absent() { let state = TriggersState::default(); - assert_eq!(state.peek_container_ip("c1", 80), Ipv4Addr::UNSPECIFIED); + assert_eq!( + state.peek_container_ip("c1", DST, 80), + Ipv4Addr::UNSPECIFIED + ); } #[tokio::test] async fn forget_drops_entry() { let state = TriggersState::default(); - let _ = state.mark_pending("c1", 80, IP); - state.forget("c1", 80); - assert!(matches!(state.state("c1", 80), TriggerState::Fresh)); + let _ = state.mark_pending("c1", DST, 80, IP); + state.forget("c1", DST, 80); + assert!(matches!(state.state("c1", DST, 80), TriggerState::Fresh)); + } + + #[tokio::test] + async fn forget_container_drops_active_and_pending_across_ports_and_dst_ips() { + let state = TriggersState::default(); + state.mark_active("c1", DST, 80, 7, OVERLAY, IP); + let _ = state.mark_pending("c1", DST2, 443, IP); + let _ = state.mark_pending("c2", DST, 80, IP); + + state.forget_container("c1"); + + assert!(matches!(state.state("c1", DST, 80), TriggerState::Fresh)); + assert!(matches!(state.state("c1", DST2, 443), TriggerState::Fresh)); + // A different container's state is untouched. + assert!(matches!( + state.state("c2", DST, 80), + TriggerState::Pending(_) + )); } } diff --git a/members/nullnet-grpc-lib/proto/nullnet_grpc.proto b/members/nullnet-grpc-lib/proto/nullnet_grpc.proto index 3ec0e78..67265e6 100644 --- a/members/nullnet-grpc-lib/proto/nullnet_grpc.proto +++ b/members/nullnet-grpc-lib/proto/nullnet_grpc.proto @@ -222,15 +222,33 @@ message Listener { } // Response to ServicesList: per-declared-service, the set of trigger ports -// the client should observe via eBPF. When traffic is observed on one of -// these ports, the client fires BackendTrigger(service_name, port). +// the client should observe via NFQUEUE. When traffic is observed on one of +// these ports, the client fires BackendTrigger(service_name, port, target_name). message ServicesListResponse { repeated ServiceTrigger service_triggers = 1; } message ServiceTrigger { string service_name = 1; - repeated uint32 ports = 2; + reserved 2; + reserved "ports"; + repeated TriggerPort trigger_ports = 3; + // Real Docker container name of the local replica hosting this service — + // the container `target_name` (per TriggerPort) should be pre-seeded into. + // Empty when the client couldn't resolve one (host process, no docker + // context); the client skips seeding for those. + string initiator_container = 4; +} + +// One trigger port this service's initiator container should be observed on. +message TriggerPort { + uint32 port = 1; + // chain[0] for this port's dependency chain — the literal name the + // initiator resolves (e.g. a bare Docker container name). Lets the client + // pre-seed a placeholder `/etc/hosts` entry for it before any packet is + // observed, so a bare name (not just a pre-provisioned DNS alias) produces + // a real first packet to trigger on. + string target_name = 2; } message HostMapping { @@ -288,6 +306,11 @@ message BackendTriggerRequest { // sender_ip. With docker, this disambiguates the replica when multiple // replicas of the same service live on the same host. string initiator_container = 3; + // The target_name (see TriggerPort) the observed packet's destination + // placeholder IP resolved back to. Lets the server pick the right chain + // when two of this service's triggers share the same port, instead of + // relying on port alone. + string target_name = 4; } // Egress trigger — fired by the client on the first NEW flow from a registered diff --git a/members/nullnet-grpc-lib/src/lib.rs b/members/nullnet-grpc-lib/src/lib.rs index a0f0d0b..afe2602 100644 --- a/members/nullnet-grpc-lib/src/lib.rs +++ b/members/nullnet-grpc-lib/src/lib.rs @@ -1,4 +1,5 @@ mod control_tls_verifier; +mod placeholder; mod proto; use crate::control_tls_verifier::PinnedCa; @@ -8,6 +9,7 @@ use crate::nullnet_grpc::{ EgressPolicyCheck, EgressTriggerRequest, Empty, IngressPolicyCheck, MsgId, NetMessage, NetType, PortMappingBundle, ProxyRequest, ServiceReport, ServicesListResponse, Upstream, }; +pub use placeholder::last_octet_for; pub use proto::*; use std::path::Path; use tokio::sync::mpsc; @@ -114,6 +116,7 @@ impl NullnetGrpcInterface { service_name: String, port: u32, initiator_container: String, + target_name: String, ) -> Result<(), String> { self.client .clone() @@ -121,6 +124,7 @@ impl NullnetGrpcInterface { service_name, port, initiator_container, + target_name, })) .await .map(|_| ()) diff --git a/members/nullnet-grpc-lib/src/placeholder.rs b/members/nullnet-grpc-lib/src/placeholder.rs new file mode 100644 index 0000000..6016cea --- /dev/null +++ b/members/nullnet-grpc-lib/src/placeholder.rs @@ -0,0 +1,44 @@ +/// Deterministic last octet (1..=254, avoiding the network/broadcast +/// addresses) of a backend-trigger dependency's placeholder address, derived +/// from its name. Shared between nullnet-client (which builds the actual +/// `/etc/hosts` address from it — see its `placeholder.rs`) and +/// nullnet-server (which uses it at config-validation time to catch two +/// distinct chain[0] names on the same port that would land on the same +/// address) so both sides can never disagree on the mapping. +/// +/// Not collision-free — a non-cryptographic hash folded into 254 buckets +/// will alias distinct names well before 254 of them are in play. Callers +/// are responsible for deciding whether a collision matters to them. +pub fn last_octet_for(name: &str) -> u8 { + let hash = fnv1a(name.as_bytes()); + u8::try_from(hash % 254).unwrap_or(0) + 1 +} + +fn fnv1a(bytes: &[u8]) -> u32 { + let mut hash: u32 = 0x811c_9dc5; + for &b in bytes { + hash ^= u32::from(b); + hash = hash.wrapping_mul(0x0100_0193); + } + hash +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deterministic_across_calls() { + assert_eq!(last_octet_for("redis"), last_octet_for("redis")); + } + + #[test] + fn distinct_names_can_get_distinct_octets() { + assert_ne!(last_octet_for("auth"), last_octet_for("billing")); + } + + #[test] + fn stays_within_range() { + assert!((1..=254).contains(&last_octet_for("redis"))); + } +} diff --git a/members/nullnet-grpc-lib/src/proto/nullnet_grpc.rs b/members/nullnet-grpc-lib/src/proto/nullnet_grpc.rs index 33d5328..d94b9d0 100644 --- a/members/nullnet-grpc-lib/src/proto/nullnet_grpc.rs +++ b/members/nullnet-grpc-lib/src/proto/nullnet_grpc.rs @@ -210,19 +210,38 @@ pub struct Listener { pub path: ::prost::alloc::string::String, } /// Response to ServicesList: per-declared-service, the set of trigger ports -/// the client should observe via eBPF. When traffic is observed on one of -/// these ports, the client fires BackendTrigger(service_name, port). +/// the client should observe via NFQUEUE. When traffic is observed on one of +/// these ports, the client fires BackendTrigger(service_name, port, target_name). #[derive(Clone, PartialEq, ::prost::Message)] pub struct ServicesListResponse { #[prost(message, repeated, tag = "1")] pub service_triggers: ::prost::alloc::vec::Vec, } -#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ServiceTrigger { #[prost(string, tag = "1")] pub service_name: ::prost::alloc::string::String, - #[prost(uint32, repeated, tag = "2")] - pub ports: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "3")] + pub trigger_ports: ::prost::alloc::vec::Vec, + /// Real Docker container name of the local replica hosting this service — + /// the container `target_name` (per TriggerPort) should be pre-seeded into. + /// Empty when the client couldn't resolve one (host process, no docker + /// context); the client skips seeding for those. + #[prost(string, tag = "4")] + pub initiator_container: ::prost::alloc::string::String, +} +/// One trigger port this service's initiator container should be observed on. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct TriggerPort { + #[prost(uint32, tag = "1")] + pub port: u32, + /// chain\[0\] for this port's dependency chain — the literal name the + /// initiator resolves (e.g. a bare Docker container name). Lets the client + /// pre-seed a placeholder `/etc/hosts` entry for it before any packet is + /// observed, so a bare name (not just a pre-provisioned DNS alias) produces + /// a real first packet to trigger on. + #[prost(string, tag = "2")] + pub target_name: ::prost::alloc::string::String, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct HostMapping { @@ -281,6 +300,12 @@ pub struct BackendTriggerRequest { /// replicas of the same service live on the same host. #[prost(string, tag = "3")] pub initiator_container: ::prost::alloc::string::String, + /// The target_name (see TriggerPort) the observed packet's destination + /// placeholder IP resolved back to. Lets the server pick the right chain + /// when two of this service's triggers share the same port, instead of + /// relying on port alone. + #[prost(string, tag = "4")] + pub target_name: ::prost::alloc::string::String, } /// Egress trigger — fired by the client on the first NEW flow from a registered /// service to an external (non-nullnet) destination. The server builds (or reuses) diff --git a/members/nullnet-server/src/http_server/services.rs b/members/nullnet-server/src/http_server/services.rs index f058545..c722784 100644 --- a/members/nullnet-server/src/http_server/services.rs +++ b/members/nullnet-server/src/http_server/services.rs @@ -24,7 +24,7 @@ struct ServiceJson { registered: bool, replicas: Vec, proxy_dependencies: Vec>, - triggers: HashMap>, + triggers: HashMap>>, #[serde(skip_serializing_if = "Option::is_none")] timeout_secs: Option, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/members/nullnet-server/src/nullnet_grpc_impl.rs b/members/nullnet-server/src/nullnet_grpc_impl.rs index f3774bb..92f69df 100644 --- a/members/nullnet-server/src/nullnet_grpc_impl.rs +++ b/members/nullnet-server/src/nullnet_grpc_impl.rs @@ -21,7 +21,8 @@ use nullnet_grpc_lib::nullnet_grpc::{ AgentEvent, BackendTriggerRequest, CertBundle, EgressDestinationReport, EgressPolicyCheck, EgressPolicyVerdict, EgressTriggerRequest, Empty, IngressPolicyCheck, IngressPolicyVerdict, MsgId, Net, NetMessage, NetType, PortMapping, PortMappingBundle, ProxyRequest, ServiceReport, - ServiceTrigger, ServicesListResponse, Upstream, agent_event::Event as AgentEventKind, + ServiceTrigger, ServicesListResponse, TriggerPort, Upstream, + agent_event::Event as AgentEventKind, }; use nullnet_liberror::{Error, ErrorHandler, Location, location}; use std::collections::{HashMap, HashSet}; @@ -521,7 +522,7 @@ impl NullnetGrpcImpl { let Some(stack_map) = guard.get(stack) else { continue; }; - for (name, _, _) in list { + for (name, _, docker_container) in list { if !seen.insert((stack.clone(), name.clone())) { continue; } @@ -531,11 +532,31 @@ impl NullnetGrpcImpl { if triggers.is_empty() { continue; } - let mut ports: Vec = triggers.keys().map(|p| u32::from(*p)).collect(); - ports.sort_unstable(); + // More than one chain can share a port; emit one TriggerPort + // per chain so the client learns every target_name for that + // port and can seed a placeholder for (and later + // disambiguate) each one independently. + let mut trigger_ports: Vec = triggers + .iter() + .flat_map(|(port, chains)| { + chains.iter().filter_map(move |chain| { + // chain[0] is what the initiator actually resolves; a + // trigger with an empty chain can't be pre-seeded + // (nothing to target) and can't build a chain either, + // so it's already inert — skip it here too. + let target_name = chain.first()?.clone(); + Some(TriggerPort { + port: u32::from(*port), + target_name, + }) + }) + }) + .collect(); + trigger_ports.sort_unstable_by_key(|tp| tp.port); service_triggers.push(ServiceTrigger { service_name: name.clone(), - ports, + trigger_ports, + initiator_container: docker_container.clone().unwrap_or_default(), }); } } @@ -634,6 +655,7 @@ impl NullnetGrpcImpl { service_ip: IpAddr, service_docker: Option<&str>, port: u16, + target_name: &str, ) -> Result>, Error> { let guard = self.services.read().await; let stack_map = guard @@ -652,6 +674,7 @@ impl NullnetGrpcImpl { service_ip, service_docker, port, + target_name, stack_map, ) else { return Ok(None); @@ -708,8 +731,14 @@ impl NullnetGrpcImpl { } else { Some(req.initiator_container) }; - self.handle_backend_trigger(&req.service_name, port, sender_ip, container.as_deref()) - .await?; + self.handle_backend_trigger( + &req.service_name, + port, + sender_ip, + container.as_deref(), + &req.target_name, + ) + .await?; Ok(Response::new(Empty {})) } @@ -719,6 +748,7 @@ impl NullnetGrpcImpl { port: u16, sender_ip: IpAddr, initiator_container: Option<&str>, + target_name: &str, ) -> Result<(), Error> { println!( "Received backend trigger for '{initiator_name}' (port {port}) from {sender_ip} (container: {})", @@ -760,13 +790,18 @@ impl NullnetGrpcImpl { .handle_err(location!())?; let initiator_ip = replica.ip(); let initiator_docker = replica.docker_container().map(String::from); + // `chain_for` picks the right chain among possibly several + // sharing this port, by matching `target_name` (chain[0]) — the + // literal name the client's placeholder resolved. A single + // chain on this port is used regardless (covers pre-disambiguation + // clients sending an empty target_name); with several sharing the + // port, no match means genuinely ambiguous, not a guess. let first_dep = reg - .triggers() - .get(&port) - .and_then(|chain| chain.first()) + .chain_for(port, target_name) + .and_then(|c| c.first()) .cloned(); println!( - "[trigger] triggers map for '{initiator_name}': {:?}; first_dep for port {port}: {first_dep:?}", + "[trigger] triggers map for '{initiator_name}': {:?}; target_name='{target_name}'; first_dep for port {port}: {first_dep:?}", reg.triggers() ); @@ -800,6 +835,7 @@ impl NullnetGrpcImpl { initiator_ip, initiator_docker.as_deref(), port, + target_name, ) .await } @@ -811,9 +847,17 @@ impl NullnetGrpcImpl { initiator_ip: IpAddr, initiator_docker: Option<&str>, port: u16, + target_name: &str, ) -> Result<(), Error> { let Some(mut chain) = self - .build_backend_dep_chain(stack, initiator_name, initiator_ip, initiator_docker, port) + .build_backend_dep_chain( + stack, + initiator_name, + initiator_ip, + initiator_docker, + port, + target_name, + ) .await? else { println!( diff --git a/members/nullnet-server/src/services/changes.rs b/members/nullnet-server/src/services/changes.rs index 909becf..a5f5a2d 100644 --- a/members/nullnet-server/src/services/changes.rs +++ b/members/nullnet-server/src/services/changes.rs @@ -416,31 +416,33 @@ fn collect_backend_chain_edges( let Some(triggers) = services.get(initiator_name).map(ServiceInfo::triggers) else { return edges; }; - for chain in triggers.values() { - if let Some(dep) = only_through - && !chain.iter().any(|d| d == dep) - { - continue; - } - let mut current_name = initiator_name.to_string(); - let mut current_ip = initiator_ip; - let mut current_docker: Option = initiator_docker.map(String::from); - for dep_name in chain { - let hop = emit_edge_and_probe_hop( - &mut edges, - ¤t_name, - current_ip, - current_docker.as_deref(), - dep_name, - services, - ); - match hop { - Some((ip, docker)) => { - current_name.clone_from(dep_name); - current_ip = ip; - current_docker = docker; + for chains in triggers.values() { + for chain in chains { + if let Some(dep) = only_through + && !chain.iter().any(|d| d == dep) + { + continue; + } + let mut current_name = initiator_name.to_string(); + let mut current_ip = initiator_ip; + let mut current_docker: Option = initiator_docker.map(String::from); + for dep_name in chain { + let hop = emit_edge_and_probe_hop( + &mut edges, + ¤t_name, + current_ip, + current_docker.as_deref(), + dep_name, + services, + ); + match hop { + Some((ip, docker)) => { + current_name.clone_from(dep_name); + current_ip = ip; + current_docker = docker; + } + None => break, } - None => break, } } } diff --git a/members/nullnet-server/src/services/input.rs b/members/nullnet-server/src/services/input.rs index a578df5..be2ec76 100644 --- a/members/nullnet-server/src/services/input.rs +++ b/members/nullnet-server/src/services/input.rs @@ -2,7 +2,9 @@ use crate::events::Event as ServerEvent; use crate::orchestrator::Orchestrator; use crate::services::changes::{ServiceChange, apply_changes, detect_config_changes}; use crate::services::clients::Client; -use crate::services::service_info::{CountryPolicy, ServiceInfo}; +use crate::services::service_info::{ + CountryPolicy, ServiceInfo, TriggerMap, placeholder_collision, +}; use notify::{Config, Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher}; use nullnet_grpc_lib::nullnet_grpc::ServiceProtocol; use nullnet_liberror::{Error, ErrorHandler, Location, location}; @@ -293,7 +295,43 @@ impl ServicesToml { )) .handle_err(location!()); } - let triggers = s.triggers.into_iter().map(|t| (t.port, t.chain)).collect(); + // Group by port: more than one chain can share a port (two + // dependencies reached on the same real port, e.g. two + // plain-HTTPS deps both 443), disambiguated at trigger time by + // chain[0] — so within one port, every chain's chain[0] must be + // distinct, or the server could never tell them apart either. + let mut triggers: TriggerMap = HashMap::new(); + for t in s.triggers { + let chains = triggers.entry(t.port).or_default(); + if let Some(dup) = chains + .iter() + .find(|c: &&Vec| c.first() == t.chain.first()) + { + return Err(format!( + "service '{}': two triggers on port {} both resolve as '{}' — chains \ + sharing a port must have distinct chain[0] names", + s.name, + t.port, + dup.first().map(String::as_str).unwrap_or("") + )) + .handle_err(location!()); + } + chains.push(t.chain); + } + // Distinct chain[0] names are required above, but the client + // disambiguates a same-port first packet by hashing chain[0] into + // a placeholder address (see `placeholder_collision`), and that + // hash isn't collision-free. Two different names landing on the + // same address would be just as undetectable to the client as a + // literal duplicate, so reject it here too. + if let Some((port, a, b)) = placeholder_collision(&triggers) { + return Err(format!( + "service '{}': triggers '{a}' and '{b}' on port {port} hash to the same \ + placeholder address — rename one of them so the client can tell them apart", + s.name + )) + .handle_err(location!()); + } ret_val.insert( s.name, ServiceInfo::new( @@ -696,10 +734,91 @@ chain = ["dep.b"] // Not proxy-reachable... assert_eq!(map["backend.only"].timeout(), None); // ...yet it carries its triggers and proxy deps verbatim. - assert_eq!(map["backend.only"].triggers()[&5555], vec!["dep.b"]); + assert_eq!( + map["backend.only"].triggers()[&5555], + vec![vec!["dep.b".to_string()]] + ); assert_eq!(map["backend.only"].proxy_deps(), vec![vec!["dep.a"]]); } + #[test] + fn two_triggers_sharing_a_port_with_distinct_targets_both_parse() { + // Two dependencies reached on the same real port (e.g. two + // plain-HTTPS deps, both 443) — distinguishable by chain[0]. + let toml_str = r#" +[[services]] +name = "portal" + +[[services.triggers]] +port = 443 +chain = ["auth"] + +[[services.triggers]] +port = 443 +chain = ["billing"] +"#; + let parsed: ServicesToml = toml::from_str(toml_str).unwrap(); + let map = parsed.services_map().unwrap(); + + let mut chains = map["portal"].triggers()[&443].clone(); + chains.sort(); + assert_eq!( + chains, + vec![vec!["auth".to_string()], vec!["billing".to_string()]] + ); + } + + #[test] + fn two_triggers_sharing_a_port_with_same_target_is_rejected() { + // Same port, same chain[0] — genuinely ambiguous, nothing could ever + // disambiguate which chain a trigger on this port means. + let toml_str = r#" +[[services]] +name = "portal" + +[[services.triggers]] +port = 443 +chain = ["auth"] + +[[services.triggers]] +port = 443 +chain = ["auth"] +"#; + let parsed: ServicesToml = toml::from_str(toml_str).unwrap(); + let err = format!("{:?}", parsed.services_map().unwrap_err()); + assert!(err.contains("port 443"), "unexpected error: {err}"); + assert!(err.contains("auth"), "unexpected error: {err}"); + } + + #[test] + fn two_triggers_sharing_a_port_with_colliding_placeholder_is_rejected() { + // Distinct chain[0] names, but "tax" and "wallet" hash to the same + // placeholder octet under `last_octet_for` — the client's + // `resolve_target` couldn't tell these two apart on the wire any + // better than a literal duplicate could. + assert_eq!( + nullnet_grpc_lib::last_octet_for("tax"), + nullnet_grpc_lib::last_octet_for("wallet") + ); + let toml_str = r#" +[[services]] +name = "portal" + +[[services.triggers]] +port = 443 +chain = ["tax"] + +[[services.triggers]] +port = 443 +chain = ["wallet"] +"#; + let parsed: ServicesToml = toml::from_str(toml_str).unwrap(); + let err = format!("{:?}", parsed.services_map().unwrap_err()); + assert!(err.contains("port 443"), "unexpected error: {err}"); + assert!(err.contains("tax"), "unexpected error: {err}"); + assert!(err.contains("wallet"), "unexpected error: {err}"); + } + #[test] fn parses_parallel_proxy_branches() { let toml_str = r#" diff --git a/members/nullnet-server/src/services/service_info.rs b/members/nullnet-server/src/services/service_info.rs index 501925c..dedcb88 100644 --- a/members/nullnet-server/src/services/service_info.rs +++ b/members/nullnet-server/src/services/service_info.rs @@ -6,6 +6,42 @@ use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, Ipv4Addr}; use std::time::{Duration, Instant}; +/// Backend-triggered chains keyed by the trigger port observed on the +/// initiator's host. More than one chain can share a port — two dependencies +/// that happen to be reached on the same real port (e.g. two plain-HTTPS +/// deps, both 443) — disambiguated at trigger time by `target_name` +/// (chain[0]), which the client already resolves independently from the +/// packet's destination (its placeholder address — see nullnet-client's +/// `placeholder.rs`). +pub(crate) type TriggerMap = HashMap>>; + +/// First pair of chains sharing a port whose `chain[0]` names hash to the +/// same placeholder address, if any. Two *distinct* names on the same port +/// are normally fine — `chain_for` disambiguates by name, not address — but +/// the client picks which chain a first packet belongs to by matching the +/// packet's destination against each candidate's placeholder address (see +/// nullnet-client's `nfqueue/listener.rs::resolve_target` and +/// `placeholder.rs`). If two names collide under that hash, the client +/// can't tell them apart either and will silently pick whichever candidate +/// comes first, wiring the wrong dependency's traffic through. Distinct +/// `chain[0]` names are already required (see `services_map`'s dup check); +/// this catches the rarer case where two different names still land on the +/// same address. +pub(crate) fn placeholder_collision(triggers: &TriggerMap) -> Option<(u16, String, String)> { + for (&port, chains) in triggers { + for i in 0..chains.len() { + for other in &chains[i + 1..] { + let a = chains[i].first().map(String::as_str).unwrap_or(""); + let b = other.first().map(String::as_str).unwrap_or(""); + if nullnet_grpc_lib::last_octet_for(a) == nullnet_grpc_lib::last_octet_for(b) { + return Some((port, a.to_string(), b.to_string())); + } + } + } + } + None +} + /// Service names that participate in backend trigger chains: those that declare /// triggers ("have backend deps") and every service named in a trigger chain /// ("is a backend dep"). These are never paused — backend-dep networks aren't @@ -21,7 +57,7 @@ pub(crate) fn backend_involved_services( continue; } pinned.insert(name.clone()); - for dep in triggers.values().flatten() { + for dep in triggers.values().flatten().flatten() { pinned.insert(dep.clone()); } } @@ -62,7 +98,7 @@ impl ServiceInfo { #[allow(clippy::too_many_arguments)] pub(crate) fn new( proxy_deps: Vec>, - triggers: HashMap>, + triggers: TriggerMap, timeout: Option, max_networks: Option, protocol: ServiceProtocol, @@ -250,7 +286,7 @@ impl ServiceInfo { } } - pub(crate) fn triggers(&self) -> &HashMap> { + pub(crate) fn triggers(&self) -> &TriggerMap { match self { ServiceInfo::Unregistered(unreg) => &unreg.triggers, ServiceInfo::Registered(reg) => ®.triggers, @@ -260,7 +296,12 @@ impl ServiceInfo { /// True iff `other` appears in any of this service's dep lists (proxy or backend). pub(crate) fn deps_contain(&self, other: &str) -> bool { self.proxy_deps().iter().flatten().any(|d| d == other) - || self.triggers().values().flatten().any(|d| d == other) + || self + .triggers() + .values() + .flatten() + .flatten() + .any(|d| d == other) } } @@ -271,7 +312,7 @@ pub(crate) struct UnregisteredServiceInfo { proxy_deps: Vec>, /// Backend-triggered chains keyed by the trigger port observed on the /// initiator's host. One linear chain per port; no implicit fan-out. - triggers: HashMap>, + triggers: TriggerMap, /// Whether the proxy is reachable for this service, with the associated timeout. timeout: Option, /// Maximum number of networks for this service. @@ -290,7 +331,7 @@ impl UnregisteredServiceInfo { #[allow(clippy::too_many_arguments)] fn new( proxy_deps: Vec>, - triggers: HashMap>, + triggers: TriggerMap, timeout: Option, max_networks: Option, protocol: ServiceProtocol, @@ -384,8 +425,10 @@ pub(crate) struct RegisteredServiceInfo { /// is one linear branch; all branches are brought up in parallel. proxy_deps: Vec>, /// Backend-triggered chains keyed by the trigger port observed on the - /// initiator's host. One linear chain per port; no implicit fan-out. - triggers: HashMap>, + /// initiator's host. More than one chain may share a port, disambiguated + /// at trigger time by `chain_for`'s `target_name` match; no implicit + /// fan-out otherwise. + triggers: TriggerMap, /// Whether the proxy is reachable for this service, with the associated timeout. timeout: Option, /// Maximum number of networks for this service. @@ -427,17 +470,37 @@ impl RegisteredServiceInfo { .collect() } - /// Build the chain of edges for the trigger at `port`, if one exists. - /// Each chain starts at this service's replica. + /// Select the trigger chain for `port` that matches `target_name` — the + /// literal name (chain[0]) the initiator resolved via its placeholder. + /// When only one chain exists for that port, it's used regardless of + /// `target_name` — covers the common case, and callers that don't (yet) + /// know a target_name (an older client, or the empty string). With + /// multiple chains sharing a port, an exact match is required — an + /// empty or unmatched `target_name` is genuinely ambiguous and returns + /// `None` rather than guessing which dependency was meant. + pub(crate) fn chain_for(&self, port: u16, target_name: &str) -> Option<&Vec> { + let chains = self.triggers.get(&port)?; + match chains.as_slice() { + [only] => Some(only), + many => many + .iter() + .find(|c| c.first().map(String::as_str) == Some(target_name)), + } + } + + /// Build the chain of edges for the trigger at `port` matching + /// `target_name`, if one exists. Each chain starts at this service's + /// replica. pub(crate) fn backend_dependency_chain( &self, service_name: &str, service_ip: IpAddr, service_docker: Option<&str>, port: u16, + target_name: &str, services: &HashMap, ) -> Option> { - let chain = self.triggers.get(&port)?; + let chain = self.chain_for(port, target_name)?; Some(build_linear_chain( chain, service_name.to_string(), @@ -645,7 +708,7 @@ impl RegisteredServiceInfo { &self.replicas } - pub(crate) fn triggers(&self) -> &HashMap> { + pub(crate) fn triggers(&self) -> &TriggerMap { &self.triggers } @@ -798,3 +861,56 @@ fn build_linear_chain( } chain } + +#[cfg(test)] +mod chain_selection_tests { + use super::*; + + fn registered_with_triggers(triggers: TriggerMap) -> RegisteredServiceInfo { + RegisteredServiceInfo { + proxy_deps: Vec::new(), + triggers, + timeout: None, + max_networks: None, + protocol: ServiceProtocol::Http, + listen_port: None, + egress_policy: CountryPolicy::None, + ingress_policy: CountryPolicy::None, + replicas: Vec::new(), + } + } + + #[test] + fn single_chain_ignores_target_name() { + let reg = registered_with_triggers(HashMap::from([(443, vec![vec!["auth".to_string()]])])); + // Even an empty or unrelated target_name still resolves the only + // chain — covers pre-disambiguation clients and the common case. + assert_eq!(reg.chain_for(443, ""), Some(&vec!["auth".to_string()])); + assert_eq!( + reg.chain_for(443, "billing"), + Some(&vec!["auth".to_string()]) + ); + } + + #[test] + fn multiple_chains_require_exact_target_name_match() { + let reg = registered_with_triggers(HashMap::from([( + 443, + vec![vec!["auth".to_string()], vec!["billing".to_string()]], + )])); + assert_eq!(reg.chain_for(443, "auth"), Some(&vec!["auth".to_string()])); + assert_eq!( + reg.chain_for(443, "billing"), + Some(&vec!["billing".to_string()]) + ); + // Ambiguous — no guessing. + assert_eq!(reg.chain_for(443, ""), None); + assert_eq!(reg.chain_for(443, "unknown"), None); + } + + #[test] + fn unknown_port_returns_none() { + let reg = registered_with_triggers(HashMap::new()); + assert_eq!(reg.chain_for(443, "auth"), None); + } +} diff --git a/members/nullnet-server/src/tests.rs b/members/nullnet-server/src/tests.rs index 4f053f7..8cf1fd9 100644 --- a/members/nullnet-server/src/tests.rs +++ b/members/nullnet-server/src/tests.rs @@ -138,7 +138,7 @@ async fn trigger_backend_chain( port: u16, ) { server - .handle_backend_trigger(initiator_name, port, initiator_ip, None) + .handle_backend_trigger(initiator_name, port, initiator_ip, None, "") .await .expect("backend trigger failed"); } @@ -159,6 +159,7 @@ async fn setup_backend_chain_for_replica( initiator_ip, initiator_docker, port, + "", ) .await .expect("setup_backend_chain failed"); @@ -1978,7 +1979,7 @@ async fn triggers_changed_swap_A_trigger() { assert_graphviz(&guard, TRIGGERS_CHANGED, "after_swap_A_trigger.dot"); assert_eq!( stack_view(&guard)["A"].triggers().get(&5555), - Some(&vec!["D".to_string()]) + Some(&vec![vec!["D".to_string()]]) ); drop(guard); @@ -2264,7 +2265,7 @@ async fn backend_trigger_disambiguates_colocated_replica_by_container() { let server = backend_disambiguation_setup().await; server - .handle_backend_trigger("A", 5555, ip(1, 1, 1, 1), Some("a2")) + .handle_backend_trigger("A", 5555, ip(1, 1, 1, 1), Some("a2"), "") .await .expect("backend trigger for a2 should succeed"); @@ -2305,7 +2306,7 @@ async fn backend_trigger_unknown_container_errors_without_ip_fallback() { let server = backend_disambiguation_setup().await; let result = server - .handle_backend_trigger("A", 5555, ip(1, 1, 1, 1), Some("ghost")) + .handle_backend_trigger("A", 5555, ip(1, 1, 1, 1), Some("ghost"), "") .await; assert!( @@ -2322,7 +2323,7 @@ async fn backend_trigger_without_container_falls_back_to_ip_only() { let server = backend_disambiguation_setup().await; server - .handle_backend_trigger("A", 5555, ip(1, 1, 1, 1), None) + .handle_backend_trigger("A", 5555, ip(1, 1, 1, 1), None, "") .await .expect("IP-only trigger should succeed"); @@ -2855,7 +2856,7 @@ async fn backend_involved_replicas_never_suspended() { "initiator".to_string(), ServiceInfo::new( vec![], - HashMap::from([(8080u16, vec!["dep".to_string()])]), + HashMap::from([(8080u16, vec![vec!["dep".to_string()]])]), Some(30), None, ServiceProtocol::Http, diff --git a/members/nullnet-server/ui/src/pages/Config.tsx b/members/nullnet-server/ui/src/pages/Config.tsx index 5316319..1f0d169 100644 --- a/members/nullnet-server/ui/src/pages/Config.tsx +++ b/members/nullnet-server/ui/src/pages/Config.tsx @@ -203,7 +203,7 @@ export default function Config() { )} {status.kind === 'error' && ( - Parse error + Rejected {status.msg} )} diff --git a/members/nullnet-server/ui/src/pages/Services.tsx b/members/nullnet-server/ui/src/pages/Services.tsx index 231895a..a5f6826 100644 --- a/members/nullnet-server/ui/src/pages/Services.tsx +++ b/members/nullnet-server/ui/src/pages/Services.tsx @@ -140,12 +140,14 @@ export default function Services() { {svc.max_networks} )} - {Object.entries(svc.triggers).map(([port, chain]) => ( - - Trigger :{port} - {chain.join(' → ')} - - ))} + {Object.entries(svc.triggers).flatMap(([port, chains]) => + chains.map((chain, i) => ( + + Trigger :{port} + {chain.join(' → ')} + + )) + )} {svc.proxy_dependencies.flat().length > 0 && ( Dependencies diff --git a/members/nullnet-server/ui/src/types.ts b/members/nullnet-server/ui/src/types.ts index f3b51e3..690e90b 100644 --- a/members/nullnet-server/ui/src/types.ts +++ b/members/nullnet-server/ui/src/types.ts @@ -15,7 +15,10 @@ export interface ServiceJson { registered: boolean; replicas: ReplicaJson[]; proxy_dependencies: string[][]; - triggers: Record; + // Port -> the chains sharing it. Usually one chain per port; more than one + // means two dependencies reached on the same real port, disambiguated at + // trigger time by chain[0]. + triggers: Record; timeout_secs?: number; max_networks?: number; }