From 93dcb8e27fb6beea16f7929c616990680c07de36 Mon Sep 17 00:00:00 2001 From: GyulyVGC Date: Wed, 5 Aug 2026 12:36:41 +0200 Subject: [PATCH] client: derive MACsec veth MACs from net id instead of reading them --- .../vxlan_scripts/vxlan-setup.sh | 50 ++++++++++++++++--- .../vxlan_scripts/vxlan-teardown.sh | 39 ++++++++++----- 2 files changed, 68 insertions(+), 21 deletions(-) diff --git a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh index 339a39f..a239b00 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh @@ -22,6 +22,14 @@ DOCKER_CONTAINER=${11} BR_IP=$(echo $BR_NET | cut -d'/' -f1) +# Serialize every operation on this net id. Setup runs once per side (_s and +# _c) and a teardown can fire between them; unserialized, those passes +# interleave and leave the two halves of the link built on different veth +# incarnations. vxlan-teardown.sh takes the same lock. +LOCK_FILE="/var/lock/nullnet-net-${VXLAN_ID}.lock" +exec 9>"$LOCK_FILE" +flock -w 30 9 || echo "warning: net $VXLAN_ID lock timed out, proceeding unserialized" >&2 + # Overlay MTU: empirically measured via `ping -M do -s ` against a live # cross-host tunnel — an IP packet of 1104 bytes gets through cleanly, 1105 # never does, every time (a hard, reproducible ceiling, not a fragmentation @@ -77,27 +85,46 @@ if [ "$LOCAL_IP" == "$REMOTE_IP" ]; then # the veth link itself in AES-256-GCM, keyed with this tunnel's key — # no IP addressing involved, so it works regardless of what the # containers on either side are doing. + # Drop any artifacts left by a previous cross-host incarnation of this + # net id — an edge switches branch when its peer relocates onto or off + # this host, and the stale tunnel would otherwise outlive the flip. + sudo ip link del vxlan-$NS_NAME 2>/dev/null + sudo ip xfrm state deleteall proto esp spi $(printf '0x%08x' $((VXLAN_ID + 1000))) 2>/dev/null + VETH_S="veth-${VXLAN_ID}-s" VETH_C="veth-${VXLAN_ID}-c" + # A macsec interface inherits its parent veth's MAC, so derive both ends + # from the net id: each side's SCI — and the peer address the other side + # keys its RX SA to — becomes a pure function of $VXLAN_ID. Reading the + # peer's MAC instead left the RX SA on a replaced incarnation's SCI after + # a rebuild, silently dropping every frame. 0x02 = locally administered. + veth_mac() { # $1: 1 = _s end, 2 = _c end + printf '02:%02x:%02x:%02x:%02x:%02x' "$1" \ + $(( (VXLAN_ID >> 24) & 0xff )) $(( (VXLAN_ID >> 16) & 0xff )) \ + $(( (VXLAN_ID >> 8) & 0xff )) $(( VXLAN_ID & 0xff )) + } + MAC_S=$(veth_mac 1) + MAC_C=$(veth_mac 2) # Both ends are created atomically; the losing task's EEXIST is harmless - sudo ip link add "$VETH_S" type veth peer name "$VETH_C" 2>/dev/null + sudo ip link add "$VETH_S" address "$MAC_S" type veth peer name "$VETH_C" address "$MAC_C" 2>/dev/null # Attach our end to our bridge if [[ "$BR_NAME" == *_s ]]; then LOCAL_VETH="$VETH_S" - PEER_VETH="$VETH_C" MACSEC_IF="macsec-${VXLAN_ID}-s" + LOCAL_MAC="$MAC_S" + PEER_MAC="$MAC_C" else LOCAL_VETH="$VETH_C" - PEER_VETH="$VETH_S" MACSEC_IF="macsec-${VXLAN_ID}-c" + LOCAL_MAC="$MAC_C" + PEER_MAC="$MAC_S" fi if [ "$ENCRYPTED" == "true" ]; then - # The peer's MAC is available immediately: `ip link add ... peer - # name ...` creates both ends atomically in one kernel call, - # whether this invocation won the race above or lost it to the - # sibling script. - PEER_MAC=$(cat /sys/class/net/$PEER_VETH/address) + # A pair predating deterministic MACs keeps its old random address, + # which would desync the SCIs again — force our end to the derived + # value before stacking macsec on it. + sudo ip link set "$LOCAL_VETH" address "$LOCAL_MAC" KEY_ID=$(printf '%032x' $VXLAN_ID) # MACsec adds up to 32 bytes of overhead (SecTAG + ICV for @@ -130,6 +157,13 @@ if [ "$LOCAL_IP" == "$REMOTE_IP" ]; then sudo ip link set "$LOCAL_VETH" mtu $OVERLAY_MTU up fi else + # Mirror of the same-host purge above: drop veth/macsec artifacts left + # by a previous same-host incarnation of this net id. Deleting either + # veth end takes its peer and any stacked macsec with it. + sudo ip link del macsec-${VXLAN_ID}-s 2>/dev/null + sudo ip link del macsec-${VXLAN_ID}-c 2>/dev/null + sudo ip link del veth-${VXLAN_ID}-s 2>/dev/null + # Create the VXLAN tunnel using your physical IP and interface. Each # tunnel gets its own dstport (instead of the IANA-standard 4789) so # the XFRM policies below can tell concurrent tunnels between the same diff --git a/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh b/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh index f64ae11..0b0c902 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh @@ -16,29 +16,42 @@ REMOTE_IP=$5 DSTPORT=$6 DOCKER_CONTAINER=$7 -# Remove this tunnel's XFRM state + policy pair, if any was installed (the -# same-host branch of vxlan-setup.sh never creates one). -if [ "$LOCAL_IP" != "$REMOTE_IP" ]; then - # Must match the same offset vxlan-setup.sh uses, to delete the actual - # installed SPI rather than the raw (and IANA-reserved) vxlan_id. - SPI=$(printf '0x%08x' $((VXLAN_ID + 1000))) +# Same lock as vxlan-setup.sh: a teardown landing between the two setup passes +# is what let the halves of a same-host link diverge in the first place. +LOCK_FILE="/var/lock/nullnet-net-${VXLAN_ID}.lock" +exec 9>"$LOCK_FILE" +flock -w 30 9 || echo "warning: net $VXLAN_ID lock timed out, proceeding unserialized" >&2 + +# Remove this tunnel's XFRM state + policy pair, if any was installed. Matched +# on SPI and dstport alone — both unique to this net id — rather than on the +# endpoints, so state survives neither a peer relocating to a different host +# nor a flip to the same-host branch, either of which leaves teardown holding +# endpoints that no longer describe what setup installed. +SPI=$(printf '0x%08x' $((VXLAN_ID + 1000))) +# Only tunnels holding a dedicated dstport ever get an XFRM policy; the rest +# share DEFAULT_VXLAN_DSTPORT (net_id_pool.rs), so matching on 4789 would +# reach across unrelated tunnels instead of just this one. +if [ "$DSTPORT" != "4789" ]; then # Same argument-order requirement as vxlan-setup.sh: selector fields - # (src/dst/proto/dport) must stay contiguous, with `dir` only after. - sudo ip xfrm policy delete src $LOCAL_IP dst $REMOTE_IP proto udp dport $DSTPORT dir out 2>/dev/null - sudo ip xfrm state delete src $LOCAL_IP dst $REMOTE_IP proto esp spi $SPI 2>/dev/null - sudo ip xfrm policy delete src $REMOTE_IP dst $LOCAL_IP proto udp dport $DSTPORT dir in 2>/dev/null - sudo ip xfrm state delete src $REMOTE_IP dst $LOCAL_IP proto esp spi $SPI 2>/dev/null + # (proto/dport) must stay contiguous, with `dir` only after. + sudo ip xfrm policy deleteall proto udp dport $DSTPORT dir out 2>/dev/null + sudo ip xfrm policy deleteall proto udp dport $DSTPORT dir in 2>/dev/null fi +sudo ip xfrm state deleteall proto esp spi $SPI 2>/dev/null # Remove the VXLAN tunnel or same-host veth pair. Deleting a veth end also # destroys its peer and cascades to remove any macsec interface stacked on # either end (the same-host branch of vxlan-setup.sh wraps each end in one), # but delete both macsec names explicitly too rather than depend solely on # that cascade. +# Both modes are swept unconditionally: whichever branch setup took, only one +# set exists, and the other's absence is expected rather than an error worth +# logging — the stray "Cannot find device" lines it used to emit made real +# failures hard to spot in the journal. sudo ip link del macsec-${VXLAN_ID}-s 2>/dev/null sudo ip link del macsec-${VXLAN_ID}-c 2>/dev/null -sudo ip link set vxlan-$NS_NAME down && sudo ip link del vxlan-$NS_NAME -sudo ip link set veth-${VXLAN_ID}-s down && sudo ip link del veth-${VXLAN_ID}-s +sudo ip link del vxlan-$NS_NAME 2>/dev/null +sudo ip link del veth-${VXLAN_ID}-s 2>/dev/null # Remove the namespace veth pair: sudo ip link set $NS_NAME-out down && sudo ip link del $NS_NAME-out