Fix stale MACsec peer SCI on same-host chain edges

members/nullnet-client/vxlan_scripts/vxlan-setup.sh

@@ -22,6 +22,14 @@ DOCKER_CONTAINER=${11}
 
 BR_IP=$(echo $BR_NET | cut -d'/' -f1)
 
+# Serialize every operation on this net id. Setup runs once per side (_s and
+# _c) and a teardown can fire between them; unserialized, those passes
+# interleave and leave the two halves of the link built on different veth
+# incarnations. vxlan-teardown.sh takes the same lock.
+LOCK_FILE="/var/lock/nullnet-net-${VXLAN_ID}.lock"
+exec 9>"$LOCK_FILE"
+flock -w 30 9 || echo "warning: net $VXLAN_ID lock timed out, proceeding unserialized" >&2
+
 # Overlay MTU: empirically measured via `ping -M do -s <N>` against a live
 # cross-host tunnel — an IP packet of 1104 bytes gets through cleanly, 1105
 # never does, every time (a hard, reproducible ceiling, not a fragmentation
@@ -77,27 +85,46 @@ if [ "$LOCAL_IP" == "$REMOTE_IP" ]; then
       # the veth link itself in AES-256-GCM, keyed with this tunnel's key —
       # no IP addressing involved, so it works regardless of what the
       # containers on either side are doing.
+      # Drop any artifacts left by a previous cross-host incarnation of this
+      # net id — an edge switches branch when its peer relocates onto or off
+      # this host, and the stale tunnel would otherwise outlive the flip.
+      sudo ip link del vxlan-$NS_NAME 2>/dev/null
+      sudo ip xfrm state deleteall proto esp spi $(printf '0x%08x' $((VXLAN_ID + 1000))) 2>/dev/null
+
       VETH_S="veth-${VXLAN_ID}-s"
       VETH_C="veth-${VXLAN_ID}-c"
+      # A macsec interface inherits its parent veth's MAC, so derive both ends
+      # from the net id: each side's SCI — and the peer address the other side
+      # keys its RX SA to — becomes a pure function of $VXLAN_ID. Reading the
+      # peer's MAC instead left the RX SA on a replaced incarnation's SCI after
+      # a rebuild, silently dropping every frame. 0x02 = locally administered.
+      veth_mac() { # $1: 1 = _s end, 2 = _c end
+          printf '02:%02x:%02x:%02x:%02x:%02x' "$1" \
+              $(( (VXLAN_ID >> 24) & 0xff )) $(( (VXLAN_ID >> 16) & 0xff )) \
+              $(( (VXLAN_ID >> 8)  & 0xff )) $((  VXLAN_ID        & 0xff ))
+      }
+      MAC_S=$(veth_mac 1)
+      MAC_C=$(veth_mac 2)
       # Both ends are created atomically; the losing task's EEXIST is harmless
-      sudo ip link add "$VETH_S" type veth peer name "$VETH_C" 2>/dev/null
+      sudo ip link add "$VETH_S" address "$MAC_S" type veth peer name "$VETH_C" address "$MAC_C" 2>/dev/null
       # Attach our end to our bridge
       if [[ "$BR_NAME" == *_s ]]; then
           LOCAL_VETH="$VETH_S"
-          PEER_VETH="$VETH_C"
           MACSEC_IF="macsec-${VXLAN_ID}-s"
+          LOCAL_MAC="$MAC_S"
+          PEER_MAC="$MAC_C"
       else
           LOCAL_VETH="$VETH_C"
-          PEER_VETH="$VETH_S"
           MACSEC_IF="macsec-${VXLAN_ID}-c"
+          LOCAL_MAC="$MAC_C"
+          PEER_MAC="$MAC_S"
       fi
 
       if [ "$ENCRYPTED" == "true" ]; then
-          # The peer's MAC is available immediately: `ip link add ... peer
-          # name ...` creates both ends atomically in one kernel call,
-          # whether this invocation won the race above or lost it to the
-          # sibling script.
-          PEER_MAC=$(cat /sys/class/net/$PEER_VETH/address)
+          # A pair predating deterministic MACs keeps its old random address,
+          # which would desync the SCIs again — force our end to the derived
+          # value before stacking macsec on it.
+          sudo ip link set "$LOCAL_VETH" address "$LOCAL_MAC"
           KEY_ID=$(printf '%032x' $VXLAN_ID)
 
           # MACsec adds up to 32 bytes of overhead (SecTAG + ICV for
@@ -130,6 +157,13 @@ if [ "$LOCAL_IP" == "$REMOTE_IP" ]; then
           sudo ip link set "$LOCAL_VETH" mtu $OVERLAY_MTU up
       fi
   else
+      # Mirror of the same-host purge above: drop veth/macsec artifacts left
+      # by a previous same-host incarnation of this net id. Deleting either
+      # veth end takes its peer and any stacked macsec with it.
+      sudo ip link del macsec-${VXLAN_ID}-s 2>/dev/null
+      sudo ip link del macsec-${VXLAN_ID}-c 2>/dev/null
+      sudo ip link del veth-${VXLAN_ID}-s 2>/dev/null
+
       # Create the VXLAN tunnel using your physical IP and interface. Each
       # tunnel gets its own dstport (instead of the IANA-standard 4789) so
       # the XFRM policies below can tell concurrent tunnels between the same

members/nullnet-client/vxlan_scripts/vxlan-teardown.sh

@@ -16,29 +16,42 @@ REMOTE_IP=$5
 DSTPORT=$6
 DOCKER_CONTAINER=$7
 
-# Remove this tunnel's XFRM state + policy pair, if any was installed (the
-# same-host branch of vxlan-setup.sh never creates one).
-if [ "$LOCAL_IP" != "$REMOTE_IP" ]; then
-    # Must match the same offset vxlan-setup.sh uses, to delete the actual
-    # installed SPI rather than the raw (and IANA-reserved) vxlan_id.
-    SPI=$(printf '0x%08x' $((VXLAN_ID + 1000)))
+# Same lock as vxlan-setup.sh: a teardown landing between the two setup passes
+# is what let the halves of a same-host link diverge in the first place.
+LOCK_FILE="/var/lock/nullnet-net-${VXLAN_ID}.lock"
+exec 9>"$LOCK_FILE"
+flock -w 30 9 || echo "warning: net $VXLAN_ID lock timed out, proceeding unserialized" >&2
+
+# Remove this tunnel's XFRM state + policy pair, if any was installed. Matched
+# on SPI and dstport alone — both unique to this net id — rather than on the
+# endpoints, so state survives neither a peer relocating to a different host
+# nor a flip to the same-host branch, either of which leaves teardown holding
+# endpoints that no longer describe what setup installed.
+SPI=$(printf '0x%08x' $((VXLAN_ID + 1000)))
+# Only tunnels holding a dedicated dstport ever get an XFRM policy; the rest
+# share DEFAULT_VXLAN_DSTPORT (net_id_pool.rs), so matching on 4789 would
+# reach across unrelated tunnels instead of just this one.
+if [ "$DSTPORT" != "4789" ]; then
     # Same argument-order requirement as vxlan-setup.sh: selector fields
-    # (src/dst/proto/dport) must stay contiguous, with `dir` only after.
-    sudo ip xfrm policy delete src $LOCAL_IP dst $REMOTE_IP proto udp dport $DSTPORT dir out 2>/dev/null
-    sudo ip xfrm state delete src $LOCAL_IP dst $REMOTE_IP proto esp spi $SPI 2>/dev/null
-    sudo ip xfrm policy delete src $REMOTE_IP dst $LOCAL_IP proto udp dport $DSTPORT dir in 2>/dev/null
-    sudo ip xfrm state delete src $REMOTE_IP dst $LOCAL_IP proto esp spi $SPI 2>/dev/null
+    # (proto/dport) must stay contiguous, with `dir` only after.
+    sudo ip xfrm policy deleteall proto udp dport $DSTPORT dir out 2>/dev/null
+    sudo ip xfrm policy deleteall proto udp dport $DSTPORT dir in 2>/dev/null
 fi
+sudo ip xfrm state deleteall proto esp spi $SPI 2>/dev/null
 
 # Remove the VXLAN tunnel or same-host veth pair. Deleting a veth end also
 # destroys its peer and cascades to remove any macsec interface stacked on
 # either end (the same-host branch of vxlan-setup.sh wraps each end in one),
 # but delete both macsec names explicitly too rather than depend solely on
 # that cascade.
+# Both modes are swept unconditionally: whichever branch setup took, only one
+# set exists, and the other's absence is expected rather than an error worth
+# logging — the stray "Cannot find device" lines it used to emit made real
+# failures hard to spot in the journal.
 sudo ip link del macsec-${VXLAN_ID}-s 2>/dev/null
 sudo ip link del macsec-${VXLAN_ID}-c 2>/dev/null
-sudo ip link set vxlan-$NS_NAME down && sudo ip link del vxlan-$NS_NAME
-sudo ip link set veth-${VXLAN_ID}-s down && sudo ip link del veth-${VXLAN_ID}-s
+sudo ip link del vxlan-$NS_NAME 2>/dev/null
+sudo ip link del veth-${VXLAN_ID}-s 2>/dev/null
 
 # Remove the namespace veth pair:
 sudo ip link set $NS_NAME-out down && sudo ip link del $NS_NAME-out