USHIFT-7382: Add MTU boundary tests for C2CC and C2CC+IPsec

test/assets/c2cc/df-bit-send.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+"""Send a UDP datagram with the DF (Don't Fragment) bit set.
+
+Uses IP_PMTUDISC_DO (IPv4) or IPV6_DONTFRAG (IPv6) so the kernel
+rejects datagrams that exceed the path MTU with EMSGSIZE instead of
+fragmenting them.  Auto-detects the address family from the
+destination address.
+
+Uses UDP instead of ICMP to avoid requiring NET_RAW capability.
+UDP overhead (IP+UDP) matches ICMP overhead (IP+ICMP): 28B for IPv4,
+48B for IPv6, so payload sizes are equivalent to ping -s values
+within each address family.
+
+Usage: python3 df-bit-send.py <dest_ip> <payload_size>
+Exit:  prints "OK" on success, raises OSError on rejection.
+"""
+
+import socket
+import sys
+
+IPPROTO_IPV6 = 41
+IPV6_DONTFRAG = 62
+IP_MTU_DISCOVER = 10
+IP_PMTUDISC_DO = 2
+
+dest = sys.argv[1]
+size = int(sys.argv[2])
+
+family = socket.AF_INET6 if ":" in dest else socket.AF_INET
+sock = socket.socket(family, socket.SOCK_DGRAM)
+
+if family == socket.AF_INET6:
+    sock.setsockopt(IPPROTO_IPV6, IPV6_DONTFRAG, 1)
+else:
+    sock.setsockopt(socket.IPPROTO_IP, IP_MTU_DISCOVER, IP_PMTUDISC_DO)
+
+try:
+    sock.sendto(b"A" * size, (dest, 9999))
+    print("OK")
+except OSError as e:
+    print(f"FAIL: {e}")
+finally:
+    sock.close()

test/assets/c2cc/nettest-pod.yaml

@@ -0,0 +1,19 @@
+kind: Pod
+apiVersion: v1
+metadata:
+  name: nettest-pod
+spec:
+  terminationGracePeriodSeconds: 0
+  containers:
+  - name: nettest
+    image: registry.access.redhat.com/ubi9/ubi:9.6
+    command: ["sleep", "infinity"]
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+      runAsNonRoot: true
+      runAsUser: 10001
+      seccompProfile:
+        type: RuntimeDefault

test/assets/network/jumbo-ipv6-network.xml

@@ -0,0 +1,14 @@
+<network>
+  <name>jumbo-ipv6</name>
+  <forward mode="nat">
+    <nat ipv6='yes'>
+      <port start='1024' end='65535'/>
+    </nat>
+  </forward>
+  <mtu size='9000'/>
+  <ip family="ipv6" address="2001:db8:ca7:fe::1" prefix="96">
+    <dhcp>
+      <range start="2001:db8:ca7:fe::1000" end="2001:db8:ca7:fe::2000" />
+    </dhcp>
+  </ip>
+</network>

test/assets/network/jumbo-network.xml

@@ -0,0 +1,10 @@
+<network>
+  <name>jumbo</name>
+  <forward mode='nat'/>
+  <mtu size='9000'/>
+  <ip address='192.168.150.1' netmask='255.255.255.0'>
+    <dhcp>
+      <range start='192.168.150.100' end='192.168.150.254'/>
+    </dhcp>
+  </ip>
+</network>

test/bin/c2cc_common.sh

@@ -27,6 +27,36 @@ CLUSTER_C_SVC_CIDR_DUAL=""
 
 export TEST_RANDOMIZATION=suites
 
+# c2cc_setup_ipv6 overrides CIDRs and mirror registry for single-stack IPv6
+# scenarios. If a network name is provided, also sets VM_BRIDGE_IP and
+# WEB_SERVER_URL for that network (standard IPv6 scenarios). Jumbo IPv6
+# scenarios omit the network arg and set VM_BRIDGE_IP later, after creating
+# the jumbo-ipv6 network.
+# shellcheck disable=SC2120
+c2cc_setup_ipv6() {
+    local -r network="${1:-}"
+
+    # shellcheck disable=SC2034  # used by scenario.sh and kickstart templates
+    MIRROR_REGISTRY_URL="$(hostname):${MIRROR_REGISTRY_PORT}/microshift"
+
+    CLUSTER_A_POD_CIDR="fd01::/48"
+    CLUSTER_A_SVC_CIDR="fd02::/112"
+    CLUSTER_A_DOMAIN="cluster-a.remote"
+    CLUSTER_B_POD_CIDR="fd04::/48"
+    CLUSTER_B_SVC_CIDR="fd05::/112"
+    CLUSTER_B_DOMAIN="cluster-b.remote"
+    CLUSTER_C_POD_CIDR="fd07::/48"
+    CLUSTER_C_SVC_CIDR="fd08::/112"
+    CLUSTER_C_DOMAIN="cluster-c.remote"
+
+    if [[ -n "${network}" ]]; then
+        # shellcheck disable=SC2034  # used by scenario.sh
+        VM_BRIDGE_IP="$(get_vm_bridge_ip "${network}")"
+        # shellcheck disable=SC2034  # used by scenario.sh
+        WEB_SERVER_URL="http://[${VM_BRIDGE_IP}]:${WEB_SERVER_PORT}"
+    fi
+}
+
 get_host_ip() {
     local host=$1
     get_vm_property "${host}" ip || { echo "failed to get ${host} ip" >&2; return 1; }
@@ -188,11 +218,77 @@ ${network_yaml}IEOF
 EOF
 }
 
+# inject_kickstart_mtu configures jumbo MTU on the guest VM via kickstart.
+# The NIC and pod MTU must be set before MicroShift first starts — OVN-K
+# bakes the MTU into its database at initial creation and does not update
+# it on restart. Three mechanisms set the NIC MTU (NM conf.d, NM dispatcher,
+# systemd oneshot) and ovn.yaml sets the pod MTU.
+# Args: host mtu
+inject_kickstart_mtu() {
+    local -r host="${1}"
+    local -r mtu="${2}"
+
+    local -r ks_dir="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${host}"
+    cat >> "${ks_dir}/post-microshift.cfg" <<EOF
+# NM conf.d -- default ethernet MTU for auto-created connections
+mkdir -p /etc/NetworkManager/conf.d
+cat > /etc/NetworkManager/conf.d/99-jumbo-mtu.conf <<'NMCONF'
+[connection-ethernet-jumbo]
+match-device=type:ethernet
+ethernet.mtu=${mtu}
+NMCONF
+
+# NM dispatcher -- runs ip link set during connection activation
+mkdir -p /etc/NetworkManager/dispatcher.d
+cat > /etc/NetworkManager/dispatcher.d/99-jumbo-mtu <<'DISPEOF'
+#!/bin/bash
+[ "\$2" != "up" ] && exit 0
+ip link set dev "\$1" mtu ${mtu} 2>&1 || logger -t jumbo-mtu "Failed to set MTU ${mtu} on \$1"
+DISPEOF
+chmod 755 /etc/NetworkManager/dispatcher.d/99-jumbo-mtu
+restorecon -R /etc/NetworkManager/dispatcher.d/ 2>&1 || logger -t jumbo-mtu "restorecon failed for dispatcher.d"
+
+# Systemd service -- runs before microshift.service.
+# Script in /etc/ because only /etc/ and /var/ persist across reboots on bootc.
+cat > /etc/set-jumbo-mtu.sh <<'SCRIPTEOF'
+#!/bin/bash
+for dev in \$(ip -o link show type ether | awk -F: '{print \$2}' | tr -d ' '); do
+    ip link set dev "\$dev" mtu ${mtu}
+done
+SCRIPTEOF
+chmod 755 /etc/set-jumbo-mtu.sh
+restorecon /etc/set-jumbo-mtu.sh 2>&1 || logger -t jumbo-mtu "restorecon failed for set-jumbo-mtu.sh"
+
+cat > /etc/systemd/system/set-jumbo-mtu.service <<'SVCEOF'
+[Unit]
+Description=Set jumbo frame MTU on ethernet interfaces
+Before=microshift.service
+After=NetworkManager-wait-online.service
+
+[Service]
+Type=oneshot
+ExecStart=/etc/set-jumbo-mtu.sh
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
+SVCEOF
+mkdir -p /etc/systemd/system/multi-user.target.wants
+ln -sf /etc/systemd/system/set-jumbo-mtu.service /etc/systemd/system/multi-user.target.wants/set-jumbo-mtu.service
+
+# Set pod MTU to match NIC MTU. MicroShift auto-detection fails in C2CC
+# VMs (no default route), falling back to 1500 regardless of NIC MTU.
+mkdir -p /etc/microshift
+echo "mtu: ${mtu}" > /etc/microshift/ovn.yaml
+EOF
+}
+
 c2cc_create_vms() {
     local -r boot_commit_ref="${1}"
     local -r boot_blueprint="${2}"
     local -r network="${3:-default}"
     local -r ip_family="${4:-ipv4}"
+    local -r network_mtu="${5:-}"
 
     # Prepare kickstart for all hosts
     # prepare_kickstart args: vmname template commit_ref fips_enabled ipv6_only
@@ -223,9 +319,19 @@ c2cc_create_vms() {
         "${CLUSTER_C_POD_CIDR}" "${CLUSTER_C_SVC_CIDR}" \
         "${CLUSTER_C_POD_CIDR_DUAL}" "${CLUSTER_C_SVC_CIDR_DUAL}"
 
-    launch_vm "${boot_blueprint}" --vmname host1 --network "${network}"
-    launch_vm "${boot_blueprint}" --vmname host2 --network "${network}"
-    launch_vm "${boot_blueprint}" --vmname host3 --network "${network}"
+    # Set the NIC MTU via NetworkManager in the kickstart so the guest boots
+    # with the correct MTU before MicroShift starts.
+    local mtu_args=()
+    if [ -n "${network_mtu}" ]; then
+        mtu_args=(--network_mtu "${network_mtu}")
+        for host in host1 host2 host3; do
+            inject_kickstart_mtu "${host}" "${network_mtu}"
+        done
+    fi
+
+    launch_vm "${boot_blueprint}" --vmname host1 --network "${network}" "${mtu_args[@]}"
+    launch_vm "${boot_blueprint}" --vmname host2 --network "${network}" "${mtu_args[@]}"
+    launch_vm "${boot_blueprint}" --vmname host3 --network "${network}" "${mtu_args[@]}"
 }
 
 c2cc_remove_vms() {
@@ -238,6 +344,7 @@ c2cc_run_tests() {
     local -r suites_dir="${1}"
     local -r foreign_cidr="${2:-}"
     local -r ip_family="${3:-}"
+    local -r extra_vars="${4:-}"
 
     local foreign_cidr_var=""
     if [ -n "${foreign_cidr}" ]; then
@@ -317,6 +424,7 @@ c2cc_run_tests() {
         ${ip_family_var} \
         ${target_ref_var} \
         ${dual_cidr_vars} \
+        ${extra_vars} \
         --variable "BOOTC_REGISTRY:${MIRROR_REGISTRY_URL}" \
         "${suites_dir}"
 }

test/bin/ci_phase_boot_and_test.sh

@@ -85,16 +85,30 @@ else
     progress=""
 fi
 
-# Limit amount of parallel scenarios for the C2CC jobs to avoid
-# over-provisioning the resources: each C2CC scenario runs 3 VMs
-# (2 vCPUs / 4GiB each) and the c7g.metal instance has 64 cores / 128GiB.
-# Powering off the VMs of passed scenarios makes the job limit an
-# actual cap on the number of running VMs.
+# Each C2CC scenario runs 3 VMs (2 vCPUs / 4GiB each)
+# and the c7g.metal instance has 64 cores / 128GiB.
 jobs_arg=""
 scenario_action="create-and-run"
 if [[ "${SCENARIO_SOURCES}" =~ c2cc ]]; then
+    # Limit amount of parallel scenarios for the C2CC jobs
+    # to avoid over-provisioning the resources.
     jobs_arg="-j 8"
+
+    # Power off passed VMs so the job limit is an actual cap on running VMs.
     scenario_action="create-run-shutdown"
+
+    # Each arch runs one RHEL version to halve the scenario count.
+    # The assignment rotates per commit so both combos get coverage:
+    #   flip=0: x86 → el98,  ARM → el102
+    #   flip=1: x86 → el102, ARM → el98
+    flip=$(( 16#$(git rev-parse HEAD | cut -c1) % 2 ))
+    if [[ "$(uname -m)" == "x86_64" ]]; then
+        delete_ver=(el102 el98)
+    else
+        delete_ver=(el98 el102)
+    fi
+    echo "C2CC arch split: deleting ${delete_ver[flip]}-* (flip=${flip})"
+    find "${SCENARIOS_TO_RUN}" -name "${delete_ver[flip]}-*.sh" -delete
 elif [[ "${SCENARIO_SOURCES}" =~ .*releases.* ]]; then
     # Release scenarios have grown (e.g. the router scenarios were split
     # from 1 into 5) and no longer fit if every scenario's VMs stay up for

test/bin/manage_hypervisor_config.sh

@@ -37,7 +37,7 @@ firewall_settings() {
     # Enable mDNS over libvirt network
     sudo firewall-cmd --permanent --zone=libvirt "--${action}-service=mdns"
 
-    for netname in default "${VM_ISOLATED_NETWORK}" "${VM_MULTUS_NETWORK}" "${VM_IPV6_NETWORK}" "${VM_DUAL_STACK_NETWORK}"; do
+    for netname in default "${VM_ISOLATED_NETWORK}" "${VM_MULTUS_NETWORK}" "${VM_IPV6_NETWORK}" "${VM_DUAL_STACK_NETWORK}" jumbo jumbo-ipv6; do
         if ! sudo virsh net-info "${netname}" &>/dev/null ; then
             continue
         fi
@@ -134,6 +134,20 @@ action_create() {
         sudo ip link add name "${bridge_name}p0" up master "${bridge_name}" type dummy
     fi
 
+    # Jumbo MTU network (IPv4)
+    if ! sudo virsh net-info jumbo &>/dev/null ; then
+        sudo virsh net-define    "${TESTDIR}/assets/network/jumbo-network.xml"
+        sudo virsh net-start     jumbo
+        sudo virsh net-autostart jumbo
+    fi
+
+    # Jumbo MTU network (IPv6)
+    if ! sudo virsh net-info jumbo-ipv6 &>/dev/null ; then
+        sudo virsh net-define    "${TESTDIR}/assets/network/jumbo-ipv6-network.xml"
+        sudo virsh net-start     jumbo-ipv6
+        sudo virsh net-autostart jumbo-ipv6
+    fi
+
     # Firewall
     firewall_settings "add"
 

test/bin/scenario.sh

@@ -804,6 +804,7 @@ EOF
 #           <boot_blueprint> \
 #           [--vmname <name>] \
 #           [--network <name>[,<name>...]] \
+#           [--network_mtu <size>] \
 #           [--vm_vcpus <vcpus>] \
 #           [--vm_memory <memory>] \
 #           [--vm_disksize <disksize>] \
@@ -820,6 +821,14 @@ EOF
 #   [--network <name>[,<name>...]]: A comma-separated list for the networks used
 #                                   when creating the VM. Each network entry will
 #                                   create a NIC and they are repeatable.
+#   [--network_mtu <size>]:         Sets the guest-visible MTU on every NIC via
+#                                   virt-install's mtu.size sub-option. Required
+#                                   in addition to a libvirt network's own <mtu>
+#                                   element — QEMU only negotiates a larger MTU
+#                                   with the guest's virtio-net driver when the
+#                                   domain's own <interface> XML requests it;
+#                                   the network-level MTU alone only affects the
+#                                   host-side bridge and tap devices.
 #   [--no_network]:                 Do not configure any network attachments (and
 #                                   therefore no NICs) for the VM.
 #   [--vm_vcpus <vcpus>]:           Number of vCPUs for the VM.
@@ -830,6 +839,7 @@ launch_vm() {
     # Set default values for the optional arguments
     local vmname="host1"
     local network="default"
+    local network_mtu=""
     local vm_memory=4096
     local vm_vcpus=2
     local vm_disksize=20
@@ -848,7 +858,7 @@ launch_vm() {
     # Parse the optional arguments
     while [ $# -gt 0 ]; do
         case "$1" in
-            --vmname|--vm_vcpus|--vm_memory|--vm_disksize)
+            --vmname|--vm_vcpus|--vm_memory|--vm_disksize|--network_mtu)
                 var="${1/--/}"
                 if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then
                     declare "${var}=$2"
@@ -960,6 +970,12 @@ launch_vm() {
         if sudo virsh nwfilter-list | awk '{print $2}' | grep -qx "${n}"; then
             vm_network_args+=",filterref=${n}"
         fi
+
+        # Propagate MTU to the domain interface (see --network_mtu doc above).
+        if [ -n "${network_mtu}" ]; then
+            vm_network_args+=",mtu.size=${network_mtu}"
+        fi
+
         vm_network_args+=" "
     done
     if [ -z "${vm_network_args}" ] ; then