From 0e26a8ad82b6a5c5743881196307b8e138b3117e Mon Sep 17 00:00:00 2001 From: Alejandro Gullon Date: Tue, 21 Jul 2026 21:09:48 +0200 Subject: [PATCH 1/5] feat: add jumbo MTU infrastructure for C2CC testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add infrastructure to support jumbo frame (MTU 9000) testing in C2CC scenarios: - nettest-pod: non-root pod for UDP DF-bit testing without NET_RAW - df-bit-send.py: sends DF-bit UDP datagrams using IP_PMTUDISC_DO (IPv4) or IPV6_DONTFRAG (IPv6), auto-detects address family - jumbo/jumbo-ipv6 libvirt networks created in manage_hypervisor_config - inject_kickstart_mtu(): configures guest NIC MTU and pod MTU (ovn.yaml) before first boot — OVN-K bakes the MTU into its database at initial creation - c2cc_setup_ipv6(): shared IPv6 CIDRs and mirror registry setup - launch_vm --network_mtu: propagates MTU to QEMU's virtio-net driver - Shared keywords: MTU verification, DF-bit testing, large payload transfers across all cluster pairs - Deploy nettest-pod alongside existing test workloads Co-Authored-By: Claude Opus 4.6 (1M context) pre-commit.check-secrets: ENABLED --- test/assets/c2cc/df-bit-send.py | 43 ++++++++ test/assets/c2cc/nettest-pod.yaml | 19 ++++ test/assets/network/jumbo-ipv6-network.xml | 14 +++ test/assets/network/jumbo-network.xml | 10 ++ test/bin/c2cc_common.sh | 114 ++++++++++++++++++++- test/bin/manage_hypervisor_config.sh | 16 ++- test/bin/scenario.sh | 18 +++- test/resources/c2cc.resource | 114 ++++++++++++++++++++- 8 files changed, 341 insertions(+), 7 deletions(-) create mode 100644 test/assets/c2cc/df-bit-send.py create mode 100644 test/assets/c2cc/nettest-pod.yaml create mode 100644 test/assets/network/jumbo-ipv6-network.xml create mode 100644 test/assets/network/jumbo-network.xml diff --git a/test/assets/c2cc/df-bit-send.py b/test/assets/c2cc/df-bit-send.py new file mode 100644 index 0000000000..84f4ac65eb --- /dev/null +++ b/test/assets/c2cc/df-bit-send.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Send a UDP datagram with the DF (Don't Fragment) bit set. + +Uses IP_PMTUDISC_DO (IPv4) or IPV6_DONTFRAG (IPv6) so the kernel +rejects datagrams that exceed the path MTU with EMSGSIZE instead of +fragmenting them. Auto-detects the address family from the +destination address. + +Uses UDP instead of ICMP to avoid requiring NET_RAW capability. +UDP overhead (IP+UDP) matches ICMP overhead (IP+ICMP): 28B for IPv4, +48B for IPv6, so payload sizes are equivalent to ping -s values +within each address family. + +Usage: python3 df-bit-send.py +Exit: prints "OK" on success, raises OSError on rejection. +""" + +import socket +import sys + +IPPROTO_IPV6 = 41 +IPV6_DONTFRAG = 62 +IP_MTU_DISCOVER = 10 +IP_PMTUDISC_DO = 2 + +dest = sys.argv[1] +size = int(sys.argv[2]) + +family = socket.AF_INET6 if ":" in dest else socket.AF_INET +sock = socket.socket(family, socket.SOCK_DGRAM) + +if family == socket.AF_INET6: + sock.setsockopt(IPPROTO_IPV6, IPV6_DONTFRAG, 1) +else: + sock.setsockopt(socket.IPPROTO_IP, IP_MTU_DISCOVER, IP_PMTUDISC_DO) + +try: + sock.sendto(b"A" * size, (dest, 9999)) + print("OK") +except OSError as e: + print(f"FAIL: {e}") +finally: + sock.close() diff --git a/test/assets/c2cc/nettest-pod.yaml b/test/assets/c2cc/nettest-pod.yaml new file mode 100644 index 0000000000..2dd1d0303c --- /dev/null +++ b/test/assets/c2cc/nettest-pod.yaml @@ -0,0 +1,19 @@ +kind: Pod +apiVersion: v1 +metadata: + name: nettest-pod +spec: + terminationGracePeriodSeconds: 0 + containers: + - name: nettest + image: registry.access.redhat.com/ubi9/ubi:9.6 + command: ["sleep", "infinity"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault diff --git a/test/assets/network/jumbo-ipv6-network.xml b/test/assets/network/jumbo-ipv6-network.xml new file mode 100644 index 0000000000..1f2f855d12 --- /dev/null +++ b/test/assets/network/jumbo-ipv6-network.xml @@ -0,0 +1,14 @@ + + jumbo-ipv6 + + + + + + + + + + + + diff --git a/test/assets/network/jumbo-network.xml b/test/assets/network/jumbo-network.xml new file mode 100644 index 0000000000..f4a038d340 --- /dev/null +++ b/test/assets/network/jumbo-network.xml @@ -0,0 +1,10 @@ + + jumbo + + + + + + + + diff --git a/test/bin/c2cc_common.sh b/test/bin/c2cc_common.sh index 9268aa2811..1d6819c135 100644 --- a/test/bin/c2cc_common.sh +++ b/test/bin/c2cc_common.sh @@ -27,6 +27,36 @@ CLUSTER_C_SVC_CIDR_DUAL="" export TEST_RANDOMIZATION=suites +# c2cc_setup_ipv6 overrides CIDRs and mirror registry for single-stack IPv6 +# scenarios. If a network name is provided, also sets VM_BRIDGE_IP and +# WEB_SERVER_URL for that network (standard IPv6 scenarios). Jumbo IPv6 +# scenarios omit the network arg and set VM_BRIDGE_IP later, after creating +# the jumbo-ipv6 network. +# shellcheck disable=SC2120 +c2cc_setup_ipv6() { + local -r network="${1:-}" + + # shellcheck disable=SC2034 # used by scenario.sh and kickstart templates + MIRROR_REGISTRY_URL="$(hostname):${MIRROR_REGISTRY_PORT}/microshift" + + CLUSTER_A_POD_CIDR="fd01::/48" + CLUSTER_A_SVC_CIDR="fd02::/112" + CLUSTER_A_DOMAIN="cluster-a.remote" + CLUSTER_B_POD_CIDR="fd04::/48" + CLUSTER_B_SVC_CIDR="fd05::/112" + CLUSTER_B_DOMAIN="cluster-b.remote" + CLUSTER_C_POD_CIDR="fd07::/48" + CLUSTER_C_SVC_CIDR="fd08::/112" + CLUSTER_C_DOMAIN="cluster-c.remote" + + if [[ -n "${network}" ]]; then + # shellcheck disable=SC2034 # used by scenario.sh + VM_BRIDGE_IP="$(get_vm_bridge_ip "${network}")" + # shellcheck disable=SC2034 # used by scenario.sh + WEB_SERVER_URL="http://[${VM_BRIDGE_IP}]:${WEB_SERVER_PORT}" + fi +} + get_host_ip() { local host=$1 get_vm_property "${host}" ip || { echo "failed to get ${host} ip" >&2; return 1; } @@ -188,11 +218,77 @@ ${network_yaml}IEOF EOF } +# inject_kickstart_mtu configures jumbo MTU on the guest VM via kickstart. +# The NIC and pod MTU must be set before MicroShift first starts — OVN-K +# bakes the MTU into its database at initial creation and does not update +# it on restart. Three mechanisms set the NIC MTU (NM conf.d, NM dispatcher, +# systemd oneshot) and ovn.yaml sets the pod MTU. +# Args: host mtu +inject_kickstart_mtu() { + local -r host="${1}" + local -r mtu="${2}" + + local -r ks_dir="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${host}" + cat >> "${ks_dir}/post-microshift.cfg" < /etc/NetworkManager/conf.d/99-jumbo-mtu.conf <<'NMCONF' +[connection-ethernet-jumbo] +match-device=type:ethernet +ethernet.mtu=${mtu} +NMCONF + +# NM dispatcher -- runs ip link set during connection activation +mkdir -p /etc/NetworkManager/dispatcher.d +cat > /etc/NetworkManager/dispatcher.d/99-jumbo-mtu <<'DISPEOF' +#!/bin/bash +[ "\$2" != "up" ] && exit 0 +ip link set dev "\$1" mtu ${mtu} 2>&1 || logger -t jumbo-mtu "Failed to set MTU ${mtu} on \$1" +DISPEOF +chmod 755 /etc/NetworkManager/dispatcher.d/99-jumbo-mtu +restorecon -R /etc/NetworkManager/dispatcher.d/ 2>&1 || logger -t jumbo-mtu "restorecon failed for dispatcher.d" + +# Systemd service -- runs before microshift.service. +# Script in /etc/ because only /etc/ and /var/ persist across reboots on bootc. +cat > /etc/set-jumbo-mtu.sh <<'SCRIPTEOF' +#!/bin/bash +for dev in \$(ip -o link show type ether | awk -F: '{print \$2}' | tr -d ' '); do + ip link set dev "\$dev" mtu ${mtu} +done +SCRIPTEOF +chmod 755 /etc/set-jumbo-mtu.sh +restorecon /etc/set-jumbo-mtu.sh 2>&1 || logger -t jumbo-mtu "restorecon failed for set-jumbo-mtu.sh" + +cat > /etc/systemd/system/set-jumbo-mtu.service <<'SVCEOF' +[Unit] +Description=Set jumbo frame MTU on ethernet interfaces +Before=microshift.service +After=NetworkManager-wait-online.service + +[Service] +Type=oneshot +ExecStart=/etc/set-jumbo-mtu.sh +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target +SVCEOF +mkdir -p /etc/systemd/system/multi-user.target.wants +ln -sf /etc/systemd/system/set-jumbo-mtu.service /etc/systemd/system/multi-user.target.wants/set-jumbo-mtu.service + +# Set pod MTU to match NIC MTU. MicroShift auto-detection fails in C2CC +# VMs (no default route), falling back to 1500 regardless of NIC MTU. +mkdir -p /etc/microshift +echo "mtu: ${mtu}" > /etc/microshift/ovn.yaml +EOF +} + c2cc_create_vms() { local -r boot_commit_ref="${1}" local -r boot_blueprint="${2}" local -r network="${3:-default}" local -r ip_family="${4:-ipv4}" + local -r network_mtu="${5:-}" # Prepare kickstart for all hosts # prepare_kickstart args: vmname template commit_ref fips_enabled ipv6_only @@ -223,9 +319,19 @@ c2cc_create_vms() { "${CLUSTER_C_POD_CIDR}" "${CLUSTER_C_SVC_CIDR}" \ "${CLUSTER_C_POD_CIDR_DUAL}" "${CLUSTER_C_SVC_CIDR_DUAL}" - launch_vm "${boot_blueprint}" --vmname host1 --network "${network}" - launch_vm "${boot_blueprint}" --vmname host2 --network "${network}" - launch_vm "${boot_blueprint}" --vmname host3 --network "${network}" + # Set the NIC MTU via NetworkManager in the kickstart so the guest boots + # with the correct MTU before MicroShift starts. + local mtu_args=() + if [ -n "${network_mtu}" ]; then + mtu_args=(--network_mtu "${network_mtu}") + for host in host1 host2 host3; do + inject_kickstart_mtu "${host}" "${network_mtu}" + done + fi + + launch_vm "${boot_blueprint}" --vmname host1 --network "${network}" "${mtu_args[@]}" + launch_vm "${boot_blueprint}" --vmname host2 --network "${network}" "${mtu_args[@]}" + launch_vm "${boot_blueprint}" --vmname host3 --network "${network}" "${mtu_args[@]}" } c2cc_remove_vms() { @@ -238,6 +344,7 @@ c2cc_run_tests() { local -r suites_dir="${1}" local -r foreign_cidr="${2:-}" local -r ip_family="${3:-}" + local -r extra_vars="${4:-}" local foreign_cidr_var="" if [ -n "${foreign_cidr}" ]; then @@ -317,6 +424,7 @@ c2cc_run_tests() { ${ip_family_var} \ ${target_ref_var} \ ${dual_cidr_vars} \ + ${extra_vars} \ --variable "BOOTC_REGISTRY:${MIRROR_REGISTRY_URL}" \ "${suites_dir}" } diff --git a/test/bin/manage_hypervisor_config.sh b/test/bin/manage_hypervisor_config.sh index 10daee0a0e..806a628cc4 100755 --- a/test/bin/manage_hypervisor_config.sh +++ b/test/bin/manage_hypervisor_config.sh @@ -37,7 +37,7 @@ firewall_settings() { # Enable mDNS over libvirt network sudo firewall-cmd --permanent --zone=libvirt "--${action}-service=mdns" - for netname in default "${VM_ISOLATED_NETWORK}" "${VM_MULTUS_NETWORK}" "${VM_IPV6_NETWORK}" "${VM_DUAL_STACK_NETWORK}"; do + for netname in default "${VM_ISOLATED_NETWORK}" "${VM_MULTUS_NETWORK}" "${VM_IPV6_NETWORK}" "${VM_DUAL_STACK_NETWORK}" jumbo jumbo-ipv6; do if ! sudo virsh net-info "${netname}" &>/dev/null ; then continue fi @@ -134,6 +134,20 @@ action_create() { sudo ip link add name "${bridge_name}p0" up master "${bridge_name}" type dummy fi + # Jumbo MTU network (IPv4) + if ! sudo virsh net-info jumbo &>/dev/null ; then + sudo virsh net-define "${TESTDIR}/assets/network/jumbo-network.xml" + sudo virsh net-start jumbo + sudo virsh net-autostart jumbo + fi + + # Jumbo MTU network (IPv6) + if ! sudo virsh net-info jumbo-ipv6 &>/dev/null ; then + sudo virsh net-define "${TESTDIR}/assets/network/jumbo-ipv6-network.xml" + sudo virsh net-start jumbo-ipv6 + sudo virsh net-autostart jumbo-ipv6 + fi + # Firewall firewall_settings "add" diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index cef81e6be3..9071a1559b 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -804,6 +804,7 @@ EOF # \ # [--vmname ] \ # [--network [,...]] \ +# [--network_mtu ] \ # [--vm_vcpus ] \ # [--vm_memory ] \ # [--vm_disksize ] \ @@ -820,6 +821,14 @@ EOF # [--network [,...]]: A comma-separated list for the networks used # when creating the VM. Each network entry will # create a NIC and they are repeatable. +# [--network_mtu ]: Sets the guest-visible MTU on every NIC via +# virt-install's mtu.size sub-option. Required +# in addition to a libvirt network's own +# element — QEMU only negotiates a larger MTU +# with the guest's virtio-net driver when the +# domain's own XML requests it; +# the network-level MTU alone only affects the +# host-side bridge and tap devices. # [--no_network]: Do not configure any network attachments (and # therefore no NICs) for the VM. # [--vm_vcpus ]: Number of vCPUs for the VM. @@ -830,6 +839,7 @@ launch_vm() { # Set default values for the optional arguments local vmname="host1" local network="default" + local network_mtu="" local vm_memory=4096 local vm_vcpus=2 local vm_disksize=20 @@ -848,7 +858,7 @@ launch_vm() { # Parse the optional arguments while [ $# -gt 0 ]; do case "$1" in - --vmname|--vm_vcpus|--vm_memory|--vm_disksize) + --vmname|--vm_vcpus|--vm_memory|--vm_disksize|--network_mtu) var="${1/--/}" if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then declare "${var}=$2" @@ -960,6 +970,12 @@ launch_vm() { if sudo virsh nwfilter-list | awk '{print $2}' | grep -qx "${n}"; then vm_network_args+=",filterref=${n}" fi + + # Propagate MTU to the domain interface (see --network_mtu doc above). + if [ -n "${network_mtu}" ]; then + vm_network_args+=",mtu.size=${network_mtu}" + fi + vm_network_args+=" " done if [ -z "${vm_network_args}" ] ; then diff --git a/test/resources/c2cc.resource b/test/resources/c2cc.resource index 5d20fb6514..bc31891c7a 100644 --- a/test/resources/c2cc.resource +++ b/test/resources/c2cc.resource @@ -13,6 +13,7 @@ Resource microshift-host.resource *** Variables *** +${DF_SCRIPT} ${EXECDIR}/assets/c2cc/df-bit-send.py ${CLUSTER_A_POD_CIDR} ${EMPTY} ${CLUSTER_A_SVC_CIDR} ${EMPTY} ${CLUSTER_A_DOMAIN} ${EMPTY} @@ -299,13 +300,14 @@ Verify Corefile Does Not Contain C2CC Server Block Should Not Contain ${stdout} ${domain}:5353 Deploy Test Workloads - [Documentation] Create namespace and deploy hello-microshift + curl-pod on all clusters. + [Documentation] Create namespace and deploy hello-microshift, curl-pod, and nettest-pod on all clusters. VAR ${assets}= ${EXECDIR}/assets/c2cc FOR ${alias} IN cluster-a cluster-b cluster-c ${ns}= Create Unique Namespace On Cluster ${alias} Set To Dictionary ${NAMESPACES} ${alias} ${ns} Oc On Cluster ${alias} oc apply -n ${ns} -f ${assets}/hello-microshift.yaml Oc On Cluster ${alias} oc apply -n ${ns} -f ${assets}/curl-pod.yaml + Oc On Cluster ${alias} oc apply -n ${ns} -f ${assets}/nettest-pod.yaml END Wait For Test Pods Wait For Service Endpoints @@ -315,7 +317,7 @@ Wait For Test Pods FOR ${alias} IN cluster-a cluster-b cluster-c Oc On Cluster ... ${alias} - ... oc wait pod/hello-microshift pod/curl-pod -n ${NAMESPACES}[${alias}] --for=condition=Ready --timeout=120s + ... oc wait pod/hello-microshift pod/curl-pod pod/nettest-pod -n ${NAMESPACES}[${alias}] --for=condition=Ready --timeout=120s END Wait For Service Endpoints @@ -800,3 +802,111 @@ Verify Full C2CC Stack END Wait Until Keyword Succeeds 10m 10s Verify C2CC Health Probes Wait Until Keyword Succeeds 10m 10s Verify Full C2CC DNS + +Restart MicroShift On All Clusters + [Documentation] Restart MicroShift on all clusters and wait for health. + FOR ${alias} IN @{ALL_CLUSTERS} + Command On Cluster ${alias} systemctl restart --no-block microshift + END + FOR ${alias} IN @{ALL_CLUSTERS} + Wait Until Keyword Succeeds 5m 15s + ... Command On Cluster ${alias} microshift healthcheck --timeout=5m + END + +Verify Pod MTU On All Clusters + [Documentation] Verify the pod network interface reports the expected MTU. + [Arguments] ${expected_mtu} + FOR ${alias} IN @{ALL_CLUSTERS} + ${mtu}= Get Pod Interface MTU ${alias} nettest-pod ${NAMESPACES}[${alias}] + Should Be Equal As Strings ${mtu} ${expected_mtu} + ... ${alias}: pod MTU is ${mtu}, expected ${expected_mtu} + END + +Ping With DF Bit And Verify + [Documentation] Send a DF-bit UDP datagram via df-bit-send.py and assert success. + ... Auto-detects IPv4/IPv6. See the script for protocol details. + [Arguments] ${alias} ${dest_ip} ${payload_size} + ${stdout}= Oc On Cluster + ... ${alias} + ... oc exec -i nettest-pod -n ${NAMESPACES}[${alias}] -- python3 - ${dest_ip} ${payload_size} < ${DF_SCRIPT} 2>&1 + ... allow_fail=${TRUE} + Should Contain ${stdout} OK + ... DF-bit UDP send of ${payload_size}B to ${dest_ip} failed: ${stdout} + +Ping With DF Bit Should Fail + [Documentation] Send a UDP datagram with DF bit set. Asserts EMSGSIZE (Message too long). + ... Proves the datagram exceeds the path MTU. Auto-detects IPv4 vs + ... IPv6 from the destination address. Checks for the specific EMSGSIZE error + ... to distinguish MTU rejection from other failures. + [Arguments] ${alias} ${dest_ip} ${payload_size} + ${stdout}= Oc On Cluster + ... ${alias} + ... oc exec -i nettest-pod -n ${NAMESPACES}[${alias}] -- python3 - ${dest_ip} ${payload_size} < ${DF_SCRIPT} 2>&1 || true + ... allow_fail=${TRUE} + Should Not Contain ${stdout} OK + ... DF-bit UDP send of ${payload_size}B should have been rejected but succeeded + Should Contain ${stdout} Message too long + ... DF-bit UDP send of ${payload_size}B expected EMSGSIZE but got: ${stdout} + +Get Pod Interface MTU + [Documentation] Return the MTU of the pod's eth0 interface. + [Arguments] ${alias} ${pod} ${ns} + ${stdout}= Oc On Cluster ${alias} + ... oc exec ${pod} -n ${ns} -- cat /sys/class/net/eth0/mtu + ${mtu}= Strip String ${stdout} + RETURN ${mtu} + +Send Large Payload And Verify + [Documentation] Send a large payload via curl POST and verify it succeeds. + [Arguments] ${alias} ${ip} ${size} + ${is_ipv6}= Evaluate ':' in '''${ip}''' + ${url}= Set Variable If + ... ${is_ipv6} + ... http://[${ip}]:8080/cgi-bin/hello + ... http://${ip}:8080/cgi-bin/hello + ${stdout}= Oc On Cluster + ... ${alias} + ... oc exec curl-pod -n ${NAMESPACES}[${alias}] -- sh -c 'head -c ${size} /dev/zero | curl -sS --max-time 15 --data-binary @- ${url}' + Should Contain ${stdout} Hello from + +Ping DF Bit Between All Clusters + [Documentation] Send DF-bit UDP payload across all 6 cluster pairs. + [Arguments] ${size} + FOR ${src} ${dst} IN + ... cluster-a cluster-b + ... cluster-a cluster-c + ... cluster-b cluster-a + ... cluster-b cluster-c + ... cluster-c cluster-a + ... cluster-c cluster-b + ${ip}= Get Hello Pod IP ${dst} + Ping With DF Bit And Verify ${src} ${ip} ${size} + END + +Ping DF Bit Should Fail Between All Clusters + [Documentation] Send DF-bit UDP payload expecting rejection on all 6 pairs. + [Arguments] ${size} + FOR ${src} ${dst} IN + ... cluster-a cluster-b + ... cluster-a cluster-c + ... cluster-b cluster-a + ... cluster-b cluster-c + ... cluster-c cluster-a + ... cluster-c cluster-b + ${ip}= Get Hello Pod IP ${dst} + Ping With DF Bit Should Fail ${src} ${ip} ${size} + END + +Large Payload Between All Clusters + [Documentation] Send large TCP payload across all 6 cluster pairs. + [Arguments] ${size} + FOR ${src} ${dst} IN + ... cluster-a cluster-b + ... cluster-a cluster-c + ... cluster-b cluster-a + ... cluster-b cluster-c + ... cluster-c cluster-a + ... cluster-c cluster-b + ${ip}= Get Hello Pod IP ${dst} + Send Large Payload And Verify ${src} ${ip} ${size} + END From e2f8a092beab83b94c606ca211d303c5ba7ae1d5 Mon Sep 17 00:00:00 2001 From: Alejandro Gullon Date: Tue, 21 Jul 2026 21:09:52 +0200 Subject: [PATCH 2/5] feat: add IPv6 support and ESP verification to IPsec keywords - Fix nftables enforcement for IPv6 (ip6 daddr vs ip daddr) - Fix curl URLs for IPv6 (bracket notation) - Add curl failure mode assertions (timeout/refused/unreachable) - Add Verify ESP Encapsulation On All Clusters keyword Co-Authored-By: Claude Opus 4.6 (1M context) pre-commit.check-secrets: ENABLED --- test/resources/ipsec.resource | 37 +++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/test/resources/ipsec.resource b/test/resources/ipsec.resource index 7aa5cbc7e6..37faee2aae 100644 --- a/test/resources/ipsec.resource +++ b/test/resources/ipsec.resource @@ -100,11 +100,14 @@ Add NFTables IPsec Enforcement Rules ... to the host but forwarded into the OVN network (local gateway mode), so an ... input-hook chain would never see them. [Arguments] ${alias} ${local_pod_cidr} + ${is_ipv6}= Evaluate ':' in '''${local_pod_cidr}''' + ${daddr_family}= Set Variable If ${is_ipv6} ip6 ip Command On Cluster ${alias} nft add table inet c2cc_ipsec_test Command On Cluster ${alias} ... nft 'add chain inet c2cc_ipsec_test enforce { type filter hook forward priority -150; policy accept; }' - Command On Cluster ${alias} - ... nft add rule inet c2cc_ipsec_test enforce ip daddr ${local_pod_cidr} meta ipsec missing counter drop + Command On Cluster + ... ${alias} + ... nft add rule inet c2cc_ipsec_test enforce ${daddr_family} daddr ${local_pod_cidr} meta ipsec missing counter drop Remove NFTables IPsec Enforcement Rules [Documentation] Remove C2CC IPsec enforcement nftables table and all its rules. @@ -115,17 +118,43 @@ Remove NFTables IPsec Enforcement Rules Curl Should Fail From Cluster [Documentation] Verify curl from curl-pod times out or fails (no "Hello from" in response). + ... Asserts a recognizable curl error to distinguish IPsec blocking from broken infra. [Arguments] ${alias} ${ip} ${port} ${ns} + ${is_ipv6}= Evaluate ':' in '''${ip}''' + ${url}= Set Variable If + ... ${is_ipv6} + ... http://[${ip}]:${port}/cgi-bin/hello + ... http://${ip}:${port}/cgi-bin/hello ${stdout}= Oc On Cluster ${alias} - ... oc exec curl-pod -n ${ns} -- curl -sS --max-time 5 http://${ip}:${port}/cgi-bin/hello 2>&1 || true + ... oc exec curl-pod -n ${ns} -- curl -sS --max-time 5 ${url} 2>&1 || true ... allow_fail=${TRUE} Should Not Contain ${stdout} Hello from + Should Match Regexp ${stdout} (?i)(timed out|curl:|refused|unreachable) + ... Expected curl timeout or connection error but got: ${stdout} Curl From Host Should Fail [Documentation] Curl directly from the host (not from a pod) to a remote pod IP. ... Expects failure — host-originated traffic is not matched by tunnel-mode ... IPsec selectors scoped to pod/service CIDRs. + ... Asserts a recognizable curl error to distinguish from broken infra. [Arguments] ${alias} ${pod_ip} ${port} + ${is_ipv6}= Evaluate ':' in '''${pod_ip}''' + ${url}= Set Variable If + ... ${is_ipv6} + ... http://[${pod_ip}]:${port}/cgi-bin/hello + ... http://${pod_ip}:${port}/cgi-bin/hello ${stdout}= Disruptive Command On Cluster ${alias} - ... curl -sS --max-time 5 http://${pod_ip}:${port}/cgi-bin/hello 2>&1 || true + ... curl -sS --max-time 5 ${url} 2>&1 || true Should Not Contain ${stdout} Hello from + Should Match Regexp ${stdout} (?i)(timed out|curl:|refused|unreachable) + ... Expected curl timeout or connection error but got: ${stdout} + +Verify ESP Encapsulation On All Clusters + [Documentation] Record XFRM byte counters, send traffic, verify counters incremented. + ${baseline_a}= Get XFRM Byte Counters cluster-a + ${baseline_b}= Get XFRM Byte Counters cluster-b + ${baseline_c}= Get XFRM Byte Counters cluster-c + Verify Full C2CC Connectivity + Verify XFRM Counters Incremented cluster-a ${baseline_a} + Verify XFRM Counters Incremented cluster-b ${baseline_b} + Verify XFRM Counters Incremented cluster-c ${baseline_c} From dd4d5c155499ac9eb2c1231de25be5f2ff161e24 Mon Sep 17 00:00:00 2001 From: Alejandro Gullon Date: Tue, 21 Jul 2026 21:09:55 +0200 Subject: [PATCH 3/5] feat: add IPsec E2E test suite with DF-bit boundary tests Add ipsec.robot (10 tests) covering ESP encapsulation, cross-cluster connectivity, source IP preservation, policy enforcement, plaintext rejection, host-to-pod rejection, MTU boundary and TCP transfers at 1500, DNS resolution, and tunnel recovery after restart. Move ipsec.robot from suites/c2cc/ipsec/ to suites/c2cc/extra/ and rename c2cc-ipsec scenarios to c2cc-ipsec-ipv4 for consistency with new c2cc-ipsec-ipv6 variants that run the same suite over IPv6. Co-Authored-By: Claude Opus 4.6 (1M context) pre-commit.check-secrets: ENABLED --- ...-ipsec.sh => el102-src@c2cc-ipsec-ipv4.sh} | 3 +- .../c2cc/el102-src@c2cc-ipsec-ipv6.sh | 30 ++++++++++++ ...c-ipsec.sh => el98-src@c2cc-ipsec-ipv4.sh} | 3 +- .../c2cc/el98-src@c2cc-ipsec-ipv6.sh | 30 ++++++++++++ test/suites/c2cc/{ipsec => extra}/ipsec.robot | 46 ++++++------------- 5 files changed, 75 insertions(+), 37 deletions(-) rename test/scenarios-bootc/c2cc/{el102-src@c2cc-ipsec.sh => el102-src@c2cc-ipsec-ipv4.sh} (92%) create mode 100644 test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec-ipv6.sh rename test/scenarios-bootc/c2cc/{el98-src@c2cc-ipsec.sh => el98-src@c2cc-ipsec-ipv4.sh} (92%) create mode 100644 test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec-ipv6.sh rename test/suites/c2cc/{ipsec => extra}/ipsec.robot (78%) diff --git a/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec.sh b/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec-ipv4.sh similarity index 92% rename from test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec.sh rename to test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec-ipv4.sh index 2c6e13d50d..58f5c5234b 100644 --- a/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec.sh +++ b/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec-ipv4.sh @@ -12,7 +12,6 @@ source "${SCRIPTDIR}/c2cc_common.sh" # IPsec tests have ordering dependencies (setup verification must pass before # enforcement tests), so disable randomization. -export TEST_RANDOMIZATION=none scenario_create_vms() { c2cc_create_vms "rhel102-bootc-source-ipsec" "rhel102-bootc" @@ -26,5 +25,5 @@ scenario_run_tests() { configure_c2cc_hosts "c2cc_ipsec_pre_greenboot" "c2cc_ipsec_greenboot" configure_ipsec - c2cc_run_tests "suites/c2cc/ipsec/" + c2cc_run_tests "suites/c2cc/extra/ipsec.robot" } diff --git a/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec-ipv6.sh b/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec-ipv6.sh new file mode 100644 index 0000000000..2ab214d531 --- /dev/null +++ b/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec-ipv6.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. +# +# Sets up 3 MicroShift clusters with C2CC and Libreswan IPsec (tunnel mode +# protecting pod/service CIDRs) on a single-stack IPv6 network. Same test +# suite as c2cc-ipsec, run over IPv6 instead of IPv4. + +# shellcheck source=test/bin/c2cc_common.sh +source "${SCRIPTDIR}/c2cc_common.sh" + +# IPsec tests have ordering dependencies (setup verification must pass before +# enforcement tests), so disable randomization. + +c2cc_setup_ipv6 "${VM_IPV6_NETWORK}" + +scenario_create_vms() { + c2cc_create_vms "rhel102-bootc-source-ipsec" "rhel102-bootc" "${VM_IPV6_NETWORK}" ipv6 +} + +scenario_remove_vms() { + c2cc_remove_vms +} + +scenario_run_tests() { + configure_c2cc_hosts "c2cc_ipsec_pre_greenboot" "c2cc_ipsec_greenboot" + configure_ipsec + + c2cc_run_tests "suites/c2cc/extra/ipsec.robot" "" ipv6 +} diff --git a/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec.sh b/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec-ipv4.sh similarity index 92% rename from test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec.sh rename to test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec-ipv4.sh index e9900c4c7a..e056ce3f9f 100644 --- a/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec.sh +++ b/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec-ipv4.sh @@ -12,7 +12,6 @@ source "${SCRIPTDIR}/c2cc_common.sh" # IPsec tests have ordering dependencies (setup verification must pass before # enforcement tests), so disable randomization. -export TEST_RANDOMIZATION=none scenario_create_vms() { c2cc_create_vms "rhel98-bootc-source-ipsec" "rhel98-bootc" @@ -26,5 +25,5 @@ scenario_run_tests() { configure_c2cc_hosts "c2cc_ipsec_pre_greenboot" "c2cc_ipsec_greenboot" configure_ipsec - c2cc_run_tests "suites/c2cc/ipsec/" + c2cc_run_tests "suites/c2cc/extra/ipsec.robot" } diff --git a/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec-ipv6.sh b/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec-ipv6.sh new file mode 100644 index 0000000000..b7a9b78ac4 --- /dev/null +++ b/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec-ipv6.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. +# +# Sets up 3 MicroShift clusters with C2CC and Libreswan IPsec (tunnel mode +# protecting pod/service CIDRs) on a single-stack IPv6 network. Same test +# suite as c2cc-ipsec, run over IPv6 instead of IPv4. + +# shellcheck source=test/bin/c2cc_common.sh +source "${SCRIPTDIR}/c2cc_common.sh" + +# IPsec tests have ordering dependencies (setup verification must pass before +# enforcement tests), so disable randomization. + +c2cc_setup_ipv6 "${VM_IPV6_NETWORK}" + +scenario_create_vms() { + c2cc_create_vms "rhel98-bootc-source-ipsec" "rhel98-bootc" "${VM_IPV6_NETWORK}" ipv6 +} + +scenario_remove_vms() { + c2cc_remove_vms +} + +scenario_run_tests() { + configure_c2cc_hosts "c2cc_ipsec_pre_greenboot" "c2cc_ipsec_greenboot" + configure_ipsec + + c2cc_run_tests "suites/c2cc/extra/ipsec.robot" "" ipv6 +} diff --git a/test/suites/c2cc/ipsec/ipsec.robot b/test/suites/c2cc/extra/ipsec.robot similarity index 78% rename from test/suites/c2cc/ipsec/ipsec.robot rename to test/suites/c2cc/extra/ipsec.robot index 04717e4e76..357e8ee201 100644 --- a/test/suites/c2cc/ipsec/ipsec.robot +++ b/test/suites/c2cc/extra/ipsec.robot @@ -5,7 +5,8 @@ Documentation IPsec E2E tests for C2CC. ... ... Tests cover ESP encapsulation, connectivity through the tunnel, ... source IP preservation, policy enforcement (SA flush/restore), -... plaintext rejection, host-to-pod rejection, and MTU validation. +... plaintext rejection, host-to-pod rejection, MTU boundary (1500), +... and tunnel recovery. Jumbo MTU tests are in the c2cc-ipsec-mtu scenario. Resource ../../../resources/microshift-process.resource Resource ../../../resources/kubeconfig.resource @@ -28,20 +29,12 @@ IPsec Tunnels Established On All Clusters Verify All IPsec Tunnels On Cluster cluster-c expected_count=8 ESP Encapsulation On Wire - [Documentation] Capture packets on the wire and verify ESP encapsulation. - ... Records XFRM byte counters before and after traffic, captures ESP packets - ... via tcpdump, and verifies counters incremented. - ${baseline_a}= Get XFRM Byte Counters cluster-a - ${baseline_b}= Get XFRM Byte Counters cluster-b - + [Documentation] Verify all cross-cluster traffic is ESP-encapsulated by + ... capturing packets via tcpdump and checking XFRM counters. ${pcap_file}= Start Tcpdump For ESP On Cluster cluster-b - ${ip_dest}= Get Hello Pod IP cluster-b - Curl From Cluster cluster-a ${ip_dest} 8080 + Verify ESP Encapsulation On All Clusters Wait For Tcpdump And Verify ESP cluster-b ${pcap_file} - Verify XFRM Counters Incremented cluster-a ${baseline_a} - Verify XFRM Counters Incremented cluster-b ${baseline_b} - Cross Cluster Connectivity Through IPsec [Documentation] Verify pods on all clusters can reach pods/services on all ... other clusters through the IPsec tunnel. @@ -95,13 +88,14 @@ Host To Remote Pod Rejected Without IPsec ${pod_ip}= Get Hello Pod IP cluster-b Curl From Host Should Fail cluster-a ${pod_ip} 8080 -Near MTU Packet Through IPsec Tunnel - [Documentation] Send near-MTU-sized payloads through IPsec tunnel-mode - ... encapsulation. Verifies no MTU blackhole from DF-bit issues. - ... ESP overhead ~36-52B total. - ${ip_dest}= Get Hello Pod IP cluster-b - Send Large Payload And Verify cluster-a ${ip_dest} 1300 - Send Large Payload And Verify cluster-a ${ip_dest} 1400 +MTU Boundary And TCP Transfer Through IPsec + [Documentation] Validate DF-bit boundaries at the ESP-adjusted PMTU and + ... large TCP transfers through IPsec on all cluster pairs. + Ping DF Bit Between All Clusters 64 + Ping DF Bit Between All Clusters 1450 + Ping DF Bit Should Fail Between All Clusters 1472 + Large Payload Between All Clusters 1400 + Large Payload Between All Clusters 65536 Cross Cluster DNS Through IPsec [Documentation] Verify pods can resolve and reach services on remote @@ -150,23 +144,9 @@ Ensure IPsec Running On All Clusters Start IPsec Service On Cluster ${alias} END -Restore IPsec And Verify - [Documentation] Restart ipsec and wait for tunnels to come back up. - [Arguments] ${alias} - Restart IPsec Service On Cluster ${alias} - Wait For IPsec Tunnel Reestablishment ${alias} expected_count=8 - Restore IPsec With Enforcement Cleanup [Documentation] Remove enforcement rules and restore IPsec. [Arguments] ${ipsec_alias} ${enforcement_alias} Remove NFTables IPsec Enforcement Rules ${enforcement_alias} Start IPsec Service On Cluster ${ipsec_alias} Wait For IPsec Tunnel Reestablishment ${ipsec_alias} expected_count=8 - -Send Large Payload And Verify - [Documentation] Send a large payload via curl POST and verify it succeeds. - [Arguments] ${alias} ${ip} ${size} - ${stdout}= Oc On Cluster - ... ${alias} - ... oc exec curl-pod -n ${NAMESPACES}[${alias}] -- sh -c 'dd if=/dev/zero bs=${size} count=1 2>/dev/null | curl -sS --max-time 15 --data-binary @- http://${ip}:8080/cgi-bin/hello' - Should Contain ${stdout} Hello from From 8f36b72fa1190bdabb7e5115978d4e494d93b742 Mon Sep 17 00:00:00 2001 From: Alejandro Gullon Date: Tue, 21 Jul 2026 21:09:57 +0200 Subject: [PATCH 4/5] feat: add jumbo MTU test suite for C2CC and C2CC+IPsec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add mtu.robot (1 test) validating MTU boundary behavior at jumbo frame size (9000 MTU). Shared by both plain C2CC and IPsec scenarios via the IPSEC variable — when "true", also verifies ESP encapsulation. Pod MTU 9000 is set via kickstart ovn.yaml (OVN-K bakes the MTU into its database at initial creation and does not update on restart). Includes IPv4 and IPv6 scenario scripts for both el98 and el102, with and without IPsec (8 scenario files total). Co-Authored-By: Claude Opus 4.6 (1M context) pre-commit.check-secrets: ENABLED --- .../c2cc/el102-src@c2cc-ipsec-mtu-ipv4.sh | 27 ++++++++++++ .../c2cc/el102-src@c2cc-ipsec-mtu-ipv6.sh | 35 +++++++++++++++ .../c2cc/el102-src@c2cc-ipv6.sh | 27 +----------- .../c2cc/el102-src@c2cc-mtu-ipv4.sh | 24 +++++++++++ .../c2cc/el102-src@c2cc-mtu-ipv6.sh | 33 ++++++++++++++ .../c2cc/el98-src@c2cc-ipsec-mtu-ipv4.sh | 27 ++++++++++++ .../c2cc/el98-src@c2cc-ipsec-mtu-ipv6.sh | 35 +++++++++++++++ .../c2cc/el98-src@c2cc-ipv6.sh | 27 +----------- .../c2cc/el98-src@c2cc-mtu-ipv4.sh | 24 +++++++++++ .../c2cc/el98-src@c2cc-mtu-ipv6.sh | 33 ++++++++++++++ test/suites/c2cc/extra/mtu.robot | 43 +++++++++++++++++++ 11 files changed, 283 insertions(+), 52 deletions(-) create mode 100644 test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec-mtu-ipv4.sh create mode 100644 test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec-mtu-ipv6.sh create mode 100644 test/scenarios-bootc/c2cc/el102-src@c2cc-mtu-ipv4.sh create mode 100644 test/scenarios-bootc/c2cc/el102-src@c2cc-mtu-ipv6.sh create mode 100644 test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec-mtu-ipv4.sh create mode 100644 test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec-mtu-ipv6.sh create mode 100644 test/scenarios-bootc/c2cc/el98-src@c2cc-mtu-ipv4.sh create mode 100644 test/scenarios-bootc/c2cc/el98-src@c2cc-mtu-ipv6.sh create mode 100644 test/suites/c2cc/extra/mtu.robot diff --git a/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec-mtu-ipv4.sh b/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec-mtu-ipv4.sh new file mode 100644 index 0000000000..a51a43b99e --- /dev/null +++ b/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec-mtu-ipv4.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. +# +# Sets up 3 MicroShift clusters with C2CC and Libreswan IPsec on a +# jumbo-frame network (MTU 9000). Tests validate MTU boundary behavior +# through IPsec tunnel-mode ESP encapsulation and the recommended +# pod MTU reduction to 8900. + +# shellcheck source=test/bin/c2cc_common.sh +source "${SCRIPTDIR}/c2cc_common.sh" + + +scenario_create_vms() { + c2cc_create_vms "rhel102-bootc-source-ipsec" "rhel102-bootc" "jumbo" "ipv4" "9000" +} + +scenario_remove_vms() { + c2cc_remove_vms +} + +scenario_run_tests() { + configure_c2cc_hosts "c2cc_ipsec_mtu_pre_greenboot" "c2cc_ipsec_mtu_greenboot" + configure_ipsec + + c2cc_run_tests "suites/c2cc/extra/mtu.robot" "" "" "--variable IPSEC:true" +} diff --git a/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec-mtu-ipv6.sh b/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec-mtu-ipv6.sh new file mode 100644 index 0000000000..f9f650995e --- /dev/null +++ b/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec-mtu-ipv6.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. +# +# Sets up 3 MicroShift clusters with C2CC and Libreswan IPsec on a +# single-stack IPv6 jumbo-frame network (MTU 9000). Same test suite as +# c2cc-ipsec-mtu, run over IPv6 instead of IPv4. + +# shellcheck source=test/bin/c2cc_common.sh +source "${SCRIPTDIR}/c2cc_common.sh" + + +# shellcheck disable=SC2119 +c2cc_setup_ipv6 + +scenario_create_vms() { + # Read jumbo-ipv6 bridge IP for /etc/hosts resolution in VMs. + # Keep WEB_SERVER_URL at the default IPv4 — bootc kickstarts don't + # use REPLACE_OSTREE_SERVER_URL, and the hypervisor-side curl can't + # reach the IPv6 bridge due to libvirt's nftables filtering. + # shellcheck disable=SC2034 # used elsewhere + VM_BRIDGE_IP="$(get_vm_bridge_ip "jumbo-ipv6")" + c2cc_create_vms "rhel102-bootc-source-ipsec" "rhel102-bootc" "jumbo-ipv6" "ipv6" "9000" +} + +scenario_remove_vms() { + c2cc_remove_vms +} + +scenario_run_tests() { + configure_c2cc_hosts "c2cc_ipsec_mtu_pre_greenboot" "c2cc_ipsec_mtu_greenboot" + configure_ipsec + + c2cc_run_tests "suites/c2cc/extra/mtu.robot" "" ipv6 "--variable IPSEC:true" +} diff --git a/test/scenarios-bootc/c2cc/el102-src@c2cc-ipv6.sh b/test/scenarios-bootc/c2cc/el102-src@c2cc-ipv6.sh index 93040a4bdd..4ecb14282a 100644 --- a/test/scenarios-bootc/c2cc/el102-src@c2cc-ipv6.sh +++ b/test/scenarios-bootc/c2cc/el102-src@c2cc-ipv6.sh @@ -5,32 +5,7 @@ # shellcheck source=test/bin/c2cc_common.sh source "${SCRIPTDIR}/c2cc_common.sh" -# Redefine network-related settings to use the dedicated IPv6 network bridge -# shellcheck disable=SC2034 # used elsewhere -VM_BRIDGE_IP="$(get_vm_bridge_ip "${VM_IPV6_NETWORK}")" -# shellcheck disable=SC2034 # used elsewhere -WEB_SERVER_URL="http://[${VM_BRIDGE_IP}]:${WEB_SERVER_PORT}" -# Using `hostname` here instead of a raw ip because skopeo only allows either -# ipv4 or fqdn's, but not ipv6. Since the registry is hosted on the ipv6 -# network gateway in the host, we need to use a combination of the hostname -# plus /etc/hosts resolution (which is taken care of by kickstart). -# shellcheck disable=SC2034 # used elsewhere -MIRROR_REGISTRY_URL="$(hostname):${MIRROR_REGISTRY_PORT}/microshift" - -# Cluster A (host1): non-overlapping CIDRs -CLUSTER_A_POD_CIDR="fd01::/48" -CLUSTER_A_SVC_CIDR="fd02::/112" -CLUSTER_A_DOMAIN="cluster-a.remote" - -# Cluster B (host2): non-overlapping CIDRs -CLUSTER_B_POD_CIDR="fd04::/48" -CLUSTER_B_SVC_CIDR="fd05::/112" -CLUSTER_B_DOMAIN="cluster-b.remote" - -# Cluster C (host3): non-overlapping CIDRs -CLUSTER_C_POD_CIDR="fd07::/48" -CLUSTER_C_SVC_CIDR="fd08::/112" -CLUSTER_C_DOMAIN="cluster-c.remote" +c2cc_setup_ipv6 "${VM_IPV6_NETWORK}" scenario_create_vms() { c2cc_create_vms rhel102-bootc-source rhel102-bootc "${VM_IPV6_NETWORK}" ipv6 diff --git a/test/scenarios-bootc/c2cc/el102-src@c2cc-mtu-ipv4.sh b/test/scenarios-bootc/c2cc/el102-src@c2cc-mtu-ipv4.sh new file mode 100644 index 0000000000..301432cdf2 --- /dev/null +++ b/test/scenarios-bootc/c2cc/el102-src@c2cc-mtu-ipv4.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. +# +# Sets up 3 MicroShift clusters with C2CC on a jumbo-frame network +# (MTU 9000). Tests validate MTU boundary behavior for plain C2CC +# traffic without IPsec. + +# shellcheck source=test/bin/c2cc_common.sh +source "${SCRIPTDIR}/c2cc_common.sh" + +scenario_create_vms() { + c2cc_create_vms "rhel102-bootc-source" "rhel102-bootc" "jumbo" "ipv4" "9000" +} + +scenario_remove_vms() { + c2cc_remove_vms +} + +scenario_run_tests() { + configure_c2cc_hosts "c2cc_mtu_pre_greenboot" "c2cc_mtu_greenboot" + + c2cc_run_tests "suites/c2cc/extra/mtu.robot" +} diff --git a/test/scenarios-bootc/c2cc/el102-src@c2cc-mtu-ipv6.sh b/test/scenarios-bootc/c2cc/el102-src@c2cc-mtu-ipv6.sh new file mode 100644 index 0000000000..0c740106b5 --- /dev/null +++ b/test/scenarios-bootc/c2cc/el102-src@c2cc-mtu-ipv6.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. +# +# Sets up 3 MicroShift clusters with C2CC on a single-stack IPv6 jumbo-frame +# network (MTU 9000). Same test suite as c2cc-mtu, run over IPv6 instead of +# IPv4, without IPsec. + +# shellcheck source=test/bin/c2cc_common.sh +source "${SCRIPTDIR}/c2cc_common.sh" + +# shellcheck disable=SC2119 +c2cc_setup_ipv6 + +scenario_create_vms() { + # Read jumbo-ipv6 bridge IP for /etc/hosts resolution in VMs. + # Keep WEB_SERVER_URL at the default IPv4 — bootc kickstarts don't + # use REPLACE_OSTREE_SERVER_URL, and the hypervisor-side curl can't + # reach the IPv6 bridge due to libvirt's nftables filtering. + # shellcheck disable=SC2034 # used elsewhere + VM_BRIDGE_IP="$(get_vm_bridge_ip "jumbo-ipv6")" + c2cc_create_vms "rhel102-bootc-source" "rhel102-bootc" "jumbo-ipv6" "ipv6" "9000" +} + +scenario_remove_vms() { + c2cc_remove_vms +} + +scenario_run_tests() { + configure_c2cc_hosts "c2cc_mtu_pre_greenboot" "c2cc_mtu_greenboot" + + c2cc_run_tests "suites/c2cc/extra/mtu.robot" "" ipv6 +} diff --git a/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec-mtu-ipv4.sh b/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec-mtu-ipv4.sh new file mode 100644 index 0000000000..2b298cb7c3 --- /dev/null +++ b/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec-mtu-ipv4.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. +# +# Sets up 3 MicroShift clusters with C2CC and Libreswan IPsec on a +# jumbo-frame network (MTU 9000). Tests validate MTU boundary behavior +# through IPsec tunnel-mode ESP encapsulation and the recommended +# pod MTU reduction to 8900. + +# shellcheck source=test/bin/c2cc_common.sh +source "${SCRIPTDIR}/c2cc_common.sh" + + +scenario_create_vms() { + c2cc_create_vms "rhel98-bootc-source-ipsec" "rhel98-bootc" "jumbo" "ipv4" "9000" +} + +scenario_remove_vms() { + c2cc_remove_vms +} + +scenario_run_tests() { + configure_c2cc_hosts "c2cc_ipsec_mtu_pre_greenboot" "c2cc_ipsec_mtu_greenboot" + configure_ipsec + + c2cc_run_tests "suites/c2cc/extra/mtu.robot" "" "" "--variable IPSEC:true" +} diff --git a/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec-mtu-ipv6.sh b/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec-mtu-ipv6.sh new file mode 100644 index 0000000000..c7ff4b5a39 --- /dev/null +++ b/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec-mtu-ipv6.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. +# +# Sets up 3 MicroShift clusters with C2CC and Libreswan IPsec on a +# single-stack IPv6 jumbo-frame network (MTU 9000). Same test suite as +# c2cc-ipsec-mtu, run over IPv6 instead of IPv4. + +# shellcheck source=test/bin/c2cc_common.sh +source "${SCRIPTDIR}/c2cc_common.sh" + + +# shellcheck disable=SC2119 +c2cc_setup_ipv6 + +scenario_create_vms() { + # Read jumbo-ipv6 bridge IP for /etc/hosts resolution in VMs. + # Keep WEB_SERVER_URL at the default IPv4 — bootc kickstarts don't + # use REPLACE_OSTREE_SERVER_URL, and the hypervisor-side curl can't + # reach the IPv6 bridge due to libvirt's nftables filtering. + # shellcheck disable=SC2034 # used elsewhere + VM_BRIDGE_IP="$(get_vm_bridge_ip "jumbo-ipv6")" + c2cc_create_vms "rhel98-bootc-source-ipsec" "rhel98-bootc" "jumbo-ipv6" "ipv6" "9000" +} + +scenario_remove_vms() { + c2cc_remove_vms +} + +scenario_run_tests() { + configure_c2cc_hosts "c2cc_ipsec_mtu_pre_greenboot" "c2cc_ipsec_mtu_greenboot" + configure_ipsec + + c2cc_run_tests "suites/c2cc/extra/mtu.robot" "" ipv6 "--variable IPSEC:true" +} diff --git a/test/scenarios-bootc/c2cc/el98-src@c2cc-ipv6.sh b/test/scenarios-bootc/c2cc/el98-src@c2cc-ipv6.sh index 349cebf36b..33eca10c11 100644 --- a/test/scenarios-bootc/c2cc/el98-src@c2cc-ipv6.sh +++ b/test/scenarios-bootc/c2cc/el98-src@c2cc-ipv6.sh @@ -5,32 +5,7 @@ # shellcheck source=test/bin/c2cc_common.sh source "${SCRIPTDIR}/c2cc_common.sh" -# Redefine network-related settings to use the dedicated IPv6 network bridge -# shellcheck disable=SC2034 # used elsewhere -VM_BRIDGE_IP="$(get_vm_bridge_ip "${VM_IPV6_NETWORK}")" -# shellcheck disable=SC2034 # used elsewhere -WEB_SERVER_URL="http://[${VM_BRIDGE_IP}]:${WEB_SERVER_PORT}" -# Using `hostname` here instead of a raw ip because skopeo only allows either -# ipv4 or fqdn's, but not ipv6. Since the registry is hosted on the ipv6 -# network gateway in the host, we need to use a combination of the hostname -# plus /etc/hosts resolution (which is taken care of by kickstart). -# shellcheck disable=SC2034 # used elsewhere -MIRROR_REGISTRY_URL="$(hostname):${MIRROR_REGISTRY_PORT}/microshift" - -# Cluster A (host1): non-overlapping CIDRs -CLUSTER_A_POD_CIDR="fd01::/48" -CLUSTER_A_SVC_CIDR="fd02::/112" -CLUSTER_A_DOMAIN="cluster-a.remote" - -# Cluster B (host2): non-overlapping CIDRs -CLUSTER_B_POD_CIDR="fd04::/48" -CLUSTER_B_SVC_CIDR="fd05::/112" -CLUSTER_B_DOMAIN="cluster-b.remote" - -# Cluster C (host3): non-overlapping CIDRs -CLUSTER_C_POD_CIDR="fd07::/48" -CLUSTER_C_SVC_CIDR="fd08::/112" -CLUSTER_C_DOMAIN="cluster-c.remote" +c2cc_setup_ipv6 "${VM_IPV6_NETWORK}" scenario_create_vms() { c2cc_create_vms rhel98-bootc-source rhel98-bootc "${VM_IPV6_NETWORK}" ipv6 diff --git a/test/scenarios-bootc/c2cc/el98-src@c2cc-mtu-ipv4.sh b/test/scenarios-bootc/c2cc/el98-src@c2cc-mtu-ipv4.sh new file mode 100644 index 0000000000..22ad8a3c50 --- /dev/null +++ b/test/scenarios-bootc/c2cc/el98-src@c2cc-mtu-ipv4.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. +# +# Sets up 3 MicroShift clusters with C2CC on a jumbo-frame network +# (MTU 9000). Tests validate MTU boundary behavior for plain C2CC +# traffic without IPsec. + +# shellcheck source=test/bin/c2cc_common.sh +source "${SCRIPTDIR}/c2cc_common.sh" + +scenario_create_vms() { + c2cc_create_vms "rhel98-bootc-source" "rhel98-bootc" "jumbo" "ipv4" "9000" +} + +scenario_remove_vms() { + c2cc_remove_vms +} + +scenario_run_tests() { + configure_c2cc_hosts "c2cc_mtu_pre_greenboot" "c2cc_mtu_greenboot" + + c2cc_run_tests "suites/c2cc/extra/mtu.robot" +} diff --git a/test/scenarios-bootc/c2cc/el98-src@c2cc-mtu-ipv6.sh b/test/scenarios-bootc/c2cc/el98-src@c2cc-mtu-ipv6.sh new file mode 100644 index 0000000000..c7f7521b10 --- /dev/null +++ b/test/scenarios-bootc/c2cc/el98-src@c2cc-mtu-ipv6.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. +# +# Sets up 3 MicroShift clusters with C2CC on a single-stack IPv6 jumbo-frame +# network (MTU 9000). Same test suite as c2cc-mtu, run over IPv6 instead of +# IPv4, without IPsec. + +# shellcheck source=test/bin/c2cc_common.sh +source "${SCRIPTDIR}/c2cc_common.sh" + +# shellcheck disable=SC2119 +c2cc_setup_ipv6 + +scenario_create_vms() { + # Read jumbo-ipv6 bridge IP for /etc/hosts resolution in VMs. + # Keep WEB_SERVER_URL at the default IPv4 — bootc kickstarts don't + # use REPLACE_OSTREE_SERVER_URL, and the hypervisor-side curl can't + # reach the IPv6 bridge due to libvirt's nftables filtering. + # shellcheck disable=SC2034 # used elsewhere + VM_BRIDGE_IP="$(get_vm_bridge_ip "jumbo-ipv6")" + c2cc_create_vms "rhel98-bootc-source" "rhel98-bootc" "jumbo-ipv6" "ipv6" "9000" +} + +scenario_remove_vms() { + c2cc_remove_vms +} + +scenario_run_tests() { + configure_c2cc_hosts "c2cc_mtu_pre_greenboot" "c2cc_mtu_greenboot" + + c2cc_run_tests "suites/c2cc/extra/mtu.robot" "" ipv6 +} diff --git a/test/suites/c2cc/extra/mtu.robot b/test/suites/c2cc/extra/mtu.robot new file mode 100644 index 0000000000..7220c78d84 --- /dev/null +++ b/test/suites/c2cc/extra/mtu.robot @@ -0,0 +1,43 @@ +*** Settings *** +Documentation MTU boundary tests for C2CC at jumbo frame size (9000 MTU). +... Pod MTU 9000 is set via kickstart ovn.yaml (OVN-K bakes the +... MTU into its database at initial creation). Validates DF-bit +... acceptance and rejection at the boundary, large TCP transfers, +... and full cross-cluster connectivity. +... When IPsec is configured (IPSEC variable set to true), also +... verifies ESP encapsulation and tunnel reestablishment. +... Used by both c2cc-mtu and c2cc-ipsec-mtu scenarios. + +Resource ../../../resources/microshift-process.resource +Resource ../../../resources/kubeconfig.resource +Resource ../../../resources/oc.resource +Resource ../../../resources/c2cc.resource +Resource ../../../resources/ipsec.resource + +Suite Setup C2CC Suite Setup deploy_workloads=${TRUE} +Suite Teardown C2CC Suite Teardown cleanup_workloads=${TRUE} + +Test Tags c2cc mtu + + +*** Variables *** +${IPSEC} false + + +*** Test Cases *** +C2CC At Jumbo MTU 9000 + [Documentation] Validate MTU boundary behavior across all cluster pairs. + ... Pod MTU 9000 is set via kickstart ovn.yaml. When IPSEC=true, + ... also verifies ESP encapsulation. + Verify Pod MTU On All Clusters 9000 + IF "${IPSEC}" == "true" + Verify ESP Encapsulation On All Clusters + ELSE + Verify Full C2CC Connectivity + END + Ping DF Bit Between All Clusters 64 + # 8952 + 48 (IPv6 headers) = 9000 = pod MTU → passes + Ping DF Bit Between All Clusters 8952 + # 8973 + 28 (IPv4 headers) = 9001 > 9000 pod MTU → rejected + Ping DF Bit Should Fail Between All Clusters 8973 + Large Payload Between All Clusters 65536 From 153e6082d65a64845e7f92db5e3cd457f5fb8ba5 Mon Sep 17 00:00:00 2001 From: Alejandro Gullon Date: Tue, 21 Jul 2026 21:09:59 +0200 Subject: [PATCH 5/5] feat: split C2CC scenarios by architecture to halve job load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each arch now runs only one RHEL version: x86 runs el98 (RHEL 9.8), ARM runs el102 (RHEL 10.2). The assignment rotates per commit using the first hex digit of HEAD so both combinations get exercised over time. Both arch jobs derive the same flip from the same merge commit. This halves the scenario count per CI job (23 → ~12), reducing VM resource contention on the c7g.metal instances. Co-Authored-By: Claude Opus 4.6 (1M context) pre-commit.check-secrets: ENABLED --- test/bin/ci_phase_boot_and_test.sh | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/test/bin/ci_phase_boot_and_test.sh b/test/bin/ci_phase_boot_and_test.sh index 0dba688df4..56aef1cb30 100755 --- a/test/bin/ci_phase_boot_and_test.sh +++ b/test/bin/ci_phase_boot_and_test.sh @@ -85,16 +85,30 @@ else progress="" fi -# Limit amount of parallel scenarios for the C2CC jobs to avoid -# over-provisioning the resources: each C2CC scenario runs 3 VMs -# (2 vCPUs / 4GiB each) and the c7g.metal instance has 64 cores / 128GiB. -# Powering off the VMs of passed scenarios makes the job limit an -# actual cap on the number of running VMs. +# Each C2CC scenario runs 3 VMs (2 vCPUs / 4GiB each) +# and the c7g.metal instance has 64 cores / 128GiB. jobs_arg="" scenario_action="create-and-run" if [[ "${SCENARIO_SOURCES}" =~ c2cc ]]; then + # Limit amount of parallel scenarios for the C2CC jobs + # to avoid over-provisioning the resources. jobs_arg="-j 8" + + # Power off passed VMs so the job limit is an actual cap on running VMs. scenario_action="create-run-shutdown" + + # Each arch runs one RHEL version to halve the scenario count. + # The assignment rotates per commit so both combos get coverage: + # flip=0: x86 → el98, ARM → el102 + # flip=1: x86 → el102, ARM → el98 + flip=$(( 16#$(git rev-parse HEAD | cut -c1) % 2 )) + if [[ "$(uname -m)" == "x86_64" ]]; then + delete_ver=(el102 el98) + else + delete_ver=(el98 el102) + fi + echo "C2CC arch split: deleting ${delete_ver[flip]}-* (flip=${flip})" + find "${SCENARIOS_TO_RUN}" -name "${delete_ver[flip]}-*.sh" -delete elif [[ "${SCENARIO_SOURCES}" =~ .*releases.* ]]; then # Release scenarios have grown (e.g. the router scenarios were split # from 1 into 5) and no longer fit if every scenario's VMs stay up for