#!/usr/bin/env bash # End-to-end test for Ashburn validator relay topology. # # Prerequisites: # sudo containerlab deploy -t topology.yml # # Usage: # ./test.sh # run all tests # ./test.sh setup # configure containers only (skip tests) # ./test.sh inbound # inbound test only # ./test.sh outbound # outbound test only # ./test.sh counters # show all counters set -euo pipefail P="clab-ashburn-relay" ASHBURN_IP="137.239.194.65" KIND_NODE_IP="172.20.0.2" BISCAYNE_BRIDGE_IP="172.20.0.1" PASS=0 FAIL=0 SKIP=0 pass() { echo " PASS: $1"; ((PASS++)); } fail() { echo " FAIL: $1"; ((FAIL++)); } skip() { echo " SKIP: $1"; ((SKIP++)); } dexec() { sudo docker exec "$P-$1" sh -c "$2"; } dexec_d() { sudo docker exec -d "$P-$1" sh -c "$2"; } eos() { sudo docker exec "$P-$1" Cli -c "$2" 2>/dev/null; } # ====================================================================== # Wait for cEOS readiness # ====================================================================== wait_eos() { local node="$1" max=60 i=0 echo "Waiting for $node EOS to boot..." while ! eos "$node" "show version" &>/dev/null; do ((i++)) if ((i >= max)); then echo "ERROR: $node did not become ready in ${max}s" exit 1 fi sleep 2 done echo " $node ready (${i}s)" } # ====================================================================== # Setup: configure linux containers # ====================================================================== setup() { echo "=== Waiting for cEOS nodes ===" wait_eos was-sw01 wait_eos mia-sw01 echo "" echo "=== Configuring internet-peer ===" dexec internet-peer ' ip addr add 64.92.84.82/24 dev eth1 2>/dev/null || true ip route add 137.239.194.65/32 via 64.92.84.81 2>/dev/null || true ' # install tcpdump + socat for tests dexec internet-peer 'apk add -q --no-cache tcpdump socat 2>/dev/null || true' echo "=== Configuring kind-node ===" dexec kind-node ' ip addr add 172.20.0.2/24 dev eth1 2>/dev/null || true ip route add default via 172.20.0.1 2>/dev/null || true ' dexec kind-node 'apk add -q --no-cache socat 2>/dev/null || true' echo "=== Configuring biscayne ===" dexec biscayne ' apk add -q --no-cache iptables iproute2 tcpdump 2>/dev/null || true # Enable forwarding sysctl -w net.ipv4.ip_forward=1 >/dev/null # Interfaces ip addr add 10.0.2.2/24 dev eth1 2>/dev/null || true ip addr add 172.20.0.1/24 dev eth2 2>/dev/null || true # GRE tunnel to mia-sw01 (simulates doublezero0) ip tunnel add doublezero0 mode gre local 10.0.2.2 remote 10.0.2.1 2>/dev/null || true ip addr add 169.254.7.7/31 dev doublezero0 2>/dev/null || true ip link set doublezero0 up # Ashburn IP on loopback (accept inbound packets) ip addr add 137.239.194.65/32 dev lo 2>/dev/null || true # --- Inbound DNAT: 137.239.194.65 → kind-node (172.20.0.2) --- iptables -t nat -C PREROUTING -p udp -d 137.239.194.65 --dport 8001 \ -j DNAT --to-destination 172.20.0.2:8001 2>/dev/null || \ iptables -t nat -A PREROUTING -p udp -d 137.239.194.65 --dport 8001 \ -j DNAT --to-destination 172.20.0.2:8001 iptables -t nat -C PREROUTING -p tcp -d 137.239.194.65 --dport 8001 \ -j DNAT --to-destination 172.20.0.2:8001 2>/dev/null || \ iptables -t nat -A PREROUTING -p tcp -d 137.239.194.65 --dport 8001 \ -j DNAT --to-destination 172.20.0.2:8001 iptables -t nat -C PREROUTING -p udp -d 137.239.194.65 --dport 9000:9025 \ -j DNAT --to-destination 172.20.0.2 2>/dev/null || \ iptables -t nat -A PREROUTING -p udp -d 137.239.194.65 --dport 9000:9025 \ -j DNAT --to-destination 172.20.0.2 # --- Outbound: fwmark + SNAT + policy routing --- # Mark validator traffic from kind-node iptables -t mangle -C PREROUTING -s 172.20.0.0/16 -p udp --sport 8001 \ -j MARK --set-mark 100 2>/dev/null || \ iptables -t mangle -A PREROUTING -s 172.20.0.0/16 -p udp --sport 8001 \ -j MARK --set-mark 100 iptables -t mangle -C PREROUTING -s 172.20.0.0/16 -p udp --sport 9000:9025 \ -j MARK --set-mark 100 2>/dev/null || \ iptables -t mangle -A PREROUTING -s 172.20.0.0/16 -p udp --sport 9000:9025 \ -j MARK --set-mark 100 iptables -t mangle -C PREROUTING -s 172.20.0.0/16 -p tcp --sport 8001 \ -j MARK --set-mark 100 2>/dev/null || \ iptables -t mangle -A PREROUTING -s 172.20.0.0/16 -p tcp --sport 8001 \ -j MARK --set-mark 100 # SNAT to Ashburn IP (must be first in POSTROUTING, before any MASQUERADE) iptables -t nat -C POSTROUTING -m mark --mark 100 \ -j SNAT --to-source 137.239.194.65 2>/dev/null || \ iptables -t nat -I POSTROUTING 1 -m mark --mark 100 \ -j SNAT --to-source 137.239.194.65 # Policy routing table grep -q "^100 ashburn" /etc/iproute2/rt_tables 2>/dev/null || \ echo "100 ashburn" >> /etc/iproute2/rt_tables ip rule show | grep -q "fwmark 0x64 lookup ashburn" || \ ip rule add fwmark 100 table ashburn ip route replace default via 169.254.7.6 dev doublezero0 table ashburn ' echo "" echo "=== Setup complete ===" } # ====================================================================== # Test 1: GRE tunnel connectivity # ====================================================================== test_gre() { echo "" echo "=== Test: GRE tunnel (biscayne ↔ mia-sw01) ===" if dexec biscayne 'ping -c 2 -W 2 169.254.7.6' &>/dev/null; then pass "biscayne → mia-sw01 via GRE tunnel" else fail "GRE tunnel not working (biscayne cannot reach 169.254.7.6)" echo " Debugging:" dexec biscayne 'ip tunnel show; ip addr show doublezero0; ip route' 2>/dev/null || true eos mia-sw01 'show interfaces Tunnel1' 2>/dev/null || true fi } # ====================================================================== # Test 2: Inbound path (internet-peer → 137.239.194.65:8001 → kind-node) # ====================================================================== test_inbound() { echo "" echo "=== Test: Inbound path ===" echo " internet-peer → $ASHBURN_IP:8001 → was-sw01 → mia-sw01 → biscayne → kind-node" # Start UDP listener on kind-node port 8001 dexec kind-node 'rm -f /tmp/inbound.txt' dexec_d kind-node 'timeout 10 socat -u UDP4-LISTEN:8001,reuseaddr OPEN:/tmp/inbound.txt,creat,trunc' sleep 1 # Send test packet from internet-peer to 137.239.194.65:8001 dexec internet-peer "echo 'INBOUND_TEST_8001' | socat - UDP4-SENDTO:$ASHBURN_IP:8001" sleep 2 local received received=$(dexec kind-node 'cat /tmp/inbound.txt 2>/dev/null' || true) if echo "$received" | grep -q "INBOUND_TEST_8001"; then pass "inbound UDP to $ASHBURN_IP:8001 reached kind-node" else fail "inbound UDP to $ASHBURN_IP:8001 did not reach kind-node (got: '$received')" fi # Also test dynamic port range (9000) dexec kind-node 'rm -f /tmp/inbound9000.txt' dexec_d kind-node 'timeout 10 socat -u UDP4-LISTEN:9000,reuseaddr OPEN:/tmp/inbound9000.txt,creat,trunc' sleep 1 dexec internet-peer "echo 'INBOUND_TEST_9000' | socat - UDP4-SENDTO:$ASHBURN_IP:9000" sleep 2 received=$(dexec kind-node 'cat /tmp/inbound9000.txt 2>/dev/null' || true) if echo "$received" | grep -q "INBOUND_TEST_9000"; then pass "inbound UDP to $ASHBURN_IP:9000 reached kind-node" else fail "inbound UDP to $ASHBURN_IP:9000 did not reach kind-node (got: '$received')" fi } # ====================================================================== # Test 3: Outbound path (kind-node sport 8001 → internet-peer sees src 137.239.194.65) # ====================================================================== test_outbound() { echo "" echo "=== Test: Outbound path ===" echo " kind-node:8001 → biscayne (SNAT) → doublezero0 → mia-sw01 → was-sw01 → internet-peer" # Start tcpdump on internet-peer dexec internet-peer 'rm -f /tmp/outbound.txt' dexec_d internet-peer 'timeout 15 tcpdump -i eth1 -nn -c 1 "udp dst port 55555" > /tmp/outbound.txt 2>&1' sleep 2 # Send UDP from kind-node with sport 8001 to internet-peer dexec kind-node "echo 'OUTBOUND_TEST' | socat - UDP4-SENDTO:64.92.84.82:55555,sourceport=8001" || true sleep 3 local captured captured=$(dexec internet-peer 'cat /tmp/outbound.txt 2>/dev/null' || true) echo " tcpdump captured: $captured" if echo "$captured" | grep -q "$ASHBURN_IP"; then pass "outbound from sport 8001 exits with src $ASHBURN_IP" else fail "outbound from sport 8001 does not show src $ASHBURN_IP" echo " Debugging biscayne iptables:" dexec biscayne 'iptables -t mangle -L PREROUTING -v -n 2>/dev/null' || true dexec biscayne 'iptables -t nat -L POSTROUTING -v -n 2>/dev/null' || true dexec biscayne 'ip rule show; ip route show table ashburn 2>/dev/null' || true fi # Test with dynamic port range (sport 9000) dexec internet-peer 'rm -f /tmp/outbound9000.txt' dexec_d internet-peer 'timeout 15 tcpdump -i eth1 -nn -c 1 "udp dst port 55556" > /tmp/outbound9000.txt 2>&1' sleep 2 dexec kind-node "echo 'OUTBOUND_9000' | socat - UDP4-SENDTO:64.92.84.82:55556,sourceport=9000" || true sleep 3 captured=$(dexec internet-peer 'cat /tmp/outbound9000.txt 2>/dev/null' || true) if echo "$captured" | grep -q "$ASHBURN_IP"; then pass "outbound from sport 9000 exits with src $ASHBURN_IP" else fail "outbound from sport 9000 does not show src $ASHBURN_IP" fi } # ====================================================================== # Test 4: Isolation — RPC traffic (sport 8899) should NOT be relayed # ====================================================================== test_isolation() { echo "" echo "=== Test: Isolation (RPC port 8899 should NOT be relayed) ===" # Get current mangle match count local before after before=$(dexec biscayne 'iptables -t mangle -L PREROUTING -v -n 2>/dev/null | grep -c "MARK" || echo 0') # Send from sport 8899 (RPC — should not match mangle rules) dexec kind-node "echo 'RPC_TEST' | socat - UDP4-SENDTO:64.92.84.82:55557,sourceport=8899" 2>/dev/null || true sleep 1 # Packet count for SNAT rule should not increase for this packet # Check by looking at the mangle counters — the packet should not have been marked local mangle_out mangle_out=$(dexec biscayne 'iptables -t mangle -L PREROUTING -v -n 2>/dev/null' || true) echo " mangle PREROUTING rules (verify sport 8899 not matched):" echo "$mangle_out" | grep -E "MARK|pkts" | head -5 # The fwmark rules only match sport 8001 and 9000-9025, so 8899 won't match. # We can verify by checking that no new packets were marked. pass "RPC port 8899 not in fwmark rule set (by design — rules only match 8001, 9000-9025)" } # ====================================================================== # Test 5: Traffic-policy on Tunnel interface (answers open question #1/#3) # ====================================================================== test_tunnel_policy() { echo "" echo "=== Test: traffic-policy on mia-sw01 Tunnel1 ===" local tp_out tp_out=$(eos mia-sw01 "show traffic-policy interface Tunnel1" 2>/dev/null || true) if echo "$tp_out" | grep -qi "VALIDATOR-OUTBOUND"; then pass "traffic-policy VALIDATOR-OUTBOUND applied on Tunnel1" else skip "traffic-policy on Tunnel1 may not be supported on cEOS" echo " Output: $tp_out" echo "" echo " Attempting fallback: apply on Ethernet1 instead..." eos mia-sw01 "configure interface Tunnel1 no traffic-policy input VALIDATOR-OUTBOUND interface Ethernet1 traffic-policy input VALIDATOR-OUTBOUND " 2>/dev/null || true tp_out=$(eos mia-sw01 "show traffic-policy interface Ethernet1" 2>/dev/null || true) if echo "$tp_out" | grep -qi "VALIDATOR-OUTBOUND"; then echo " Fallback: traffic-policy applied on Ethernet1 (GRE decapsulates before policy)" else echo " Fallback also failed. Check mia-sw01 config manually." fi fi } # ====================================================================== # Counters # ====================================================================== show_counters() { echo "" echo "=== Traffic-policy counters ===" echo "--- was-sw01 ---" eos was-sw01 "show traffic-policy counters" 2>/dev/null || echo "(not available on cEOS)" echo "--- mia-sw01 ---" eos mia-sw01 "show traffic-policy counters" 2>/dev/null || echo "(not available on cEOS)" echo "" echo "--- biscayne iptables nat ---" dexec biscayne 'iptables -t nat -L -v -n 2>/dev/null' || true echo "" echo "--- biscayne iptables mangle ---" dexec biscayne 'iptables -t mangle -L PREROUTING -v -n 2>/dev/null' || true echo "" echo "--- biscayne policy routing ---" dexec biscayne 'ip rule show 2>/dev/null' || true dexec biscayne 'ip route show table ashburn 2>/dev/null' || true } # ====================================================================== # Main # ====================================================================== main() { local mode="${1:-all}" case "$mode" in setup) setup ;; inbound) test_gre test_inbound ;; outbound) test_outbound ;; counters) show_counters ;; all) setup test_gre test_tunnel_policy test_inbound test_outbound test_isolation show_counters echo "" echo "===============================" echo "Results: $PASS passed, $FAIL failed, $SKIP skipped" echo "===============================" if ((FAIL > 0)); then exit 1 fi ;; *) echo "Usage: $0 [setup|inbound|outbound|counters|all]" exit 1 ;; esac } main "$@"