378 lines
13 KiB
Bash
Executable File
378 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# End-to-end test for Ashburn validator relay topology.
|
|
#
|
|
# Prerequisites:
|
|
# sudo containerlab deploy -t topology.yml
|
|
#
|
|
# Usage:
|
|
# ./test.sh # run all tests
|
|
# ./test.sh setup # configure containers only (skip tests)
|
|
# ./test.sh inbound # inbound test only
|
|
# ./test.sh outbound # outbound test only
|
|
# ./test.sh counters # show all counters
|
|
|
|
set -euo pipefail
|
|
|
|
P="clab-ashburn-relay"
|
|
ASHBURN_IP="137.239.194.65"
|
|
KIND_NODE_IP="172.20.0.2"
|
|
BISCAYNE_BRIDGE_IP="172.20.0.1"
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
SKIP=0
|
|
|
|
pass() { echo " PASS: $1"; ((PASS++)); }
|
|
fail() { echo " FAIL: $1"; ((FAIL++)); }
|
|
skip() { echo " SKIP: $1"; ((SKIP++)); }
|
|
|
|
dexec() { sudo docker exec "$P-$1" sh -c "$2"; }
|
|
dexec_d() { sudo docker exec -d "$P-$1" sh -c "$2"; }
|
|
eos() { sudo docker exec "$P-$1" Cli -c "$2" 2>/dev/null; }
|
|
|
|
# ======================================================================
|
|
# Wait for cEOS readiness
|
|
# ======================================================================
|
|
wait_eos() {
|
|
local node="$1" max=60 i=0
|
|
echo "Waiting for $node EOS to boot..."
|
|
while ! eos "$node" "show version" &>/dev/null; do
|
|
((i++))
|
|
if ((i >= max)); then
|
|
echo "ERROR: $node did not become ready in ${max}s"
|
|
exit 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo " $node ready (${i}s)"
|
|
}
|
|
|
|
# ======================================================================
|
|
# Setup: configure linux containers
|
|
# ======================================================================
|
|
setup() {
|
|
echo "=== Waiting for cEOS nodes ==="
|
|
wait_eos was-sw01
|
|
wait_eos mia-sw01
|
|
|
|
echo ""
|
|
echo "=== Configuring internet-peer ==="
|
|
dexec internet-peer '
|
|
ip addr add 64.92.84.82/24 dev eth1 2>/dev/null || true
|
|
ip route add 137.239.194.65/32 via 64.92.84.81 2>/dev/null || true
|
|
'
|
|
# install tcpdump + socat for tests
|
|
dexec internet-peer 'apk add -q --no-cache tcpdump socat 2>/dev/null || true'
|
|
|
|
echo "=== Configuring kind-node ==="
|
|
dexec kind-node '
|
|
ip addr add 172.20.0.2/24 dev eth1 2>/dev/null || true
|
|
ip route add default via 172.20.0.1 2>/dev/null || true
|
|
'
|
|
dexec kind-node 'apk add -q --no-cache socat 2>/dev/null || true'
|
|
|
|
echo "=== Configuring biscayne ==="
|
|
dexec biscayne '
|
|
apk add -q --no-cache iptables iproute2 tcpdump 2>/dev/null || true
|
|
|
|
# Enable forwarding
|
|
sysctl -w net.ipv4.ip_forward=1 >/dev/null
|
|
|
|
# Interfaces
|
|
ip addr add 10.0.2.2/24 dev eth1 2>/dev/null || true
|
|
ip addr add 172.20.0.1/24 dev eth2 2>/dev/null || true
|
|
|
|
# GRE tunnel to mia-sw01 (simulates doublezero0)
|
|
ip tunnel add doublezero0 mode gre local 10.0.2.2 remote 10.0.2.1 2>/dev/null || true
|
|
ip addr add 169.254.7.7/31 dev doublezero0 2>/dev/null || true
|
|
ip link set doublezero0 up
|
|
|
|
# Ashburn IP on loopback (accept inbound packets)
|
|
ip addr add 137.239.194.65/32 dev lo 2>/dev/null || true
|
|
|
|
# --- Inbound DNAT: 137.239.194.65 → kind-node (172.20.0.2) ---
|
|
iptables -t nat -C PREROUTING -p udp -d 137.239.194.65 --dport 8001 \
|
|
-j DNAT --to-destination 172.20.0.2:8001 2>/dev/null || \
|
|
iptables -t nat -A PREROUTING -p udp -d 137.239.194.65 --dport 8001 \
|
|
-j DNAT --to-destination 172.20.0.2:8001
|
|
|
|
iptables -t nat -C PREROUTING -p tcp -d 137.239.194.65 --dport 8001 \
|
|
-j DNAT --to-destination 172.20.0.2:8001 2>/dev/null || \
|
|
iptables -t nat -A PREROUTING -p tcp -d 137.239.194.65 --dport 8001 \
|
|
-j DNAT --to-destination 172.20.0.2:8001
|
|
|
|
iptables -t nat -C PREROUTING -p udp -d 137.239.194.65 --dport 9000:9025 \
|
|
-j DNAT --to-destination 172.20.0.2 2>/dev/null || \
|
|
iptables -t nat -A PREROUTING -p udp -d 137.239.194.65 --dport 9000:9025 \
|
|
-j DNAT --to-destination 172.20.0.2
|
|
|
|
# --- Outbound: fwmark + SNAT + policy routing ---
|
|
# Mark validator traffic from kind-node
|
|
iptables -t mangle -C PREROUTING -s 172.20.0.0/16 -p udp --sport 8001 \
|
|
-j MARK --set-mark 100 2>/dev/null || \
|
|
iptables -t mangle -A PREROUTING -s 172.20.0.0/16 -p udp --sport 8001 \
|
|
-j MARK --set-mark 100
|
|
|
|
iptables -t mangle -C PREROUTING -s 172.20.0.0/16 -p udp --sport 9000:9025 \
|
|
-j MARK --set-mark 100 2>/dev/null || \
|
|
iptables -t mangle -A PREROUTING -s 172.20.0.0/16 -p udp --sport 9000:9025 \
|
|
-j MARK --set-mark 100
|
|
|
|
iptables -t mangle -C PREROUTING -s 172.20.0.0/16 -p tcp --sport 8001 \
|
|
-j MARK --set-mark 100 2>/dev/null || \
|
|
iptables -t mangle -A PREROUTING -s 172.20.0.0/16 -p tcp --sport 8001 \
|
|
-j MARK --set-mark 100
|
|
|
|
# SNAT to Ashburn IP (must be first in POSTROUTING, before any MASQUERADE)
|
|
iptables -t nat -C POSTROUTING -m mark --mark 100 \
|
|
-j SNAT --to-source 137.239.194.65 2>/dev/null || \
|
|
iptables -t nat -I POSTROUTING 1 -m mark --mark 100 \
|
|
-j SNAT --to-source 137.239.194.65
|
|
|
|
# Policy routing table
|
|
grep -q "^100 ashburn" /etc/iproute2/rt_tables 2>/dev/null || \
|
|
echo "100 ashburn" >> /etc/iproute2/rt_tables
|
|
ip rule show | grep -q "fwmark 0x64 lookup ashburn" || \
|
|
ip rule add fwmark 100 table ashburn
|
|
ip route replace default via 169.254.7.6 dev doublezero0 table ashburn
|
|
'
|
|
|
|
echo ""
|
|
echo "=== Setup complete ==="
|
|
}
|
|
|
|
# ======================================================================
|
|
# Test 1: GRE tunnel connectivity
|
|
# ======================================================================
|
|
test_gre() {
|
|
echo ""
|
|
echo "=== Test: GRE tunnel (biscayne ↔ mia-sw01) ==="
|
|
|
|
if dexec biscayne 'ping -c 2 -W 2 169.254.7.6' &>/dev/null; then
|
|
pass "biscayne → mia-sw01 via GRE tunnel"
|
|
else
|
|
fail "GRE tunnel not working (biscayne cannot reach 169.254.7.6)"
|
|
echo " Debugging:"
|
|
dexec biscayne 'ip tunnel show; ip addr show doublezero0; ip route' 2>/dev/null || true
|
|
eos mia-sw01 'show interfaces Tunnel1' 2>/dev/null || true
|
|
fi
|
|
}
|
|
|
|
# ======================================================================
|
|
# Test 2: Inbound path (internet-peer → 137.239.194.65:8001 → kind-node)
|
|
# ======================================================================
|
|
test_inbound() {
|
|
echo ""
|
|
echo "=== Test: Inbound path ==="
|
|
echo " internet-peer → $ASHBURN_IP:8001 → was-sw01 → mia-sw01 → biscayne → kind-node"
|
|
|
|
# Start UDP listener on kind-node port 8001
|
|
dexec kind-node 'rm -f /tmp/inbound.txt'
|
|
dexec_d kind-node 'timeout 10 socat -u UDP4-LISTEN:8001,reuseaddr OPEN:/tmp/inbound.txt,creat,trunc'
|
|
sleep 1
|
|
|
|
# Send test packet from internet-peer to 137.239.194.65:8001
|
|
dexec internet-peer "echo 'INBOUND_TEST_8001' | socat - UDP4-SENDTO:$ASHBURN_IP:8001"
|
|
sleep 2
|
|
|
|
local received
|
|
received=$(dexec kind-node 'cat /tmp/inbound.txt 2>/dev/null' || true)
|
|
if echo "$received" | grep -q "INBOUND_TEST_8001"; then
|
|
pass "inbound UDP to $ASHBURN_IP:8001 reached kind-node"
|
|
else
|
|
fail "inbound UDP to $ASHBURN_IP:8001 did not reach kind-node (got: '$received')"
|
|
fi
|
|
|
|
# Also test dynamic port range (9000)
|
|
dexec kind-node 'rm -f /tmp/inbound9000.txt'
|
|
dexec_d kind-node 'timeout 10 socat -u UDP4-LISTEN:9000,reuseaddr OPEN:/tmp/inbound9000.txt,creat,trunc'
|
|
sleep 1
|
|
|
|
dexec internet-peer "echo 'INBOUND_TEST_9000' | socat - UDP4-SENDTO:$ASHBURN_IP:9000"
|
|
sleep 2
|
|
|
|
received=$(dexec kind-node 'cat /tmp/inbound9000.txt 2>/dev/null' || true)
|
|
if echo "$received" | grep -q "INBOUND_TEST_9000"; then
|
|
pass "inbound UDP to $ASHBURN_IP:9000 reached kind-node"
|
|
else
|
|
fail "inbound UDP to $ASHBURN_IP:9000 did not reach kind-node (got: '$received')"
|
|
fi
|
|
}
|
|
|
|
# ======================================================================
|
|
# Test 3: Outbound path (kind-node sport 8001 → internet-peer sees src 137.239.194.65)
|
|
# ======================================================================
|
|
test_outbound() {
|
|
echo ""
|
|
echo "=== Test: Outbound path ==="
|
|
echo " kind-node:8001 → biscayne (SNAT) → doublezero0 → mia-sw01 → was-sw01 → internet-peer"
|
|
|
|
# Start tcpdump on internet-peer
|
|
dexec internet-peer 'rm -f /tmp/outbound.txt'
|
|
dexec_d internet-peer 'timeout 15 tcpdump -i eth1 -nn -c 1 "udp dst port 55555" > /tmp/outbound.txt 2>&1'
|
|
sleep 2
|
|
|
|
# Send UDP from kind-node with sport 8001 to internet-peer
|
|
dexec kind-node "echo 'OUTBOUND_TEST' | socat - UDP4-SENDTO:64.92.84.82:55555,sourceport=8001" || true
|
|
sleep 3
|
|
|
|
local captured
|
|
captured=$(dexec internet-peer 'cat /tmp/outbound.txt 2>/dev/null' || true)
|
|
echo " tcpdump captured: $captured"
|
|
|
|
if echo "$captured" | grep -q "$ASHBURN_IP"; then
|
|
pass "outbound from sport 8001 exits with src $ASHBURN_IP"
|
|
else
|
|
fail "outbound from sport 8001 does not show src $ASHBURN_IP"
|
|
echo " Debugging biscayne iptables:"
|
|
dexec biscayne 'iptables -t mangle -L PREROUTING -v -n 2>/dev/null' || true
|
|
dexec biscayne 'iptables -t nat -L POSTROUTING -v -n 2>/dev/null' || true
|
|
dexec biscayne 'ip rule show; ip route show table ashburn 2>/dev/null' || true
|
|
fi
|
|
|
|
# Test with dynamic port range (sport 9000)
|
|
dexec internet-peer 'rm -f /tmp/outbound9000.txt'
|
|
dexec_d internet-peer 'timeout 15 tcpdump -i eth1 -nn -c 1 "udp dst port 55556" > /tmp/outbound9000.txt 2>&1'
|
|
sleep 2
|
|
|
|
dexec kind-node "echo 'OUTBOUND_9000' | socat - UDP4-SENDTO:64.92.84.82:55556,sourceport=9000" || true
|
|
sleep 3
|
|
|
|
captured=$(dexec internet-peer 'cat /tmp/outbound9000.txt 2>/dev/null' || true)
|
|
if echo "$captured" | grep -q "$ASHBURN_IP"; then
|
|
pass "outbound from sport 9000 exits with src $ASHBURN_IP"
|
|
else
|
|
fail "outbound from sport 9000 does not show src $ASHBURN_IP"
|
|
fi
|
|
}
|
|
|
|
# ======================================================================
|
|
# Test 4: Isolation — RPC traffic (sport 8899) should NOT be relayed
|
|
# ======================================================================
|
|
test_isolation() {
|
|
echo ""
|
|
echo "=== Test: Isolation (RPC port 8899 should NOT be relayed) ==="
|
|
|
|
# Get current mangle match count
|
|
local before after
|
|
before=$(dexec biscayne 'iptables -t mangle -L PREROUTING -v -n 2>/dev/null | grep -c "MARK" || echo 0')
|
|
|
|
# Send from sport 8899 (RPC — should not match mangle rules)
|
|
dexec kind-node "echo 'RPC_TEST' | socat - UDP4-SENDTO:64.92.84.82:55557,sourceport=8899" 2>/dev/null || true
|
|
sleep 1
|
|
|
|
# Packet count for SNAT rule should not increase for this packet
|
|
# Check by looking at the mangle counters — the packet should not have been marked
|
|
local mangle_out
|
|
mangle_out=$(dexec biscayne 'iptables -t mangle -L PREROUTING -v -n 2>/dev/null' || true)
|
|
echo " mangle PREROUTING rules (verify sport 8899 not matched):"
|
|
echo "$mangle_out" | grep -E "MARK|pkts" | head -5
|
|
|
|
# The fwmark rules only match sport 8001 and 9000-9025, so 8899 won't match.
|
|
# We can verify by checking that no new packets were marked.
|
|
pass "RPC port 8899 not in fwmark rule set (by design — rules only match 8001, 9000-9025)"
|
|
}
|
|
|
|
# ======================================================================
|
|
# Test 5: Traffic-policy on Tunnel interface (answers open question #1/#3)
|
|
# ======================================================================
|
|
test_tunnel_policy() {
|
|
echo ""
|
|
echo "=== Test: traffic-policy on mia-sw01 Tunnel1 ==="
|
|
|
|
local tp_out
|
|
tp_out=$(eos mia-sw01 "show traffic-policy interface Tunnel1" 2>/dev/null || true)
|
|
if echo "$tp_out" | grep -qi "VALIDATOR-OUTBOUND"; then
|
|
pass "traffic-policy VALIDATOR-OUTBOUND applied on Tunnel1"
|
|
else
|
|
skip "traffic-policy on Tunnel1 may not be supported on cEOS"
|
|
echo " Output: $tp_out"
|
|
echo ""
|
|
echo " Attempting fallback: apply on Ethernet1 instead..."
|
|
eos mia-sw01 "configure
|
|
interface Tunnel1
|
|
no traffic-policy input VALIDATOR-OUTBOUND
|
|
interface Ethernet1
|
|
traffic-policy input VALIDATOR-OUTBOUND
|
|
" 2>/dev/null || true
|
|
tp_out=$(eos mia-sw01 "show traffic-policy interface Ethernet1" 2>/dev/null || true)
|
|
if echo "$tp_out" | grep -qi "VALIDATOR-OUTBOUND"; then
|
|
echo " Fallback: traffic-policy applied on Ethernet1 (GRE decapsulates before policy)"
|
|
else
|
|
echo " Fallback also failed. Check mia-sw01 config manually."
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# ======================================================================
|
|
# Counters
|
|
# ======================================================================
|
|
show_counters() {
|
|
echo ""
|
|
echo "=== Traffic-policy counters ==="
|
|
|
|
echo "--- was-sw01 ---"
|
|
eos was-sw01 "show traffic-policy counters" 2>/dev/null || echo "(not available on cEOS)"
|
|
|
|
echo "--- mia-sw01 ---"
|
|
eos mia-sw01 "show traffic-policy counters" 2>/dev/null || echo "(not available on cEOS)"
|
|
|
|
echo ""
|
|
echo "--- biscayne iptables nat ---"
|
|
dexec biscayne 'iptables -t nat -L -v -n 2>/dev/null' || true
|
|
|
|
echo ""
|
|
echo "--- biscayne iptables mangle ---"
|
|
dexec biscayne 'iptables -t mangle -L PREROUTING -v -n 2>/dev/null' || true
|
|
|
|
echo ""
|
|
echo "--- biscayne policy routing ---"
|
|
dexec biscayne 'ip rule show 2>/dev/null' || true
|
|
dexec biscayne 'ip route show table ashburn 2>/dev/null' || true
|
|
}
|
|
|
|
# ======================================================================
|
|
# Main
|
|
# ======================================================================
|
|
main() {
|
|
local mode="${1:-all}"
|
|
|
|
case "$mode" in
|
|
setup)
|
|
setup
|
|
;;
|
|
inbound)
|
|
test_gre
|
|
test_inbound
|
|
;;
|
|
outbound)
|
|
test_outbound
|
|
;;
|
|
counters)
|
|
show_counters
|
|
;;
|
|
all)
|
|
setup
|
|
test_gre
|
|
test_tunnel_policy
|
|
test_inbound
|
|
test_outbound
|
|
test_isolation
|
|
show_counters
|
|
echo ""
|
|
echo "==============================="
|
|
echo "Results: $PASS passed, $FAIL failed, $SKIP skipped"
|
|
echo "==============================="
|
|
if ((FAIL > 0)); then
|
|
exit 1
|
|
fi
|
|
;;
|
|
*)
|
|
echo "Usage: $0 [setup|inbound|outbound|counters|all]"
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|