--- # Configure biscayne for Ashburn validator relay # # Sets up inbound DNAT (137.239.194.65 → kind node) and outbound SNAT + # policy routing (validator traffic → GRE tunnel → mia-sw01 → was-sw01). # # Uses a dedicated GRE tunnel to mia-sw01 (NOT the DoubleZero-managed # doublezero0/Tunnel500). The tunnel source is biscayne's public IP # (186.233.184.235) and the destination is mia-sw01's free LAN IP # (209.42.167.137). # # Usage: # # Full setup (inbound + outbound) # ansible-playbook playbooks/ashburn-relay-biscayne.yml # # # Inbound only (DNAT rules) # ansible-playbook playbooks/ashburn-relay-biscayne.yml -t inbound # # # Outbound only (SNAT + policy routing) # ansible-playbook playbooks/ashburn-relay-biscayne.yml -t outbound # # # Pre-flight checks only # ansible-playbook playbooks/ashburn-relay-biscayne.yml -t preflight # # # Rollback # ansible-playbook playbooks/ashburn-relay-biscayne.yml -e rollback=true - name: Configure biscayne Ashburn validator relay hosts: all gather_facts: false vars: ashburn_ip: 137.239.194.65 kind_node_ip: 172.20.0.2 kind_network: 172.20.0.0/16 # New dedicated GRE tunnel (not DZ-managed doublezero0) tunnel_device: gre-ashburn tunnel_local_ip: 169.254.100.1 # biscayne end of /31 tunnel_remote_ip: 169.254.100.0 # mia-sw01 end of /31 tunnel_src: 186.233.184.235 # biscayne public IP tunnel_dst: 209.42.167.137 # mia-sw01 free LAN IP fwmark: 100 rt_table_name: ashburn rt_table_id: 100 gossip_port: 8001 dynamic_port_range_start: 9000 dynamic_port_range_end: 9025 rollback: false tasks: # ------------------------------------------------------------------ # Rollback # ------------------------------------------------------------------ - name: Rollback all Ashburn relay rules when: rollback | bool block: - name: Remove Ashburn IP from loopback ansible.builtin.command: cmd: ip addr del {{ ashburn_ip }}/32 dev lo failed_when: false changed_when: false - name: Remove GRE tunnel ansible.builtin.shell: cmd: | ip link set {{ tunnel_device }} down 2>/dev/null || true ip tunnel del {{ tunnel_device }} 2>/dev/null || true executable: /bin/bash changed_when: false - name: Remove inbound DNAT rules ansible.builtin.shell: cmd: | set -o pipefail iptables -t nat -D PREROUTING -p udp -d {{ ashburn_ip }} \ --dport {{ gossip_port }} \ -j DNAT --to-destination {{ kind_node_ip }}:{{ gossip_port }} \ 2>/dev/null || true iptables -t nat -D PREROUTING -p tcp -d {{ ashburn_ip }} \ --dport {{ gossip_port }} \ -j DNAT --to-destination {{ kind_node_ip }}:{{ gossip_port }} \ 2>/dev/null || true iptables -t nat -D PREROUTING -p udp -d {{ ashburn_ip }} \ --dport {{ dynamic_port_range_start }}:{{ dynamic_port_range_end }} \ -j DNAT --to-destination {{ kind_node_ip }} \ 2>/dev/null || true executable: /bin/bash changed_when: false - name: Remove outbound mangle rules ansible.builtin.shell: cmd: | set -o pipefail iptables -t mangle -D PREROUTING -s {{ kind_network }} \ -p udp --sport {{ gossip_port }} \ -j MARK --set-mark {{ fwmark }} 2>/dev/null || true iptables -t mangle -D PREROUTING -s {{ kind_network }} \ -p udp --sport {{ dynamic_port_range_start }}:{{ dynamic_port_range_end }} \ -j MARK --set-mark {{ fwmark }} 2>/dev/null || true iptables -t mangle -D PREROUTING -s {{ kind_network }} \ -p tcp --sport {{ gossip_port }} \ -j MARK --set-mark {{ fwmark }} 2>/dev/null || true iptables -t mangle -D PREROUTING -s {{ kind_network }} \ -p tcp --dport {{ gossip_port }} \ -j MARK --set-mark {{ fwmark }} 2>/dev/null || true executable: /bin/bash changed_when: false - name: Remove outbound SNAT rule ansible.builtin.shell: cmd: iptables -t nat -D POSTROUTING -m mark --mark {{ fwmark }} -j SNAT --to-source {{ ashburn_ip }} 2>/dev/null || true executable: /bin/bash changed_when: false - name: Remove policy routing ansible.builtin.shell: cmd: | ip rule del fwmark {{ fwmark }} table {{ rt_table_name }} 2>/dev/null || true ip route del default table {{ rt_table_name }} 2>/dev/null || true executable: /bin/bash changed_when: false - name: Disable and remove ashburn-relay service ansible.builtin.systemd: name: ashburn-relay.service enabled: false state: stopped failed_when: false - name: Remove ashburn-relay files ansible.builtin.file: path: "{{ item }}" state: absent loop: - /etc/systemd/system/ashburn-relay.service - /usr/local/sbin/ashburn-relay-setup.sh - /etc/network/if-up.d/ashburn-routing - name: Rollback complete ansible.builtin.debug: msg: "Ashburn relay rules removed." - name: End play after rollback ansible.builtin.meta: end_play # ------------------------------------------------------------------ # Pre-flight checks # ------------------------------------------------------------------ - name: Check tunnel destination is reachable ansible.builtin.command: cmd: ping -c 1 -W 2 {{ tunnel_dst }} register: tunnel_dst_ping changed_when: false failed_when: tunnel_dst_ping.rc != 0 tags: [preflight, outbound] - name: Check kind node is reachable ansible.builtin.command: cmd: ping -c 1 -W 2 {{ kind_node_ip }} register: kind_ping changed_when: false failed_when: kind_ping.rc != 0 tags: [preflight, inbound] - name: Show existing iptables nat rules ansible.builtin.shell: cmd: set -o pipefail && iptables -t nat -L -v -n --line-numbers | head -60 executable: /bin/bash register: existing_nat changed_when: false tags: [preflight] - name: Display existing NAT rules ansible.builtin.debug: var: existing_nat.stdout_lines tags: [preflight] - name: Check for existing GRE tunnel ansible.builtin.shell: cmd: ip tunnel show {{ tunnel_device }} 2>&1 || echo "tunnel does not exist" executable: /bin/bash register: existing_tunnel changed_when: false tags: [preflight] - name: Display existing tunnel ansible.builtin.debug: var: existing_tunnel.stdout_lines tags: [preflight] # ------------------------------------------------------------------ # GRE tunnel setup # ------------------------------------------------------------------ - name: Create GRE tunnel ansible.builtin.shell: cmd: | set -o pipefail if ip tunnel show {{ tunnel_device }} 2>/dev/null; then echo "tunnel already exists" else ip tunnel add {{ tunnel_device }} mode gre local {{ tunnel_src }} remote {{ tunnel_dst }} ttl 64 ip addr add {{ tunnel_local_ip }}/31 dev {{ tunnel_device }} ip link set {{ tunnel_device }} up mtu 8972 echo "tunnel created" fi executable: /bin/bash register: tunnel_result changed_when: "'created' in tunnel_result.stdout" tags: [outbound] - name: Show tunnel result ansible.builtin.debug: var: tunnel_result.stdout_lines tags: [outbound] # ------------------------------------------------------------------ # Inbound: DNAT for 137.239.194.65 → kind node # ------------------------------------------------------------------ - name: Add Ashburn IP to loopback ansible.builtin.command: cmd: ip addr add {{ ashburn_ip }}/32 dev lo register: add_ip changed_when: add_ip.rc == 0 failed_when: "add_ip.rc != 0 and 'already assigned' not in add_ip.stderr and 'File exists' not in add_ip.stderr" tags: [inbound] - name: Add DNAT rules (inserted before DOCKER chain) ansible.builtin.shell: cmd: | set -o pipefail # DNAT rules must be before Docker's ADDRTYPE LOCAL rule, otherwise # Docker's PREROUTING chain swallows traffic to 137.239.194.65 (which # is on loopback and therefore type LOCAL). for rule in \ "-p udp -d {{ ashburn_ip }} --dport {{ gossip_port }} -j DNAT --to-destination {{ kind_node_ip }}:{{ gossip_port }}" \ "-p tcp -d {{ ashburn_ip }} --dport {{ gossip_port }} -j DNAT --to-destination {{ kind_node_ip }}:{{ gossip_port }}" \ "-p udp -d {{ ashburn_ip }} --dport {{ dynamic_port_range_start }}:{{ dynamic_port_range_end }} -j DNAT --to-destination {{ kind_node_ip }}" \ ; do if ! iptables -t nat -C PREROUTING $rule 2>/dev/null; then iptables -t nat -I PREROUTING 1 $rule echo "added: $rule" else echo "exists: $rule" fi done executable: /bin/bash register: dnat_result changed_when: "'added' in dnat_result.stdout" tags: [inbound] - name: Show DNAT result ansible.builtin.debug: var: dnat_result.stdout_lines tags: [inbound] # ------------------------------------------------------------------ # Outbound: fwmark + SNAT + policy routing via new tunnel # ------------------------------------------------------------------ - name: Mark outbound validator traffic (mangle PREROUTING) ansible.builtin.shell: cmd: | set -o pipefail for rule in \ "-p udp -s {{ kind_network }} --sport {{ gossip_port }} -j MARK --set-mark {{ fwmark }}" \ "-p udp -s {{ kind_network }} --sport {{ dynamic_port_range_start }}:{{ dynamic_port_range_end }} -j MARK --set-mark {{ fwmark }}" \ "-p tcp -s {{ kind_network }} --sport {{ gossip_port }} -j MARK --set-mark {{ fwmark }}" \ "-p tcp -s {{ kind_network }} --dport {{ gossip_port }} -j MARK --set-mark {{ fwmark }}" \ ; do if ! iptables -t mangle -C PREROUTING $rule 2>/dev/null; then iptables -t mangle -A PREROUTING $rule echo "added: $rule" else echo "exists: $rule" fi done executable: /bin/bash register: mangle_result changed_when: "'added' in mangle_result.stdout" tags: [outbound] - name: Show mangle result ansible.builtin.debug: var: mangle_result.stdout_lines tags: [outbound] - name: SNAT marked traffic to Ashburn IP (before Docker MASQUERADE) ansible.builtin.shell: cmd: | set -o pipefail if iptables -t nat -C POSTROUTING -m mark --mark {{ fwmark }} -j SNAT --to-source {{ ashburn_ip }} 2>/dev/null; then echo "SNAT rule already exists" else iptables -t nat -I POSTROUTING 1 -m mark --mark {{ fwmark }} -j SNAT --to-source {{ ashburn_ip }} echo "SNAT rule inserted at position 1" fi executable: /bin/bash register: snat_result changed_when: "'inserted' in snat_result.stdout" tags: [outbound] - name: Show SNAT result ansible.builtin.debug: var: snat_result.stdout tags: [outbound] - name: Ensure rt_tables entry exists ansible.builtin.lineinfile: path: /etc/iproute2/rt_tables line: "{{ rt_table_id }} {{ rt_table_name }}" regexp: "^{{ rt_table_id }}\\s" tags: [outbound] - name: Add policy routing rule for fwmark ansible.builtin.shell: cmd: | set -o pipefail if ip rule show | grep -q 'fwmark 0x64 lookup ashburn'; then echo "rule already exists" else ip rule add fwmark {{ fwmark }} table {{ rt_table_name }} echo "rule added" fi executable: /bin/bash register: rule_result changed_when: "'added' in rule_result.stdout" tags: [outbound] - name: Add default route via GRE tunnel in ashburn table ansible.builtin.shell: cmd: ip route replace default via {{ tunnel_remote_ip }} dev {{ tunnel_device }} table {{ rt_table_name }} executable: /bin/bash changed_when: true tags: [outbound] # ------------------------------------------------------------------ # Persistence # ------------------------------------------------------------------ # A systemd oneshot service replaces both if-up.d (which depends on # networking.service, inactive on this host) and netfilter-persistent # (which runs before Docker, so Docker's chain setup blows away rules). # This service runs After=docker.service and idempotently applies all # tunnel, iptables, and policy routing state. - name: Install ashburn-relay systemd service ansible.builtin.copy: dest: /etc/systemd/system/ashburn-relay.service mode: "0644" content: | [Unit] Description=Ashburn validator relay (GRE tunnel, iptables, policy routing) After=docker.service network-online.target Wants=network-online.target [Service] Type=oneshot RemainAfterExit=yes ExecStart=/usr/local/sbin/ashburn-relay-setup.sh [Install] WantedBy=multi-user.target register: relay_unit tags: [inbound, outbound] - name: Install ashburn-relay setup script ansible.builtin.template: src: files/ashburn-relay-setup.sh.j2 dest: /usr/local/sbin/ashburn-relay-setup.sh mode: "0755" register: relay_script tags: [inbound, outbound] - name: Reload systemd and enable ashburn-relay ansible.builtin.systemd: name: ashburn-relay.service daemon_reload: "{{ relay_unit.changed or relay_script.changed }}" enabled: true tags: [inbound, outbound] - name: Remove stale if-up.d script ansible.builtin.file: path: /etc/network/if-up.d/ashburn-routing state: absent tags: [inbound, outbound] # ------------------------------------------------------------------ # Verification # ------------------------------------------------------------------ - name: Show tunnel status ansible.builtin.shell: cmd: | echo "=== tunnel ===" ip tunnel show {{ tunnel_device }} echo "" echo "=== tunnel addr ===" ip addr show {{ tunnel_device }} echo "" echo "=== ping tunnel peer ===" ping -c 1 -W 2 {{ tunnel_remote_ip }} 2>&1 || echo "tunnel peer unreachable" executable: /bin/bash register: tunnel_status changed_when: false tags: [outbound] - name: Show NAT rules ansible.builtin.shell: cmd: set -o pipefail && iptables -t nat -L -v -n --line-numbers 2>&1 | head -40 executable: /bin/bash register: nat_rules changed_when: false tags: [inbound, outbound] - name: Show mangle rules ansible.builtin.shell: cmd: iptables -t mangle -L -v -n 2>&1 executable: /bin/bash register: mangle_rules changed_when: false tags: [outbound] - name: Show policy routing ansible.builtin.shell: cmd: | echo "=== ip rule ===" ip rule show echo "" echo "=== ashburn routing table ===" ip route show table {{ rt_table_name }} 2>/dev/null || echo "table empty" executable: /bin/bash register: routing_info changed_when: false tags: [outbound] - name: Show loopback addresses ansible.builtin.shell: cmd: set -o pipefail && ip addr show lo | grep inet executable: /bin/bash register: lo_addrs changed_when: false tags: [inbound] - name: Display verification ansible.builtin.debug: msg: tunnel: "{{ tunnel_status.stdout_lines | default([]) }}" nat_rules: "{{ nat_rules.stdout_lines }}" mangle_rules: "{{ mangle_rules.stdout_lines | default([]) }}" routing: "{{ routing_info.stdout_lines | default([]) }}" loopback: "{{ lo_addrs.stdout_lines | default([]) }}" tags: [inbound, outbound] - name: Summary ansible.builtin.debug: msg: | === Ashburn Relay Setup Complete === Ashburn IP: {{ ashburn_ip }} (on lo) GRE tunnel: {{ tunnel_device }} ({{ tunnel_src }} → {{ tunnel_dst }}) link: {{ tunnel_local_ip }}/31 ↔ {{ tunnel_remote_ip }}/31 Inbound DNAT: {{ ashburn_ip }}:8001,9000-9025 → {{ kind_node_ip }} Outbound SNAT: {{ kind_network }} sport 8001,9000-9025 → {{ ashburn_ip }} Policy route: fwmark {{ fwmark }} → table {{ rt_table_name }} → via {{ tunnel_remote_ip }} dev {{ tunnel_device }} Next steps: 1. Apply mia-sw01 config (Tunnel100 must be up on both sides) 2. Verify tunnel: ping {{ tunnel_remote_ip }} 3. Test from kelce: echo test | nc -u -w 1 137.239.194.65 9000 4. Check validator gossip ContactInfo shows {{ ashburn_ip }} for all addresses