stack-orchestrator/playbooks/ashburn-relay-biscayne.yml

---
# Configure biscayne for Ashburn validator relay
#
# Sets up inbound DNAT (137.239.194.65 → kind node) and outbound SNAT +
# policy routing (validator traffic → doublezero0 → mia-sw01 → was-sw01).
#
# Usage:
#   # Full setup (inbound + outbound)
#   ansible-playbook playbooks/ashburn-relay-biscayne.yml
#
#   # Inbound only (DNAT rules)
#   ansible-playbook playbooks/ashburn-relay-biscayne.yml -t inbound
#
#   # Outbound only (SNAT + policy routing)
#   ansible-playbook playbooks/ashburn-relay-biscayne.yml -t outbound
#
#   # Pre-flight checks only
#   ansible-playbook playbooks/ashburn-relay-biscayne.yml -t preflight
#
#   # Rollback
#   ansible-playbook playbooks/ashburn-relay-biscayne.yml -e rollback=true

- name: Configure biscayne Ashburn validator relay
  hosts: biscayne
  gather_facts: false

  vars:
    ashburn_ip: 137.239.194.65
    kind_node_ip: 172.20.0.2
    kind_network: 172.20.0.0/16
    tunnel_gateway: 169.254.7.6
    tunnel_device: doublezero0
    fwmark: 100
    rt_table_name: ashburn
    rt_table_id: 100
    gossip_port: 8001
    dynamic_port_range_start: 9000
    dynamic_port_range_end: 9025
    rollback: false

  tasks:
    # ------------------------------------------------------------------
    # Rollback
    # ------------------------------------------------------------------
    - name: Rollback all Ashburn relay rules
      when: rollback | bool
      block:
        - name: Remove Ashburn IP from loopback
          ansible.builtin.command:
            cmd: ip addr del {{ ashburn_ip }}/32 dev lo
          failed_when: false

        - name: Remove inbound DNAT rules
          ansible.builtin.shell:
            cmd: |
              set -o pipefail
              iptables -t nat -D PREROUTING -p udp -d {{ ashburn_ip }} --dport {{ gossip_port }} -j DNAT --to-destination {{ kind_node_ip }}:{{ gossip_port }} 2>/dev/null || true
              iptables -t nat -D PREROUTING -p tcp -d {{ ashburn_ip }} --dport {{ gossip_port }} -j DNAT --to-destination {{ kind_node_ip }}:{{ gossip_port }} 2>/dev/null || true
              iptables -t nat -D PREROUTING -p udp -d {{ ashburn_ip }} --dport {{ dynamic_port_range_start }}:{{ dynamic_port_range_end }} -j DNAT --to-destination {{ kind_node_ip }} 2>/dev/null || true
            executable: /bin/bash

        - name: Remove outbound mangle rules
          ansible.builtin.shell:
            cmd: |
              set -o pipefail
              iptables -t mangle -D PREROUTING -s {{ kind_network }} -p udp --sport {{ gossip_port }} -j MARK --set-mark {{ fwmark }} 2>/dev/null || true
              iptables -t mangle -D PREROUTING -s {{ kind_network }} -p udp --sport {{ dynamic_port_range_start }}:{{ dynamic_port_range_end }} -j MARK --set-mark {{ fwmark }} 2>/dev/null || true
              iptables -t mangle -D PREROUTING -s {{ kind_network }} -p tcp --sport {{ gossip_port }} -j MARK --set-mark {{ fwmark }} 2>/dev/null || true
            executable: /bin/bash

        - name: Remove outbound SNAT rule
          ansible.builtin.shell:
            cmd: iptables -t nat -D POSTROUTING -m mark --mark {{ fwmark }} -j SNAT --to-source {{ ashburn_ip }} 2>/dev/null || true
            executable: /bin/bash

        - name: Remove policy routing
          ansible.builtin.shell:
            cmd: |
              ip rule del fwmark {{ fwmark }} table {{ rt_table_name }} 2>/dev/null || true
              ip route del default table {{ rt_table_name }} 2>/dev/null || true
            executable: /bin/bash

        - name: Persist cleaned iptables
          ansible.builtin.command:
            cmd: netfilter-persistent save

        - name: Remove if-up.d script
          ansible.builtin.file:
            path: /etc/network/if-up.d/ashburn-routing
            state: absent

        - name: Rollback complete
          ansible.builtin.debug:
            msg: "Ashburn relay rules removed. Old SHRED-RELAY DNAT (64.92.84.81:20000) is still in place."

        - name: End play after rollback
          ansible.builtin.meta: end_play

    # ------------------------------------------------------------------
    # Pre-flight checks
    # ------------------------------------------------------------------
    - name: Check doublezero0 tunnel is up
      ansible.builtin.command:
        cmd: ip link show {{ tunnel_device }}
      register: tunnel_status
      changed_when: false
      failed_when: "'UP' not in tunnel_status.stdout"
      tags: [preflight, inbound, outbound]

    - name: Check kind node is reachable
      ansible.builtin.command:
        cmd: ping -c 1 -W 2 {{ kind_node_ip }}
      register: kind_ping
      changed_when: false
      failed_when: kind_ping.rc != 0
      tags: [preflight, inbound]

    - name: Verify Docker preserves source ports (5 sec sample)
      ansible.builtin.shell:
        cmd: |
          set -o pipefail
          # Check if any validator traffic is flowing with original sport
          timeout 5 tcpdump -i br-cf46a62ab5b2 -nn -c 5 'udp src port 8001 or udp src portrange 9000-9025' 2>&1 | tail -5 || echo "No validator traffic captured in 5s (validator may not be running)"
        executable: /bin/bash
      register: sport_check
      changed_when: false
      failed_when: false
      tags: [preflight]

    - name: Show sport preservation check
      ansible.builtin.debug:
        var: sport_check.stdout_lines
      tags: [preflight]

    - name: Show existing iptables nat rules
      ansible.builtin.shell:
        cmd: iptables -t nat -L -v -n --line-numbers | head -60
        executable: /bin/bash
      register: existing_nat
      changed_when: false
      tags: [preflight]

    - name: Display existing NAT rules
      ansible.builtin.debug:
        var: existing_nat.stdout_lines
      tags: [preflight]

    # ------------------------------------------------------------------
    # Inbound: DNAT for 137.239.194.65 → kind node
    # ------------------------------------------------------------------
    - name: Add Ashburn IP to loopback
      ansible.builtin.command:
        cmd: ip addr add {{ ashburn_ip }}/32 dev lo
      register: add_ip
      changed_when: add_ip.rc == 0
      failed_when: "add_ip.rc != 0 and 'RTNETLINK answers: File exists' not in add_ip.stderr"
      tags: [inbound]

    - name: Add DNAT rules (inserted before DOCKER chain)
      ansible.builtin.shell:
        cmd: |
          set -o pipefail
          # DNAT rules must be before Docker's ADDRTYPE LOCAL rule, otherwise
          # Docker's PREROUTING chain swallows traffic to 137.239.194.65 (which
          # is on loopback and therefore type LOCAL).
          for rule in \
            "-p udp -d {{ ashburn_ip }} --dport {{ gossip_port }} -j DNAT --to-destination {{ kind_node_ip }}:{{ gossip_port }}" \
            "-p tcp -d {{ ashburn_ip }} --dport {{ gossip_port }} -j DNAT --to-destination {{ kind_node_ip }}:{{ gossip_port }}" \
            "-p udp -d {{ ashburn_ip }} --dport {{ dynamic_port_range_start }}:{{ dynamic_port_range_end }} -j DNAT --to-destination {{ kind_node_ip }}" \
          ; do
            if ! iptables -t nat -C PREROUTING $rule 2>/dev/null; then
              iptables -t nat -I PREROUTING 1 $rule
              echo "added: $rule"
            else
              echo "exists: $rule"
            fi
          done
        executable: /bin/bash
      register: dnat_result
      changed_when: "'added' in dnat_result.stdout"
      tags: [inbound]

    - name: Show DNAT result
      ansible.builtin.debug:
        var: dnat_result.stdout_lines
      tags: [inbound]

    # ------------------------------------------------------------------
    # Outbound: fwmark + SNAT + policy routing
    # ------------------------------------------------------------------
    - name: Mark outbound validator traffic (mangle PREROUTING)
      ansible.builtin.shell:
        cmd: |
          set -o pipefail
          for rule in \
            "-p udp -s {{ kind_network }} --sport {{ gossip_port }} -j MARK --set-mark {{ fwmark }}" \
            "-p udp -s {{ kind_network }} --sport {{ dynamic_port_range_start }}:{{ dynamic_port_range_end }} -j MARK --set-mark {{ fwmark }}" \
            "-p tcp -s {{ kind_network }} --sport {{ gossip_port }} -j MARK --set-mark {{ fwmark }}" \
          ; do
            if ! iptables -t mangle -C PREROUTING $rule 2>/dev/null; then
              iptables -t mangle -A PREROUTING $rule
              echo "added: $rule"
            else
              echo "exists: $rule"
            fi
          done
        executable: /bin/bash
      register: mangle_result
      changed_when: "'added' in mangle_result.stdout"
      tags: [outbound]

    - name: Show mangle result
      ansible.builtin.debug:
        var: mangle_result.stdout_lines
      tags: [outbound]

    - name: SNAT marked traffic to Ashburn IP (before Docker MASQUERADE)
      ansible.builtin.shell:
        cmd: |
          set -o pipefail
          # Check if rule already exists
          if iptables -t nat -C POSTROUTING -m mark --mark {{ fwmark }} -j SNAT --to-source {{ ashburn_ip }} 2>/dev/null; then
            echo "SNAT rule already exists"
          else
            iptables -t nat -I POSTROUTING 1 -m mark --mark {{ fwmark }} -j SNAT --to-source {{ ashburn_ip }}
            echo "SNAT rule inserted at position 1"
          fi
        executable: /bin/bash
      register: snat_result
      changed_when: "'inserted' in snat_result.stdout"
      tags: [outbound]

    - name: Show SNAT result
      ansible.builtin.debug:
        var: snat_result.stdout
      tags: [outbound]

    - name: Ensure rt_tables entry exists
      ansible.builtin.lineinfile:
        path: /etc/iproute2/rt_tables
        line: "{{ rt_table_id }} {{ rt_table_name }}"
        regexp: "^{{ rt_table_id }}\\s"
      tags: [outbound]

    - name: Add policy routing rule for fwmark
      ansible.builtin.shell:
        cmd: |
          if ip rule show | grep -q 'fwmark 0x64 lookup ashburn'; then
            echo "rule already exists"
          else
            ip rule add fwmark {{ fwmark }} table {{ rt_table_name }}
            echo "rule added"
          fi
        executable: /bin/bash
      register: rule_result
      changed_when: "'added' in rule_result.stdout"
      tags: [outbound]

    - name: Add default route via doublezero0 in ashburn table
      ansible.builtin.shell:
        cmd: ip route replace default via {{ tunnel_gateway }} dev {{ tunnel_device }} table {{ rt_table_name }}
        executable: /bin/bash
      changed_when: true
      tags: [outbound]

    # ------------------------------------------------------------------
    # Persistence
    # ------------------------------------------------------------------
    - name: Save iptables rules
      ansible.builtin.command:
        cmd: netfilter-persistent save
      tags: [inbound, outbound]

    - name: Install if-up.d persistence script
      ansible.builtin.copy:
        src: files/ashburn-routing-ifup.sh
        dest: /etc/network/if-up.d/ashburn-routing
        mode: '0755'
        owner: root
        group: root
      tags: [outbound]

    # ------------------------------------------------------------------
    # Verification
    # ------------------------------------------------------------------
    - name: Show NAT rules
      ansible.builtin.shell:
        cmd: iptables -t nat -L -v -n --line-numbers 2>&1 | head -40
        executable: /bin/bash
      register: nat_rules
      changed_when: false
      tags: [inbound, outbound]

    - name: Show mangle rules
      ansible.builtin.shell:
        cmd: iptables -t mangle -L -v -n 2>&1
        executable: /bin/bash
      register: mangle_rules
      changed_when: false
      tags: [outbound]

    - name: Show policy routing
      ansible.builtin.shell:
        cmd: |
          echo "=== ip rule ==="
          ip rule show
          echo ""
          echo "=== ashburn routing table ==="
          ip route show table {{ rt_table_name }} 2>/dev/null || echo "table empty"
        executable: /bin/bash
      register: routing_info
      changed_when: false
      tags: [outbound]

    - name: Show loopback addresses
      ansible.builtin.shell:
        cmd: ip addr show lo | grep inet
        executable: /bin/bash
      register: lo_addrs
      changed_when: false
      tags: [inbound]

    - name: Display verification
      ansible.builtin.debug:
        msg:
          nat_rules: "{{ nat_rules.stdout_lines }}"
          mangle_rules: "{{ mangle_rules.stdout_lines | default([]) }}"
          routing: "{{ routing_info.stdout_lines | default([]) }}"
          loopback: "{{ lo_addrs.stdout_lines | default([]) }}"
      tags: [inbound, outbound]

    - name: Summary
      ansible.builtin.debug:
        msg: |
          === Ashburn Relay Setup Complete ===
          Ashburn IP: {{ ashburn_ip }} (on lo)
          Inbound DNAT: {{ ashburn_ip }}:8001,9000-9025 → {{ kind_node_ip }}
          Outbound SNAT: {{ kind_network }} sport 8001,9000-9025 → {{ ashburn_ip }}
          Policy route: fwmark {{ fwmark }} → table {{ rt_table_name }} → via {{ tunnel_gateway }} dev {{ tunnel_device }}
          Persisted: iptables-persistent + /etc/network/if-up.d/ashburn-routing

          Next steps:
          1. Verify inbound: ping {{ ashburn_ip }} from external host
          2. Verify outbound: tcpdump on was-sw01 for src {{ ashburn_ip }}
          3. Check validator gossip ContactInfo shows {{ ashburn_ip }} for all addresses