From 082702bc29a43ff12d0b9d2a2232d1e804be0e24 Mon Sep 17 00:00:00 2001 From: Tony Du Date: Thu, 6 Mar 2025 18:44:42 -0800 Subject: [PATCH] feat: Support port mapping more robustly --- .../inventory/full/group_vars/vpn/main.yml | 42 +++++++++++++++---- .../full/group_vars/vpn_client/main.yml | 33 +++++++++++++++ .../full/group_vars/vpn_server/main.yml | 24 +++++------ ansible/wings.yml | 3 +- tf/modules/embassy/main.tf | 4 +- 5 files changed, 84 insertions(+), 22 deletions(-) diff --git a/ansible/inventory/full/group_vars/vpn/main.yml b/ansible/inventory/full/group_vars/vpn/main.yml index 260ffa8..90d48b5 100644 --- a/ansible/inventory/full/group_vars/vpn/main.yml +++ b/ansible/inventory/full/group_vars/vpn/main.yml @@ -2,15 +2,15 @@ wireguard_remote_directory: /etc/wireguard wireguard_interface_restart: false +# wireguard_service_enabled: false +# wireguard_service_state: stopped wireguard_service_enabled: true wireguard_service_state: started # We need to keep the NAT mapping open: # https://www.wireguard.com/quickstart/#nat-and-firewall-traversal-persistence -# I've tested 25 seconds, which seems to be too low. The mapping still seems -# to be broken every once in a while. -# Or, it might be because PersistentKeepalive is actually also needed on the -# server but it's being omitted currently. See the issue I opened: +# It seems like we do need this on the server for server->client, but it's +# being omitted currently. See the issue I opened: # https://github.com/githubixx/ansible-role-wireguard/issues/217#issue-2871281915 wireguard_persistent_keepalive: 25 @@ -27,9 +27,20 @@ nat_map: vpn_ipv4: "{{ wireguard_ipv4_subnet | ansible.utils.ipaddr('16') }}" vps_ipv6: "{{ public_ipv6_subnet | ansible.utils.ipaddr('16') }}" vps_ipv4: "{{ ansible_default_ipv4.address }}" - port_mappings: - - external_port: 20050 - internal_port: 20050 + + # With IPv6, we don't have to map different internal/external ports because + # we have separate port spaces on separate IPv6 addresses + ipv6_port_ranges: + # Anything that's accepted into the --dport argument of iptables is a + # valid entry. + - 2022 + - 16261:16262 + - 20000:20100 + + # With IPv4, we do, because we share a single public IPv4 address + # ipv4_port_mapping: + # - external_port: 20050 + # internal_port: 20050 moirai-lachesis.local: vpn_ipv6: "{{ wireguard_ipv6_subnet | ansible.utils.ipaddr('17') }}" @@ -37,8 +48,25 @@ nat_map: vps_ipv6: "{{ public_ipv6_subnet | ansible.utils.ipaddr('17') }}" vps_ipv4: "{{ ansible_default_ipv4.address }}" + ipv6_port_ranges: + - 2022 + - 16261:16262 + - 20000:20100 + + ipv4_port_mapping: + # Project Zomboid + - external_port: 16261 + internal_port: 16261 + - external_port: 16262 + internal_port: 16262 + moirai-atropos.local: vpn_ipv6: "{{ wireguard_ipv6_subnet | ansible.utils.ipaddr('18') }}" vpn_ipv4: "{{ wireguard_ipv4_subnet | ansible.utils.ipaddr('18') }}" vps_ipv6: "{{ public_ipv6_subnet | ansible.utils.ipaddr('18') }}" vps_ipv4: "{{ ansible_default_ipv4.address }}" + + ipv6_port_ranges: + - 2022 + - 16261:16262 + - 20000:20100 diff --git a/ansible/inventory/full/group_vars/vpn_client/main.yml b/ansible/inventory/full/group_vars/vpn_client/main.yml index 196f015..30a7d6d 100644 --- a/ansible/inventory/full/group_vars/vpn_client/main.yml +++ b/ansible/inventory/full/group_vars/vpn_client/main.yml @@ -9,10 +9,43 @@ wireguard_endpoint: "" # wireguard_dns: 10.0.123.123 # don't route local addresses through the wg tunnel +# ipv6 is already done wireguard_preup: - ip route add 10.0.0.0/16 via 10.0.0.1 dev eth0 proto static onlink + # This is retarded. This is so, so stupid. For some absolutely cursed reason, + # **ONLY THE MINECRAFT API** runs into trouble routing through the wireguard + # tunnel **ONLY IN DOCKER IMAGES**. This makes it so that I can't get + # the public keys necessary for players to authenticate. + # + # So I'm just going to whitelist their IP address by hardcoding into here + # to route their address through my home network interface, which seems to + # work. + # + # To future me, who will inevitably think I just missed something: + # - Yes, I've verified beyond a shadow of a doubt that + # `curl https://api.minecraftservices.com/publickeys` works on the host + # machine, through wireguard. + # - Yes, this includes both IPv4 and IPv6 + # - Yes, I've verified beyond a shadow of a doubt that doing the same does + # not work in a Docker container (with non-host networking). + # - The network traffic gave me tons of mixed signals: + # - Sometimes, it seemed like packets were received on embassy, but never + # forwarded into the wireguard interface. + # - Sometimes, it seemed like packets were making it all the way back into + # the moirai node, but never made it into the pterodactyl0 interface. + # - Sometimes, it seemed like packets made it to the pterodactyl0 interface, + # but `curl` never ended up seeing any packets. + # + # If exploring this further, try lowering the MTU globally. My only guess is + # that fragmenting because of the smaller MTU through the wireguard tunnel + # causes some problem. (Why would it not cause problems with other + # connections? I have no idea.) + - ip route add 13.107.253.70 via 10.0.0.1 dev eth0 proto static onlink + - ip route add 13.107.246.70 via 10.0.0.1 dev eth0 proto static onlink wireguard_postdown: + - ip route del 13.107.246.70 via 10.0.0.1 dev eth0 proto static onlink + - ip route del 13.107.253.70 via 10.0.0.1 dev eth0 proto static onlink - ip route del 10.0.0.0/16 via 10.0.0.1 dev eth0 proto static onlink # Ok, I could not get the stuff below working properly. What I _wanted_ to do diff --git a/ansible/inventory/full/group_vars/vpn_server/main.yml b/ansible/inventory/full/group_vars/vpn_server/main.yml index 9cb3480..3ee61b9 100644 --- a/ansible/inventory/full/group_vars/vpn_server/main.yml +++ b/ansible/inventory/full/group_vars/vpn_server/main.yml @@ -25,22 +25,19 @@ wireguard_postup: | # Incoming packets to this node's public IP are DNAT'd and forwarded to the # matching internal VPN IP - - ip6tables -t nat -A PREROUTING -p tcp -d {{ value.vps_ipv6 | ansible.utils.ipaddr('address') }} --dport 20000:20100 -j DNAT --to-destination {{ value.vpn_ipv6 | ansible.utils.ipaddr('address') }} - # Same for SFTP over TCP. - - ip6tables -t nat -A PREROUTING -p tcp -d {{ value.vps_ipv6 | ansible.utils.ipaddr('address') }} --dport 2022 -j DNAT --to-destination {{ value.vpn_ipv6 | ansible.utils.ipaddr('address') }} + {% for range in value.ipv6_port_ranges | default([]) %} + - ip6tables -t nat -A PREROUTING -p tcp -d {{ value.vps_ipv6 | ansible.utils.ipaddr('address') }} --dport {{ range }} -j DNAT --to-destination {{ value.vpn_ipv6 | ansible.utils.ipaddr('address') }} + - ip6tables -t nat -A PREROUTING -p udp -d {{ value.vps_ipv6 | ansible.utils.ipaddr('address') }} --dport {{ range }} -j DNAT --to-destination {{ value.vpn_ipv6 | ansible.utils.ipaddr('address') }} + {% endfor %} # Incoming packets from an internal VPN IP are SNAT'd to use this node's public # IP. I think `-j MASQUERADE` might work here rather than doing the SNAT # manually(?), but I don't mind being explicit here. - ip6tables -t nat -A POSTROUTING -p tcp -s {{ value.vpn_ipv6 | ansible.utils.ipaddr('address') }} -j SNAT --to-source {{ value.vps_ipv6 | ansible.utils.ipaddr('address') }} - - # Same thing with UDP. We do this selectively so we don't mess with things - # like ICMP6 and whatnot. - - ip6tables -t nat -A PREROUTING -p udp -d {{ value.vps_ipv6 | ansible.utils.ipaddr('address') }} --dport 20000:20100 -j DNAT --to-destination {{ value.vpn_ipv6 | ansible.utils.ipaddr('address') }} - ip6tables -t nat -A POSTROUTING -p udp -s {{ value.vpn_ipv6 | ansible.utils.ipaddr('address') }} -j SNAT --to-source {{ value.vps_ipv6 | ansible.utils.ipaddr('address') }} # IPv4 will have manual port mapping - {% for mapping in value.port_mappings | default([]) %} + {% for mapping in value.ipv4_port_mapping | default([]) %} - iptables -t nat -A PREROUTING -p tcp -d {{ value.vps_ipv4 | ansible.utils.ipaddr('address') }} --dport {{ mapping.external_port }} -j DNAT --to-destination {{ value.vpn_ipv4 | ansible.utils.ipaddr('address') }}:{{ mapping.internal_port }} - iptables -t nat -A PREROUTING -p udp -d {{ value.vps_ipv4 | ansible.utils.ipaddr('address') }} --dport {{ mapping.external_port }} -j DNAT --to-destination {{ value.vpn_ipv4 | ansible.utils.ipaddr('address') }}:{{ mapping.internal_port }} {% endfor %} @@ -54,16 +51,17 @@ wireguard_predown: | {% filter from_yaml %} {% for value in (nat_map | dict2items | map(attribute='value') | reverse) %} - iptables -t nat -D POSTROUTING -s {{ value.vpn_ipv4 | ansible.utils.ipaddr('address') }} -j MASQUERADE - {% for mapping in value.port_mappings | default([]) | reverse %} + {% for mapping in value.ipv4_port_mapping | default([]) | reverse %} - iptables -t nat -D PREROUTING -p udp -d {{ value.vps_ipv4 | ansible.utils.ipaddr('address') }} --dport {{ mapping.external_port }} -j DNAT --to-destination {{ value.vpn_ipv4 | ansible.utils.ipaddr('address') }}:{{ mapping.internal_port }} - iptables -t nat -D PREROUTING -p tcp -d {{ value.vps_ipv4 | ansible.utils.ipaddr('address') }} --dport {{ mapping.external_port }} -j DNAT --to-destination {{ value.vpn_ipv4 | ansible.utils.ipaddr('address') }}:{{ mapping.internal_port }} {% endfor %} - ip6tables -t nat -D POSTROUTING -p udp -s {{ value.vpn_ipv6 | ansible.utils.ipaddr('address') }} -j SNAT --to-source {{ value.vps_ipv6 | ansible.utils.ipaddr('address') }} - - ip6tables -t nat -D PREROUTING -p udp -d {{ value.vps_ipv6 | ansible.utils.ipaddr('address') }} --dport 20000:20100 -j DNAT --to-destination {{ value.vpn_ipv6 | ansible.utils.ipaddr('address') }} - ip6tables -t nat -D POSTROUTING -p tcp -s {{ value.vpn_ipv6 | ansible.utils.ipaddr('address') }} -j SNAT --to-source {{ value.vps_ipv6 | ansible.utils.ipaddr('address') }} - - ip6tables -t nat -D PREROUTING -p tcp -d {{ value.vps_ipv6 | ansible.utils.ipaddr('address') }} --dport 2022 -j DNAT --to-destination {{ value.vpn_ipv6 | ansible.utils.ipaddr('address') }} - - ip6tables -t nat -D PREROUTING -p tcp -d {{ value.vps_ipv6 | ansible.utils.ipaddr('address') }} --dport 20000:20100 -j DNAT --to-destination {{ value.vpn_ipv6 | ansible.utils.ipaddr('address') }} + {% for range in value.ipv6_port_ranges | default([]) | reverse %} + - ip6tables -t nat -D PREROUTING -p udp -d {{ value.vps_ipv6 | ansible.utils.ipaddr('address') }} --dport {{ range }} -j DNAT --to-destination {{ value.vpn_ipv6 | ansible.utils.ipaddr('address') }} + - ip6tables -t nat -D PREROUTING -p tcp -d {{ value.vps_ipv6 | ansible.utils.ipaddr('address') }} --dport {{ range }} -j DNAT --to-destination {{ value.vpn_ipv6 | ansible.utils.ipaddr('address') }} + {% endfor %} - ip -6 addr del {{ value.vps_ipv6 }} dev eth0 {% endfor %} - iptables -D FORWARD -o wg0 -j ACCEPT @@ -88,4 +86,6 @@ wireguard_postdown: # a list of IPs that should be routed to the "server" (because everyone is a # peer in a fully meshed network) wireguard_allowed_ips: "0.0.0.0/0, ::0/0" +# Disable IPv4 +# wireguard_allowed_ips: "::0/0" diff --git a/ansible/wings.yml b/ansible/wings.yml index 5031650..bd64e81 100644 --- a/ansible/wings.yml +++ b/ansible/wings.yml @@ -18,6 +18,7 @@ Expected ipv6_subnet to be defined. This should have been done in Terraform or otherwise. + tasks: # As mentioned in the other file, if I set this statically on group_vars, # things seem to break. @@ -67,7 +68,7 @@ /etc/wireguard/wg0.conf - service: name: wg-quick@wg0 - state: "{{ 'restarted' if wireguard_service_state != 'stopped' }}" + state: "{{ 'restarted' if wireguard_service_state != 'stopped' else 'stopped' }}" enabled: "{{ wireguard_service_enabled }}" - name: Install wings diff --git a/tf/modules/embassy/main.tf b/tf/modules/embassy/main.tf index 178b439..9e073b0 100644 --- a/tf/modules/embassy/main.tf +++ b/tf/modules/embassy/main.tf @@ -42,7 +42,7 @@ resource "linode_firewall" "embassy" { label = "allow-forward-tcp" action = "ACCEPT" protocol = "TCP" - ports = "20000-20100" + ports = "16261-16262,20000-20100" ipv4 = ["0.0.0.0/0"] ipv6 = ["::/0"] } @@ -51,7 +51,7 @@ resource "linode_firewall" "embassy" { label = "allow-forward-udp" action = "ACCEPT" protocol = "UDP" - ports = "20000-20100" + ports = "16261-16262,20000-20100" ipv4 = ["0.0.0.0/0"] ipv6 = ["::/0"] }