diff --git a/ansible/inventory/full/group_vars/vpn/main.yml b/ansible/inventory/full/group_vars/vpn/main.yml index 3ecefe7..d543f9e 100644 --- a/ansible/inventory/full/group_vars/vpn/main.yml +++ b/ansible/inventory/full/group_vars/vpn/main.yml @@ -3,9 +3,8 @@ ansible_user: ubuntu # Directory to store WireGuard configuration on the remote hosts wireguard_remote_directory: /etc/wireguard -wireguard_interface_restart: true -# TODO: Enable this when stable -wireguard_service_enabled: false +wireguard_interface_restart: false +wireguard_service_enabled: true wireguard_service_state: started # We can generate this dynamically, but it really doesn't seem like it's worth diff --git a/ansible/inventory/full/group_vars/vpn_client/main.yml b/ansible/inventory/full/group_vars/vpn_client/main.yml index 6e53b6c..8a71155 100644 --- a/ansible/inventory/full/group_vars/vpn_client/main.yml +++ b/ansible/inventory/full/group_vars/vpn_client/main.yml @@ -15,3 +15,65 @@ wireguard_preup: wireguard_postdown: - ip route del 10.0.0.0/16 via 10.0.0.1 dev eth0 proto static onlink + +# Ok, I could not get the stuff below working properly. What I _wanted_ to do +# was make it so that _only_ traffic that was sent from the wireguard tunnel +# gets sent back to the wireguard tunnel, rather than blindly routing based +# on network. In other words, I wanted policy-based-routing (PBR). +# +# While I got TCP to work perfectly, what seems to happen for UDP is: +# - An application listens on 0.0.0.0: +# - Inbound packet on wg0 is marked in conntrack and sent to 10.4.4.33: +# on the wg0 iface +# - Outbound packet is generated with a source IP of 10.0.29.40 on the eth0 +# interface +# - Restoring the mark then fails because the source/dest IPs don't match and +# the PBR never happens. +# +# What's puzzling is that this _only_ happens for UDP. The 3-way handshake on +# TCP seems to work, as well as subsequent conversation. +# +# If we bind on 10.4.4.33: on the wg0 iface instead, the conntrack mark +# is properly restored. But binding on the wg0 iface is undesirable because that +# means we wouldn't be able to access the port over LAN. +# +# (Copium): It's not thaaat bad that we don't have PBR. In fact, on the bright +# side, game servers that STUN itself will end up STUN'ing to the VPS's IP +# without PBR! + +# # Don't add routing tables. We manage them manually so we only send conntrack- +# # marked packets from the wireguard interface back into the interface. +# wireguard_table: "off" + +# # This is a table we'll create ourselves. We can't use the `wireguard_table` +# # variable because that's used by a role, and setting it to anything other than +# # `"off"` will create entries in the table that we don't want. +# wireguard_manual_table: 200 +# # The mark we'll set on inbound packets from the wireguard interface. Doesn't +# # really matter what this is as long as it doesn't collide with another mark, +# # which it shouldn't. +# conntrack_mark: 101 + +# # Following a similar pattern to [this answer](https://serverfault.com/a/1107872). +# wireguard_postup: + # # Mark incoming packets on wireguard interface with a mark and use conntrack + # # to create an association + # - iptables -t mangle -A PREROUTING -i wg0 -m conntrack --ctstate NEW -j CONNMARK --set-mark {{ conntrack_mark }} + # # Outbound packets try to use the conntrack association to restore the mark + # - iptables -t mangle -A OUTPUT -j CONNMARK --restore-mark + # # Create a new table to route to the wireguard server + # - ip route add table {{ wireguard_manual_table }} default via 10.4.4.1 dev wg0 proto static onlink + # # Outbound packets with the mark + # - ip rule add fwmark {{ conntrack_mark }} table {{ wireguard_manual_table }} + # # Also, allow routing on the wireguard network to the server. This makes + # # debugging easier and it would be strange if this weren't added. + # - ip rule add to 10.4.4.0/24 table {{ wireguard_manual_table }} + # # - ip route add 10.4.4.0/24 via 10.4.4.1 dev wg0 proto static onlink + +# wireguard_predown: + # # - ip route del 10.4.4.0/24 via 10.4.4.1 dev wg0 proto static onlink + # - ip rule del to 10.4.4.0/24 table {{ wireguard_manual_table }} + # - ip rule del fwmark {{ conntrack_mark }} table {{ wireguard_manual_table }} + # - ip route del table {{ wireguard_manual_table }} default via 10.4.4.1 dev wg0 proto static onlink + # - iptables -t mangle -D OUTPUT -j CONNMARK --restore-mark + # - iptables -t mangle -D PREROUTING -i wg0 -m conntrack --ctstate NEW -j CONNMARK --set-mark {{ conntrack_mark }} diff --git a/ansible/inventory/full/group_vars/vpn_server/main.yml b/ansible/inventory/full/group_vars/vpn_server/main.yml index 9db9846..ae27b0f 100644 --- a/ansible/inventory/full/group_vars/vpn_server/main.yml +++ b/ansible/inventory/full/group_vars/vpn_server/main.yml @@ -11,7 +11,6 @@ wireguard_preup: wireguard_postup: | {% filter from_yaml %} {%- for value in (nat_map | dict2items | map(attribute='value')) %} - # incoming packets to vps_ip, dst port 10,000-40,000 are DNAT'd to vpn_ip # with a matching port - iptables -t nat -A PREROUTING -p tcp -d {{ value.vps_ip }} --dport 10000:40000 -j DNAT --to-destination {{ value.vpn_ip }} @@ -32,7 +31,7 @@ wireguard_predown: | {% filter from_yaml %} {%- for value in (nat_map | dict2items | map(attribute='value') | reverse) %} - iptables -t nat -D POSTROUTING -p tcp -s {{ value.vpn_ip }} -j SNAT --to-source {{ value.vps_ip }} - - iptables -t nat -D PREROUTING -p tcp -i enX0 -d {{ value.vps_ip }} --dport 10000:40000 -j DNAT --to-destination {{ value.vpn_ip }} + - iptables -t nat -D PREROUTING -p tcp -d {{ value.vps_ip }} --dport 10000:40000 -j DNAT --to-destination {{ value.vpn_ip }} - iptables -t nat -D PREROUTING -p udp -d {{ value.vps_ip }} --dport 10000:40000 -j DNAT --to-destination {{ value.vpn_ip }} - iptables -t nat -D POSTROUTING -p udp -s {{ value.vpn_ip }} -j SNAT --to-source {{ value.vps_ip }} {%- endfor %} diff --git a/dns/zones/home.mnke.org.zone b/dns/zones/home.mnke.org.zone index 6136c90..c98caef 100644 --- a/dns/zones/home.mnke.org.zone +++ b/dns/zones/home.mnke.org.zone @@ -1,7 +1,9 @@ $ORIGIN home.mnke.org. -@ 900 IN SOA dns-server. hostadmin 23 900 300 604800 900 +@ 900 IN SOA dns-server. hostadmin 27 900 300 604800 900 @ 3600 IN NS dns-server. db 600 IN CNAME truenas +jellyfin 600 IN CNAME truenas-gpu +media 600 IN CNAME jellyfin nas 600 IN CNAME truenas truenas 600 IN A 10.0.0.160 truenas-gpu 600 IN A 10.0.0.250 diff --git a/k8s/apps/cloudflared/cloudflared-mnke.yaml b/k8s/apps/cloudflared/cloudflared-mnke.yaml new file mode 100644 index 0000000..1219411 --- /dev/null +++ b/k8s/apps/cloudflared/cloudflared-mnke.yaml @@ -0,0 +1,128 @@ +--- +# Yeah, I should really turn these into kustomization overlays, huh? +# Sounds like problem for future me. +# TODO: Turn these into kustomization overlays +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cloudflared-mnke + namespace: default +spec: + selector: + matchLabels: + app: cloudflared-mnke + replicas: 2 + template: + metadata: + labels: + app: cloudflared-mnke + + spec: + containers: + - name: cloudflared + image: cloudflare/cloudflared:2025.2.0 + resources: + requests: + memory: "32Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + args: + - tunnel + # Points cloudflared to the config file, which configures what + # cloudflared will actually do. This file is created by a ConfigMap + # below. + - --config + - /etc/cloudflared/config/config.yaml + - run + livenessProbe: + httpGet: + # Cloudflared has a /ready endpoint which returns 200 if and only if + # it has an active connection to the edge. + path: /ready + port: 2000 + failureThreshold: 1 + initialDelaySeconds: 10 + periodSeconds: 10 + volumeMounts: + - name: config + mountPath: /etc/cloudflared/config + readOnly: true + # Each tunnel has an associated "credentials file" which authorizes machines + # to run the tunnel. cloudflared will read this file from its local filesystem, + # and it'll be stored in a k8s secret. + - name: creds + mountPath: /etc/cloudflared/creds + readOnly: true + volumes: + - name: creds + secret: + secretName: cf-tunnel-creds-mnke + # Create a config.yaml file from the ConfigMap below. + - name: config + configMap: + name: cloudflared-mnke + namespace: default + items: + - key: config.yaml + path: config.yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cloudflared-mnke + namespace: default +data: + config.yaml: | + # Name of the tunnel you want to run + tunnel: f0c5cab9-409e-4c9d-b89d-91a1fd6b507a + credentials-file: /etc/cloudflared/creds/credentials.json + # Serves the metrics server under /metrics and the readiness server under /ready + metrics: 0.0.0.0:2000 + # Autoupdates applied in a k8s pod will be lost when the pod is removed or restarted, so + # autoupdate doesn't make sense in Kubernetes. However, outside of Kubernetes, we strongly + # recommend using autoupdate. + no-autoupdate: true + ingress: + # - hostname: blog.mnke.org + # service: https://traefik.traefik.svc.cluster.local + # originRequest: + # Any hosts connecting to the traefik.traefik.svc.cluster.local service + # will always present the default traefik cert to cloudflared, which + # will be rejected. While we could just set the service to + # https://blog.mnke.org and resolve the domain internally to the traefik + # LB so the TLS certificate that's presented is the right one, this + # means we add an extra dependency on the DNS server at this step and + # also the domain resolves to the LB IP, possibly making the pod route + # out of the cluster rather than internally. + # + # We could also split DNS here using a sidecar and have a CNAME from + # blog.mnke.org to the traefik service domain (or otherwise make them + # resolve the same), but managing LAN DNS records, public DNS records, + # and then DNS records here sounds like too much trouble to maintain. + # + # Maybe it can be easier to manage? [This thread](https://www.reddit.com/r/kubernetes/comments/z2vogg/cloudflare_and_ingressnginx/) + # hints at it, but I haven't looked into it too deeply yet. + # + # Disabling the TLS verification right now seems fine enough + # noTLSVerify: true + # http2Origin: true + # httpHostHeader: blog.mnke.org + # Nah, screw it. It's at most like one extra hop away for a much cleaner + # configuration. I'm leaving those comments in so I don't forget about this + # though. + - hostname: blog.mnke.org + service: https://blog.mnke.org + - hostname: media.mnke.org + service: https://media.mnke.org + # The old tonydu.me domains will be routed like this though. This + # is because I no longer want to support internal DNS entries for tonydu.me + - hostname: blog.tonydu.me + service: https://traefik.traefik.svc.cluster.local + originRequest: + noTLSVerify: true + http2Origin: true + httpHostHeader: blog.tonydu.me + # This rule matches any traffic which didn't match a previous rule, and responds with HTTP 404. + - service: http_status:404 diff --git a/k8s/apps/cloudflared/cloudflared-tonydu.yaml b/k8s/apps/cloudflared/cloudflared-tonydu.yaml new file mode 100644 index 0000000..76e1456 --- /dev/null +++ b/k8s/apps/cloudflared/cloudflared-tonydu.yaml @@ -0,0 +1,82 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: default + name: cloudflared-tonydu +spec: + selector: + matchLabels: + app: cloudflared-tonydu + replicas: 2 + template: + metadata: + labels: + app: cloudflared-tonydu + + spec: + containers: + - name: cloudflared + image: cloudflare/cloudflared:2025.2.0 + resources: + requests: + memory: "32Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + args: + - tunnel + - --config + - /etc/cloudflared/config/config.yaml + - run + livenessProbe: + httpGet: + path: /ready + port: 2000 + failureThreshold: 1 + initialDelaySeconds: 10 + periodSeconds: 10 + volumeMounts: + - name: config + mountPath: /etc/cloudflared/config + readOnly: true + - name: creds + mountPath: /etc/cloudflared/creds + readOnly: true + volumes: + - name: creds + secret: + secretName: cf-tunnel-creds-tonydu + - name: config + configMap: + name: cloudflared-tonydu + namespace: default + items: + - key: config.yaml + path: config.yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cloudflared-tonydu + namespace: default +data: + config.yaml: | + # Name of the tunnel you want to run + tunnel: f84d35f1-604c-4982-87fa-deb2253703ea + credentials-file: /etc/cloudflared/creds/credentials.json + # Serves the metrics server under /metrics and the readiness server under /ready + metrics: 0.0.0.0:2000 + no-autoupdate: true + ingress: + # The old tonydu.me domains will be routed like this because I no longer + # want to support internal DNS records for tonydu.me + - hostname: blog.tonydu.me + service: https://traefik.traefik.svc.cluster.local + originRequest: + noTLSVerify: true + http2Origin: true + httpHostHeader: blog.tonydu.me + # This rule matches any traffic which didn't match a previous rule, and responds with HTTP 404. + - service: http_status:404 diff --git a/k8s/apps/cloudflared/kustomization.yaml b/k8s/apps/cloudflared/kustomization.yaml new file mode 100644 index 0000000..c7c6876 --- /dev/null +++ b/k8s/apps/cloudflared/kustomization.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - secrets.yaml + - cloudflared-mnke.yaml + - cloudflared-tonydu.yaml diff --git a/k8s/apps/cloudflared/secrets.yaml b/k8s/apps/cloudflared/secrets.yaml new file mode 100644 index 0000000..fd40939 --- /dev/null +++ b/k8s/apps/cloudflared/secrets.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: cf-tunnel-creds-mnke + namespace: default +spec: + secretStoreRef: + kind: ClusterSecretStore + name: infisical + + target: + name: cf-tunnel-creds-mnke + + data: + - secretKey: credentials.json + remoteRef: + key: cf-tunnel-creds-mnke + +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: cf-tunnel-creds-tonydu + namespace: default +spec: + secretStoreRef: + kind: ClusterSecretStore + name: infisical + + target: + name: cf-tunnel-creds-tonydu + + data: + - secretKey: credentials.json + remoteRef: + key: cf-tunnel-creds-tonydu diff --git a/k8s/apps/ghost/release.yaml b/k8s/apps/ghost/release.yaml index d32d581..147fcc8 100644 --- a/k8s/apps/ghost/release.yaml +++ b/k8s/apps/ghost/release.yaml @@ -26,6 +26,8 @@ spec: global: defaultStorageClass: longhorn + resourcesPreset: small + ingress: enabled: true annotations: diff --git a/k8s/apps/kustomization.yaml b/k8s/apps/kustomization.yaml index a2f3413..15426ba 100644 --- a/k8s/apps/kustomization.yaml +++ b/k8s/apps/kustomization.yaml @@ -7,4 +7,5 @@ resources: - ghost - authentik - ingressroutes + - cloudflared # - twingate diff --git a/k8s/infrastructure/controllers/traefik/middlewares/crowdsec-bouncer.yaml b/k8s/infrastructure/controllers/traefik/middlewares/crowdsec-bouncer.yaml index c562c47..405a3d7 100644 --- a/k8s/infrastructure/controllers/traefik/middlewares/crowdsec-bouncer.yaml +++ b/k8s/infrastructure/controllers/traefik/middlewares/crowdsec-bouncer.yaml @@ -7,7 +7,7 @@ spec: plugin: crowdsec-bouncer-traefik-plugin: enabled: true - logLevel: DEBUG + logLevel: INFO crowdsecMode: live crowdsecLapiScheme: http # crowdsecLapiTLSInsecureVerify: true diff --git a/tf/modules/embassy/main.tf b/tf/modules/embassy/main.tf index 02c92b7..9905d83 100644 --- a/tf/modules/embassy/main.tf +++ b/tf/modules/embassy/main.tf @@ -36,17 +36,17 @@ resource "aws_security_group" "embassy" { cidr_blocks = ["0.0.0.0/0"] } - # everything else + # We'll selectively open ports, but we'll reserve these for general purpose ingress { - from_port = 10000 - to_port = 40000 + from_port = 20000 + to_port = 20100 protocol = "tcp" cidr_blocks = ["0.0.0.0/0"] } ingress { - from_port = 10000 - to_port = 40000 + from_port = 20000 + to_port = 20100 protocol = "udp" cidr_blocks = ["0.0.0.0/0"] }