feat: Add cloudflare tunnel

ansible/inventory/full/group_vars/vpn/main.yml

@@ -3,9 +3,8 @@ ansible_user: ubuntu
 # Directory to store WireGuard configuration on the remote hosts
 wireguard_remote_directory: /etc/wireguard
 
-wireguard_interface_restart: true
+wireguard_interface_restart: false
-# TODO: Enable this when stable
+wireguard_service_enabled: true
-wireguard_service_enabled: false
 wireguard_service_state: started
 
 # We can generate this dynamically, but it really doesn't seem like it's worth

ansible/inventory/full/group_vars/vpn_client/main.yml

@@ -15,3 +15,65 @@ wireguard_preup:
 
 wireguard_postdown:
   - ip route del 10.0.0.0/16 via 10.0.0.1 dev eth0 proto static onlink
+
+# Ok, I could not get the stuff below working properly. What I _wanted_ to do
+# was make it so that _only_ traffic that was sent from the wireguard tunnel
+# gets sent back to the wireguard tunnel, rather than blindly routing based
+# on network. In other words, I wanted policy-based-routing (PBR).
+#
+# While I got TCP to work perfectly, what seems to happen for UDP is:
+# - An application listens on 0.0.0.0:<port>
+# - Inbound packet on wg0 is marked in conntrack and sent to 10.4.4.33:<port>
+#   on the wg0 iface
+# - Outbound packet is generated with a source IP of 10.0.29.40 on the eth0
+#   interface
+# - Restoring the mark then fails because the source/dest IPs don't match and
+#   the PBR never happens.
+#
+# What's puzzling is that this _only_ happens for UDP. The 3-way handshake on
+# TCP seems to work, as well as subsequent conversation.
+#
+# If we bind on 10.4.4.33:<port> on the wg0 iface instead, the conntrack mark
+# is properly restored. But binding on the wg0 iface is undesirable because that
+# means we wouldn't be able to access the port over LAN.
+#
+# (Copium): It's not thaaat bad that we don't have PBR. In fact, on the bright
+# side, game servers that STUN itself will end up STUN'ing to the VPS's IP
+# without PBR!
+
+# # Don't add routing tables. We manage them manually so we only send conntrack-
+# # marked packets from the wireguard interface back into the interface.
+# wireguard_table: "off"
+
+# # This is a table we'll create ourselves. We can't use the `wireguard_table`
+# # variable because that's used by a role, and setting it to anything other than
+# # `"off"` will create entries in the table that we don't want.
+# wireguard_manual_table: 200
+# # The mark we'll set on inbound packets from the wireguard interface. Doesn't
+# # really matter what this is as long as it doesn't collide with another mark,
+# # which it shouldn't.
+# conntrack_mark: 101
+
+# # Following a similar pattern to [this answer](https://serverfault.com/a/1107872).
+# wireguard_postup:
+  # # Mark incoming packets on wireguard interface with a mark and use conntrack
+  # # to create an association
+  # - iptables -t mangle -A PREROUTING -i wg0 -m conntrack --ctstate NEW -j CONNMARK --set-mark {{ conntrack_mark }}
+  # # Outbound packets try to use the conntrack association to restore the mark
+  # - iptables -t mangle -A OUTPUT -j CONNMARK --restore-mark
+  # # Create a new table to route to the wireguard server
+  # - ip route add table {{ wireguard_manual_table }} default via 10.4.4.1 dev wg0 proto static onlink
+  # # Outbound packets with the mark
+  # - ip rule add fwmark {{ conntrack_mark }} table {{ wireguard_manual_table }}
+  # # Also, allow routing on the wireguard network to the server. This makes
+  # # debugging easier and it would be strange if this weren't added.
+  # - ip rule add to 10.4.4.0/24 table {{ wireguard_manual_table }}
+  # # - ip route add 10.4.4.0/24 via 10.4.4.1 dev wg0 proto static onlink
+
+# wireguard_predown:
+  # # - ip route del 10.4.4.0/24 via 10.4.4.1 dev wg0 proto static onlink
+  # - ip rule del to 10.4.4.0/24 table {{ wireguard_manual_table }}
+  # - ip rule del fwmark {{ conntrack_mark }} table {{ wireguard_manual_table }}
+  # - ip route del table {{ wireguard_manual_table }} default via 10.4.4.1 dev wg0 proto static onlink
+  # - iptables -t mangle -D OUTPUT -j CONNMARK --restore-mark
+  # - iptables -t mangle -D PREROUTING -i wg0 -m conntrack --ctstate NEW -j CONNMARK --set-mark {{ conntrack_mark }}

ansible/inventory/full/group_vars/vpn_server/main.yml

@@ -11,7 +11,6 @@ wireguard_preup:
 wireguard_postup: |
   {% filter from_yaml %}
   {%- for value in (nat_map | dict2items | map(attribute='value')) %}
-
   # incoming packets to vps_ip, dst port 10,000-40,000 are DNAT'd to vpn_ip
   # with a matching port
   - iptables -t nat -A PREROUTING -p tcp -d {{ value.vps_ip }} --dport 10000:40000 -j DNAT --to-destination {{ value.vpn_ip }}
@@ -32,7 +31,7 @@ wireguard_predown: |
   {% filter from_yaml %}
   {%- for value in (nat_map | dict2items | map(attribute='value') | reverse) %}
   - iptables -t nat -D POSTROUTING -p tcp -s {{ value.vpn_ip }} -j SNAT --to-source {{ value.vps_ip }}
-  - iptables -t nat -D PREROUTING -p tcp -i enX0 -d {{ value.vps_ip }} --dport 10000:40000 -j DNAT --to-destination {{ value.vpn_ip }}
+  - iptables -t nat -D PREROUTING -p tcp -d {{ value.vps_ip }} --dport 10000:40000 -j DNAT --to-destination {{ value.vpn_ip }}
   - iptables -t nat -D PREROUTING -p udp -d {{ value.vps_ip }} --dport 10000:40000 -j DNAT --to-destination {{ value.vpn_ip }}
   - iptables -t nat -D POSTROUTING -p udp -s {{ value.vpn_ip }} -j SNAT --to-source {{ value.vps_ip }}
   {%- endfor %}

dns/zones/home.mnke.org.zone

@@ -1,7 +1,9 @@
 $ORIGIN home.mnke.org.
-@                     900       IN  SOA           dns-server. hostadmin 23 900 300 604800 900
+@                     900       IN  SOA           dns-server. hostadmin 27 900 300 604800 900
 @                     3600      IN  NS            dns-server.
 db                    600       IN  CNAME         truenas
+jellyfin              600       IN  CNAME         truenas-gpu
+media                 600       IN  CNAME         jellyfin
 nas                   600       IN  CNAME         truenas
 truenas               600       IN  A             10.0.0.160
 truenas-gpu           600       IN  A             10.0.0.250

k8s/apps/cloudflared/cloudflared-mnke.yaml

@@ -0,0 +1,128 @@
+---
+# Yeah, I should really turn these into kustomization overlays, huh?
+# Sounds like problem for future me.
+# TODO: Turn these into kustomization overlays
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cloudflared-mnke
+  namespace: default
+spec:
+  selector:
+    matchLabels:
+      app: cloudflared-mnke
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: cloudflared-mnke
+
+    spec:
+      containers:
+      - name: cloudflared
+        image: cloudflare/cloudflared:2025.2.0
+        resources:
+          requests:
+            memory: "32Mi"
+            cpu: "100m"
+          limits:
+            memory: "512Mi"
+            cpu: "500m"
+        args:
+        - tunnel
+        # Points cloudflared to the config file, which configures what
+        # cloudflared will actually do. This file is created by a ConfigMap
+        # below.
+        - --config
+        - /etc/cloudflared/config/config.yaml
+        - run
+        livenessProbe:
+          httpGet:
+            # Cloudflared has a /ready endpoint which returns 200 if and only if
+            # it has an active connection to the edge.
+            path: /ready
+            port: 2000
+          failureThreshold: 1
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        volumeMounts:
+        - name: config
+          mountPath: /etc/cloudflared/config
+          readOnly: true
+        # Each tunnel has an associated "credentials file" which authorizes machines
+        # to run the tunnel. cloudflared will read this file from its local filesystem,
+        # and it'll be stored in a k8s secret.
+        - name: creds
+          mountPath: /etc/cloudflared/creds
+          readOnly: true
+      volumes:
+      - name: creds
+        secret:
+          secretName: cf-tunnel-creds-mnke
+      # Create a config.yaml file from the ConfigMap below.
+      - name: config
+        configMap:
+          name: cloudflared-mnke
+          namespace: default
+          items:
+          - key: config.yaml
+            path: config.yaml
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cloudflared-mnke
+  namespace: default
+data:
+  config.yaml: |
+    # Name of the tunnel you want to run
+    tunnel: f0c5cab9-409e-4c9d-b89d-91a1fd6b507a
+    credentials-file: /etc/cloudflared/creds/credentials.json
+    # Serves the metrics server under /metrics and the readiness server under /ready
+    metrics: 0.0.0.0:2000
+    # Autoupdates applied in a k8s pod will be lost when the pod is removed or restarted, so
+    # autoupdate doesn't make sense in Kubernetes. However, outside of Kubernetes, we strongly
+    # recommend using autoupdate.
+    no-autoupdate: true
+    ingress:
+    # - hostname: blog.mnke.org
+      # service: https://traefik.traefik.svc.cluster.local
+      # originRequest:
+        # Any hosts connecting to the traefik.traefik.svc.cluster.local service
+        # will always present the default traefik cert to cloudflared, which
+        # will be rejected. While we could just set the service to
+        # https://blog.mnke.org and resolve the domain internally to the traefik
+        # LB so the TLS certificate that's presented is the right one, this
+        # means we add an extra dependency on the DNS server at this step and
+        # also the domain resolves to the LB IP, possibly making the pod route
+        # out of the cluster rather than internally.
+        #
+        # We could also split DNS here using a sidecar and have a CNAME from
+        # blog.mnke.org to the traefik service domain (or otherwise make them
+        # resolve the same), but managing LAN DNS records, public DNS records,
+        # and then DNS records here sounds like too much trouble to maintain.
+        #
+        # Maybe it can be easier to manage? [This thread](https://www.reddit.com/r/kubernetes/comments/z2vogg/cloudflare_and_ingressnginx/)
+        # hints at it, but I haven't looked into it too deeply yet.
+        #
+        # Disabling the TLS verification right now seems fine enough
+        # noTLSVerify: true
+        # http2Origin: true
+        # httpHostHeader: blog.mnke.org
+    # Nah, screw it. It's at most like one extra hop away for a much cleaner
+    # configuration. I'm leaving those comments in so I don't forget about this
+    # though.
+    - hostname: blog.mnke.org
+      service: https://blog.mnke.org
+    - hostname: media.mnke.org
+      service: https://media.mnke.org
+    # The old tonydu.me domains will be routed like this though. This
+    # is because I no longer want to support internal DNS entries for tonydu.me
+    - hostname: blog.tonydu.me
+      service: https://traefik.traefik.svc.cluster.local
+      originRequest:
+        noTLSVerify: true
+        http2Origin: true
+        httpHostHeader: blog.tonydu.me
+    # This rule matches any traffic which didn't match a previous rule, and responds with HTTP 404.
+    - service: http_status:404

k8s/apps/cloudflared/cloudflared-tonydu.yaml

@@ -0,0 +1,82 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: default
+  name: cloudflared-tonydu
+spec:
+  selector:
+    matchLabels:
+      app: cloudflared-tonydu
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: cloudflared-tonydu
+
+    spec:
+      containers:
+      - name: cloudflared
+        image: cloudflare/cloudflared:2025.2.0
+        resources:
+          requests:
+            memory: "32Mi"
+            cpu: "100m"
+          limits:
+            memory: "512Mi"
+            cpu: "500m"
+        args:
+        - tunnel
+        - --config
+        - /etc/cloudflared/config/config.yaml
+        - run
+        livenessProbe:
+          httpGet:
+            path: /ready
+            port: 2000
+          failureThreshold: 1
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        volumeMounts:
+        - name: config
+          mountPath: /etc/cloudflared/config
+          readOnly: true
+        - name: creds
+          mountPath: /etc/cloudflared/creds
+          readOnly: true
+      volumes:
+      - name: creds
+        secret:
+          secretName: cf-tunnel-creds-tonydu
+      - name: config
+        configMap:
+          name: cloudflared-tonydu
+          namespace: default
+          items:
+          - key: config.yaml
+            path: config.yaml
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cloudflared-tonydu
+  namespace: default
+data:
+  config.yaml: |
+    # Name of the tunnel you want to run
+    tunnel: f84d35f1-604c-4982-87fa-deb2253703ea
+    credentials-file: /etc/cloudflared/creds/credentials.json
+    # Serves the metrics server under /metrics and the readiness server under /ready
+    metrics: 0.0.0.0:2000
+    no-autoupdate: true
+    ingress:
+    # The old tonydu.me domains will be routed like this because I no longer
+    # want to support internal DNS records for tonydu.me
+    - hostname: blog.tonydu.me
+      service: https://traefik.traefik.svc.cluster.local
+      originRequest:
+        noTLSVerify: true
+        http2Origin: true
+        httpHostHeader: blog.tonydu.me
+    # This rule matches any traffic which didn't match a previous rule, and responds with HTTP 404.
+    - service: http_status:404

k8s/apps/cloudflared/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - secrets.yaml
+  - cloudflared-mnke.yaml
+  - cloudflared-tonydu.yaml

k8s/apps/cloudflared/secrets.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: cf-tunnel-creds-mnke
+  namespace: default
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: infisical
+
+  target:
+    name: cf-tunnel-creds-mnke
+
+  data:
+    - secretKey: credentials.json
+      remoteRef:
+        key: cf-tunnel-creds-mnke
+
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: cf-tunnel-creds-tonydu
+  namespace: default
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: infisical
+
+  target:
+    name: cf-tunnel-creds-tonydu
+
+  data:
+    - secretKey: credentials.json
+      remoteRef:
+        key: cf-tunnel-creds-tonydu

k8s/apps/ghost/release.yaml

@@ -26,6 +26,8 @@ spec:
     global:
       defaultStorageClass: longhorn
 
+    resourcesPreset: small
+
     ingress:
       enabled: true
       annotations:

k8s/apps/kustomization.yaml

@@ -7,4 +7,5 @@ resources:
   - ghost
   - authentik
   - ingressroutes
+  - cloudflared
   # - twingate

k8s/infrastructure/controllers/traefik/middlewares/crowdsec-bouncer.yaml

@@ -7,7 +7,7 @@ spec:
     plugin:
         crowdsec-bouncer-traefik-plugin:
             enabled: true
-            logLevel: DEBUG
+            logLevel: INFO
             crowdsecMode: live
             crowdsecLapiScheme: http
             # crowdsecLapiTLSInsecureVerify: true

tf/modules/embassy/main.tf

@@ -36,17 +36,17 @@ resource "aws_security_group" "embassy" {
     cidr_blocks = ["0.0.0.0/0"]
   }
 
-  # everything else
+  # We'll selectively open ports, but we'll reserve these for general purpose
   ingress {
-    from_port   = 10000
+    from_port   = 20000
-    to_port     = 40000
+    to_port     = 20100
     protocol    = "tcp"
     cidr_blocks = ["0.0.0.0/0"]
   }
 
   ingress {
-    from_port   = 10000
+    from_port   = 20000
-    to_port     = 40000
+    to_port     = 20100
     protocol    = "udp"
     cidr_blocks = ["0.0.0.0/0"]
   }