feat: Revamp infrastructure

ansible/lvm.yml

@@ -1,14 +0,0 @@
----
-- name: Create LVM and mount it
-  hosts: lvm
-  remote_user: ubuntu
-  become: true
-  vars:
-    pv_disks: "{{ lvm.pv_disks }}"
-    vg_name: "{{ lvm.vg_name }}"
-    lv_name: "{{ lvm.lv_name }}"
-    lv_size: "{{ lvm.lv_size }}"
-    fs_type: "{{ lvm.fs_type }}"
-    mount_path: "{{ lvm.mount_path }}"
-  roles:
-    - lvm

ansible/site.yml

@@ -18,6 +18,20 @@
         - multipathd.service
         - multipathd.socket
 
+- name: Create LVM and mount it
+  hosts: lvm
+  become: true
+  vars:
+    pv_disks: "{{ lvm.pv_disks }}"
+    vg_name: "{{ lvm.vg_name }}"
+    lv_name: "{{ lvm.lv_name }}"
+    lv_size: "{{ lvm.lv_size }}"
+    fs_type: "{{ lvm.fs_type }}"
+    # Consider mounting directly at the /var/lib/longhorn in the future
+    mount_path: "{{ lvm.mount_path }}"
+  roles:
+    - lvm
+
 - name: Prepare Proxmox cluster
   hosts: proxmox
   gather_facts: true
@@ -45,7 +59,6 @@
       become: true
       when: custom_registries
 
-
 - name: Setup k3s servers
   hosts: master
   environment: "{{ proxy_env | default({}) }}"

docker/compose/media/docker-compose.yml

@@ -11,6 +11,7 @@ volumes:
   prowlarr_config:
   radarr_config:
   sonarr_config:
+  wizarr_config:
 
 services:
   transmission-openvpn:
@@ -206,3 +207,24 @@ services:
           cpus: '0.1'
           memory: 64M
 
+  wizarr:
+    image: tonyd33/wizarr
+    environment:
+      # This is intentionally not WIZARR_HOST. I'm still in the process of
+      # migrating everything into mnke.org domain.
+      - 'APP_URL=https://wizarr.tonydu.me'
+      - DISABLE_BUILTIN_AUTH=false
+      - TZ=America/Vancouver
+    volumes:
+      - wizarr_config:/data/database
+    networks:
+      - media
+      - traefik
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.wizarr.rule=Host(`${WIZARR_HOST:-wizarr.jumper.mnke.org}`)"
+      - "traefik.http.routers.wizarr.entrypoints=websecure"
+      - "traefik.http.routers.wizarr.tls=true"
+      - "traefik.http.services.wizarr.loadbalancer.server.port=5690"
+      - "traefik.docker.network=traefik"

k8s/apps/ghost/release.yaml

@@ -29,9 +29,9 @@ spec:
       annotations:
         cert-manager.io/cluster-issuer: le-cf-issuer
         kubernetes.io/ingress.class: traefik
-      ingressClassName: traefik
+      # ingressClassName: traefik
       hostname: blog.mnke.org
-      tls: true
+      # tls: true
 
     allowEmptyPassword: false
     ghostEmail: tonydu121@hotmail.com

k8s/apps/kube-prometheus-stack/release.yaml

@@ -1,38 +0,0 @@
----
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: kube-prometheus-stack
-  namespace: kube-prometheus-stack
-spec:
-  interval: 10m
-  chart:
-    spec:
-      chart: kube-prometheus-stack
-      sourceRef:
-        kind: HelmRepository
-        name: prometheus-community
-        namespace: kube-prometheus-stack
-      interval: 10m
-  values:
-    grafana:
-      adminPassword: admin
-      defaultDashboardsTimezone: browser
-      ingress:
-        enabled: true
-        annotations:
-          cert-manager.io/cluster-issuer: le-cf-issuer
-          kubernetes.io/ingress.class: traefik
-        hosts:
-          - gf.dolo.mnke.org
-    prometheus:
-      prometheusSpec:
-        storageSpec:
-          volumeClaimTemplate:
-            spec:
-              storageClassName: longhorn
-              accessModes: ["ReadWriteOnce"]
-              resources:
-                requests:
-                  storage: 4Gi
-

k8s/apps/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - common
-  - kube-prometheus-stack
+  # - kube-prometheus-stack
   - uptime-kuma
-  - rancher
+  # - rancher
   - ghost

k8s/apps/rancher/repository.yaml

@@ -1,9 +0,0 @@
----
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: rancher-stable
-  namespace: cattle-system
-spec:
-  interval: 1m
-  url: https://releases.rancher.com/server-charts/stable

k8s/clusters/dolo/apps.yaml

@@ -1,18 +1,18 @@
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: apps
-  namespace: flux-system
-spec:
-  interval: 10m0s
-  retryInterval: 30s
-  dependsOn:
-    - name: infrastructure
-  sourceRef:
-    kind: GitRepository
-    name: flux-system
-  path: ./k8s/apps
-  prune: true
-  wait: true
-  timeout: 5m0s
+# ---
+# apiVersion: kustomize.toolkit.fluxcd.io/v1
+# kind: Kustomization
+# metadata:
+  # name: apps
+  # namespace: flux-system
+# spec:
+  # interval: 10m0s
+  # retryInterval: 30s
+  # dependsOn:
+    # - name: infrastructure
+  # sourceRef:
+    # kind: GitRepository
+    # name: flux-system
+  # path: ./k8s/apps
+  # prune: true
+  # wait: true
+  # timeout: 5m0s

k8s/clusters/dolo/infrastructure.yaml

@@ -2,7 +2,7 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: external-secrets
+  name: infrastructure-01
   namespace: flux-system
 spec:
   interval: 1h
@@ -11,7 +11,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: flux-system
-  path: ./k8s/infrastructure/external-secrets
+  path: ./k8s/infrastructure/01
   wait: true
   prune: true
 
@@ -19,7 +19,7 @@ spec:
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: secrets
+  name: infrastructure-02
   namespace: flux-system
 spec:
   interval: 1h
@@ -28,67 +28,11 @@ spec:
   sourceRef:
     kind: GitRepository
     name: flux-system
-  path: ./k8s/infrastructure/secrets
+  path: ./k8s/infrastructure/02
   wait: true
   prune: true
   dependsOn:
-    - name: external-secrets
-
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: cert-manager
-  namespace: flux-system
-spec:
-  interval: 1h
-  retryInterval: 30s
-  timeout: 5m
-  sourceRef:
-    kind: GitRepository
-    name: flux-system
-  path: ./k8s/infrastructure/cert-manager
-  wait: true
-  prune: true
-  dependsOn:
-    - name: secrets
-
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: traefik
-  namespace: flux-system
-spec:
-  interval: 1h
-  retryInterval: 30s
-  timeout: 5m
-  sourceRef:
-    kind: GitRepository
-    name: flux-system
-  path: ./k8s/infrastructure/traefik
-  wait: true
-  prune: true
-  dependsOn:
-    - name: cert-manager
-    - name: secrets
-
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: storage
-  namespace: flux-system
-spec:
-  interval: 1h
-  retryInterval: 30s
-  timeout: 5m
-  sourceRef:
-    kind: GitRepository
-    name: flux-system
-  path: ./k8s/infrastructure/storage
-  wait: true
-  prune: true
+    - name: infrastructure-01
 
 ---
 # What I want is one single unit that the rest of my applications relying on
@@ -115,8 +59,5 @@ spec:
   wait: true
   prune: false
   dependsOn:
-    - name: external-secrets
-    - name: secrets
-    - name: cert-manager
-    - name: traefik
-    - name: storage
+    - name: infrastructure-01
+    - name: infrastructure-02

k8s/infrastructure/01/cert-manager/namespace.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cert-manager
+  labels:
+    external-secrets.io/secrets.cloudflare: require

k8s/infrastructure/01/cert-manager/release.yaml

@@ -1,36 +1,20 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: cert-manager
-  labels:
-    external-secrets.io/secrets.cloudflare: require
-
----
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: jetstack
-  namespace: cert-manager
-spec:
-  interval: 1m
-  url: https://charts.jetstack.io
-
 ---
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
   name: cert-manager
-  namespace: cert-manager
+  namespace: flux-system
 spec:
   interval: 10m
+  releaseName: cert-manager
+  targetNamespace: cert-manager
   chart:
     spec:
       chart: cert-manager
       sourceRef:
         kind: HelmRepository
         name: jetstack
-        namespace: cert-manager
+        namespace: flux-system
       interval: 10m
   values:
     crds:

k8s/infrastructure/01/cert-manager/repository.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: jetstack
+  namespace: flux-system
+spec:
+  interval: 1m
+  url: https://charts.jetstack.io

k8s/infrastructure/01/common/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespaces

k8s/infrastructure/01/common/namespaces/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - monitor.yaml

k8s/infrastructure/01/common/namespaces/monitor.yaml

@@ -2,5 +2,4 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: cattle-system
-
+  name: monitor

k8s/infrastructure/01/external-secrets/namespace.yaml

@@ -2,4 +2,4 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: kube-prometheus-stack
+  name: external-secrets

k8s/infrastructure/01/external-secrets/release.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: external-secrets
+  namespace: flux-system
+spec:
+  interval: 10m
+  releaseName: external-secrets
+  targetNamespace: external-secrets
+  chart:
+    spec:
+      chart: external-secrets
+      sourceRef:
+        kind: HelmRepository
+        name: external-secrets
+        namespace: flux-system
+      interval: 10m

k8s/infrastructure/01/external-secrets/repository.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: external-secrets
+  namespace: flux-system
+spec:
+  interval: 1m
+  url: https://charts.external-secrets.io

k8s/infrastructure/01/kube-prometheus-stack/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - repository.yaml
+  - release.yaml

k8s/infrastructure/01/kube-prometheus-stack/release.yaml

@@ -0,0 +1,59 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: kube-prometheus-stack
+  namespace: flux-system
+spec:
+  interval: 10m
+  releaseName: kube-prometheus-stack
+  targetNamespace: monitor
+  chart:
+    spec:
+      chart: kube-prometheus-stack
+      sourceRef:
+        kind: HelmRepository
+        name: prometheus-community
+        namespace: flux-system
+      interval: 10m
+  values:
+    grafana:
+      adminPassword: admin
+      defaultDashboardsTimezone: browser
+      # This kind of sucks, but this is a forward declaration of the issuer and
+      # ingress class. The problem is that we want Traefik and other services
+      # to be able to use Prometheus operators, but they require CRDs installed
+      # within this chart.
+      #
+      # By sequencing Prometheus to be installed first, these labels just won't
+      # be recognized by the ingress and cluster issuer until they're installed
+      # later -- undesirable, but acceptable -- as opposed to flatly failing
+      # from missing CRDs by installing Traefik first.
+      #
+      # Really, the ideal solution is probably to install all CRDs first, but
+      # I'm not sure how to do that in a way that guarantees compatibility
+      # with the CRDs that might be installed in Helm charts later. We can skip
+      # installing CRDs from the Helm chart, but if the CRDs get updated, we
+      # need to manually update the CRDs in our repository.
+      #
+      # Alternatively, we could declare an Ingress/IngressRoute after Traefik
+      # is installed, but it wouldn't solve the root problem around dependent
+      # CRDs
+      ingress:
+        enabled: true
+        annotations:
+          cert-manager.io/cluster-issuer: le-cf-issuer
+          kubernetes.io/ingress.class: traefik
+        hosts:
+          - gf.dolo.mnke.org
+    prometheus:
+      prometheusSpec:
+        storageSpec:
+          volumeClaimTemplate:
+            spec:
+              storageClassName: longhorn
+              accessModes: ["ReadWriteOnce"]
+              resources:
+                requests:
+                  storage: 4Gi
+

k8s/infrastructure/01/kube-prometheus-stack/repository.yaml

@@ -3,7 +3,7 @@ apiVersion: source.toolkit.fluxcd.io/v1
 kind: HelmRepository
 metadata:
   name: prometheus-community
-  namespace: kube-prometheus-stack
+  namespace: flux-system
 spec:
   interval: 10m
   url: https://prometheus-community.github.io/helm-charts

k8s/infrastructure/01/kustomization.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - common
+  - external-secrets
+  - cert-manager
+  - longhorn
+  - nfs-subdir-external-provisioner
+  - kube-prometheus-stack
+  - loki
+  - promtail

k8s/infrastructure/01/loki/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - secret.yaml
+  - repository.yaml
+  - release.yaml

k8s/infrastructure/01/loki/release.yaml

@@ -0,0 +1,98 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: loki
+  namespace: flux-system
+spec:
+  interval: 10m
+  releaseName: loki
+  targetNamespace: monitor
+  chart:
+    spec:
+      chart: loki
+      sourceRef:
+        kind: HelmRepository
+        name: grafana
+        namespace: flux-system
+      interval: 10m
+  valuesFrom:
+    - kind: Secret
+      name: loki-creds
+      valuesKey: minio-password
+      targetPath: minio.rootPassword
+  values:
+    # https://grafana.com/docs/loki/latest/setup/install/helm/install-monolithic/
+    loki:
+      auth_enabled: false
+      commonConfig:
+        replication_factor: 1
+      schemaConfig:
+        configs:
+          - from: "2024-04-01"
+            store: tsdb
+            object_store: s3
+            schema: v13
+            index:
+              prefix: loki_index_
+              period: 24h
+      pattern_ingester:
+          enabled: true
+      # compactor:
+        # retention_enabled: true
+        # retention_delete_delay: 2h
+      limits_config:
+        retention_period: 744h
+        allow_structured_metadata: true
+        volume_enabled: true
+      ruler:
+        enable_api: true
+
+    minio:
+      enabled: true
+      persistence:
+        size: 8Gi
+      rootUser: root
+      # rootPassword: ''
+
+    deploymentMode: SingleBinary
+
+    singleBinary:
+      replicas: 1
+
+    # Zero out replica counts of other deployment modes
+    backend:
+      replicas: 0
+    read:
+      replicas: 0
+    write:
+      replicas: 0
+
+    # Turn this for debugging
+    lokiCanary:
+      enabled: false
+    # If the canary is turned off, this has to be turned off too
+    test:
+      enabled: false
+
+    ingester:
+      replicas: 0
+    querier:
+      replicas: 0
+    queryFrontend:
+      replicas: 0
+    queryScheduler:
+      replicas: 0
+    distributor:
+      replicas: 0
+    compactor:
+      replicas: 0
+    indexGateway:
+      replicas: 0
+    bloomCompactor:
+      replicas: 0
+    bloomGateway:
+      replicas: 0
+
+    chunksCache:
+      allocatedMemory: 512

k8s/infrastructure/01/loki/repository.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana
+  namespace: flux-system
+spec:
+  interval: 1m
+  url: https://grafana.github.io/helm-charts

k8s/infrastructure/01/loki/secret.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: loki-creds
+  namespace: flux-system
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: infisical
+
+  target:
+    name: loki-creds
+
+  data:
+    - secretKey: minio-password
+      remoteRef:
+        key: loki-minio-password

k8s/infrastructure/01/longhorn/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - repository.yaml
+  - release.yaml

k8s/infrastructure/01/longhorn/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: longhorn-system

k8s/infrastructure/01/longhorn/release.yaml

@@ -2,27 +2,25 @@
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
-  name: rancher
-  namespace: cattle-system
+  name: longhorn
+  namespace: flux-system
 spec:
   interval: 10m
+  releaseName: longhorn
+  targetNamespace: longhorn-system
   chart:
     spec:
-      chart: rancher
+      chart: longhorn
       sourceRef:
         kind: HelmRepository
-        name: rancher-stable
-        namespace: cattle-system
+        name: longhorn
+        namespace: flux-system
       interval: 10m
   values:
-    bootstrapPassword: 'admin'
-    hostname: rancher.dolo.mnke.org
+    # This is a forward declaration!
     ingress:
       enabled: true
-      extraAnnotations:
-        kubernetes.io/ingress.class: traefik
+      annotations:
         cert-manager.io/cluster-issuer: le-cf-issuer
-      ingressClassName: traefik
-      tls:
-        source: secret
-        secretName: rancher-tls
+        kubernetes.io/ingress.class: traefik
+      host: longhorn.dolo.mnke.org

k8s/infrastructure/01/longhorn/repository.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: longhorn
+  namespace: flux-system
+spec:
+  interval: 1m
+  url: https://charts.longhorn.io

k8s/infrastructure/01/nfs-subdir-external-provisioner/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - repository.yaml
+  - release.yaml

k8s/infrastructure/01/nfs-subdir-external-provisioner/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: nfs-subdir-external-provisioner

k8s/infrastructure/01/nfs-subdir-external-provisioner/release.yaml

@@ -1,34 +1,20 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: nfs-subdir-external-provisioner
-
----
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: nfs-subdir-external-provisioner
-  namespace: nfs-subdir-external-provisioner
-spec:
-  interval: 1m
-  url: https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
-
 ---
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
   name: nfs-subdir-external-provisioner
-  namespace: nfs-subdir-external-provisioner
+  namespace: flux-system
 spec:
   interval: 10m
+  releaseName: nfs-subdir-external-provisioner
+  targetNamespace: nfs-subdir-external-provisioner
   chart:
     spec:
       chart: nfs-subdir-external-provisioner
       sourceRef:
         kind: HelmRepository
         name: nfs-subdir-external-provisioner
-        namespace: nfs-subdir-external-provisioner
+        namespace: flux-system
       interval: 10m
   values:
     nfs:
@@ -38,5 +24,3 @@ spec:
       accessModes: ReadWriteMany
       name: nfs-client
       defaultClass: false
-
-

k8s/infrastructure/01/nfs-subdir-external-provisioner/repository.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: nfs-subdir-external-provisioner
+  namespace: flux-system
+spec:
+  interval: 1m
+  url: https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/

k8s/infrastructure/01/promtail/clusterrole.yaml

@@ -0,0 +1,16 @@
+--- # Clusterrole.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: promtail-clusterrole
+  namespace: monitor
+rules:
+  - apiGroups: [""]
+    resources:
+    - nodes
+    - services
+    - pods
+    verbs:
+    - get
+    - watch
+    - list

k8s/infrastructure/01/promtail/configmap.yaml

@@ -0,0 +1,61 @@
+--- # configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: promtail-config
+  namespace: monitor
+data:
+  promtail.yaml: |
+    server:
+      http_listen_port: 9080
+      grpc_listen_port: 0
+
+    clients:
+    - url: http://loki:3100/loki/api/v1/push
+
+    positions:
+      filename: /tmp/positions.yaml
+    target_config:
+      sync_period: 10s
+    scrape_configs:
+    - job_name: pod-logs
+      kubernetes_sd_configs:
+        - role: pod
+      pipeline_stages:
+        - docker: {}
+      relabel_configs:
+        # Longhorn hits the label limit of 15. The longhorn_io labels don't
+        # seem really relevant for logging purposes
+        - action: labeldrop
+          regex: longhorn_io_*
+        - source_labels:
+            - __meta_kubernetes_pod_node_name
+          target_label: __host__
+        - action: labelmap
+          regex: __meta_kubernetes_pod_label_(.+)
+        - action: replace
+          replacement: $1
+          separator: /
+          source_labels:
+            - __meta_kubernetes_namespace
+            - __meta_kubernetes_pod_name
+          target_label: job
+        - action: replace
+          source_labels:
+            - __meta_kubernetes_namespace
+          target_label: namespace
+        - action: replace
+          source_labels:
+            - __meta_kubernetes_pod_name
+          target_label: pod
+        - action: replace
+          source_labels:
+            - __meta_kubernetes_pod_container_name
+          target_label: container
+        - replacement: /var/log/pods/*$1/*.log
+          separator: /
+          source_labels:
+            - __meta_kubernetes_pod_uid
+            - __meta_kubernetes_pod_container_name
+          target_label: __path__
+

k8s/infrastructure/01/promtail/daemonset.yaml

@@ -0,0 +1,44 @@
+--- # Daemonset.yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: promtail-daemonset
+  namespace: monitor
+spec:
+  selector:
+    matchLabels:
+      name: promtail
+  template:
+    metadata:
+      labels:
+        name: promtail
+    spec:
+      serviceAccount: promtail-serviceaccount
+      containers:
+      - name: promtail-container
+        image: grafana/promtail
+        args:
+        - -config.file=/etc/promtail/promtail.yaml
+        env:
+        - name: 'HOSTNAME' # needed when using kubernetes_sd_configs
+          valueFrom:
+            fieldRef:
+              fieldPath: 'spec.nodeName'
+        volumeMounts:
+        - name: logs
+          mountPath: /var/log
+        - name: promtail-config
+          mountPath: /etc/promtail
+        - mountPath: /var/lib/docker/containers
+          name: varlibdockercontainers
+          readOnly: true
+      volumes:
+      - name: logs
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: promtail-config
+        configMap:
+          name: promtail-config

k8s/infrastructure/01/promtail/kustomization.yaml

@@ -0,0 +1,11 @@
+---
+# Recommended by Grafana to install through raw manifests
+# https://grafana.com/docs/loki/latest/send-data/promtail/installation/#install-as-kubernetes-daemonset-recommended
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - daemonset.yaml
+  - configmap.yaml
+  - clusterrole.yaml
+  - serviceaccount.yaml
+  - rolebinding.yaml

k8s/infrastructure/01/promtail/rolebinding.yaml

@@ -0,0 +1,14 @@
+--- # Rolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: promtail-clusterrolebinding
+  namespace: monitor
+subjects:
+    - kind: ServiceAccount
+      name: promtail-serviceaccount
+      namespace: monitor
+roleRef:
+    kind: ClusterRole
+    name: promtail-clusterrole
+    apiGroup: rbac.authorization.k8s.io

k8s/infrastructure/01/promtail/serviceaccount.yaml

@@ -0,0 +1,6 @@
+--- # ServiceAccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: promtail-serviceaccount
+  namespace: monitor

k8s/infrastructure/02/alerts/discord.yaml

@@ -47,3 +47,4 @@ spec:
       name: '*'
     - kind: Kustomization
       name: '*'
+

k8s/infrastructure/02/alerts/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - discord.yaml

k8s/infrastructure/02/issuers/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - le-cf-issuer.yaml

k8s/infrastructure/02/issuers/le-cf-issuer.yaml

@@ -5,8 +5,8 @@ metadata:
   name: le-cf-issuer
 spec:
   acme:
-    server: https://acme-v02.api.letsencrypt.org/directory
-    # server: https://acme-staging-v02.api.letsencrypt.org/directory
+    # server: https://acme-v02.api.letsencrypt.org/directory
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
     email: tonydu121@hotmail.com
     privateKeySecretRef:
       name: le-cf-issuer-pk

k8s/infrastructure/02/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - secret-stores
+  - issuers
+  - secrets
+  - traefik
+  - alerts

k8s/infrastructure/02/secret-stores/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - infisical.yaml

k8s/infrastructure/02/secrets/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - cloudflare.yaml

k8s/infrastructure/02/traefik/certificates/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - wildcard-mnke-org.yaml

k8s/infrastructure/02/traefik/certificates/wildcard-mnke-org.yaml

@@ -12,4 +12,3 @@ spec:
   issuerRef:
     name: le-cf-issuer
     kind: ClusterIssuer
-

k8s/infrastructure/02/traefik/kustomization.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - repository.yaml
+  - release.yaml
+  - certificates
+

k8s/infrastructure/02/traefik/namespace.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: traefik
+  labels:
+    external-secrets.io/secrets.cloudflare: require

k8s/infrastructure/02/traefik/release.yaml

@@ -1,36 +1,20 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: traefik
-  labels:
-    external-secrets.io/secrets.cloudflare: require
-
----
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: traefik
-  namespace: traefik
-spec:
-  interval: 1m
-  url: https://helm.traefik.io/traefik
-
 ---
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
   name: traefik
-  namespace: traefik
+  namespace: flux-system
 spec:
   interval: 10m
+  releaseName: traefik
+  targetNamespace: traefik
   chart:
     spec:
       chart: traefik
       sourceRef:
         kind: HelmRepository
         name: traefik
-        namespace: traefik
+        namespace: flux-system
       interval: 10m
   values:
     globalArguments:
@@ -39,7 +23,14 @@ spec:
 
     additionalArguments:
       - "--serversTransport.insecureSkipVerify=true"
-      - "--log.level=DEBUG"
+
+    logs:
+      general:
+        level: INFO
+        format: json
+      access:
+        enabled: true
+        format: json
 
     deployment:
       enabled: true
@@ -109,6 +100,7 @@ spec:
         defaultCertificate:
           secretName: wildcard-mnke-org-tls
 
+    # Mostly from https://github.com/traefik/traefik-helm-chart/blob/master/EXAMPLES.md#use-prometheus-operator
     metrics:
       prometheus:
         service:
@@ -116,6 +108,10 @@ spec:
         disableAPICheck: false
         serviceMonitor:
           enabled: true
+          # IMPORTANT:
+          # This must match the kube-prometheus-stack release name
+          additionalLabels:
+            release: kube-prometheus-stack
           metricRelabelings:
             - sourceLabels: [__name__]
               separator: ;
@@ -134,6 +130,10 @@ spec:
           honorLabels: true
         prometheusRule:
           enabled: true
+          # IMPORTANT:
+          # This must match the kube-prometheus-stack release name
+          additionalLabels:
+            release: kube-prometheus-stack
           rules:
             - alert: TraefikDown
               expr: up{job="traefik"} == 0
@@ -144,3 +144,4 @@ spec:
               annotations:
                 summary: "Traefik Down"
                 description: "{{ $labels.pod }} on {{ $labels.nodename }} is down"
+

k8s/infrastructure/02/traefik/repository.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: traefik
+  namespace: flux-system
+spec:
+  interval: 1m
+  url: https://helm.traefik.io/traefik

k8s/infrastructure/dummy/kustomization.yaml

@@ -1,5 +1,3 @@
----
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-resources:
-  - alerts.yaml
+resources: []

k8s/infrastructure/external-secrets/external-secrets.yaml

@@ -1,33 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: external-secrets
-
----
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: external-secrets
-  namespace: external-secrets
-spec:
-  interval: 1m
-  url: https://charts.external-secrets.io
-
----
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: external-secrets
-  namespace: external-secrets
-spec:
-  interval: 10m
-  chart:
-    spec:
-      chart: external-secrets
-      sourceRef:
-        kind: HelmRepository
-        name: external-secrets
-        namespace: external-secrets
-      interval: 10m
-

k8s/infrastructure/storage/longhorn.yaml

@@ -1,33 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: longhorn-system
-
----
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: longhorn
-  namespace: longhorn-system
-spec:
-  interval: 1m
-  url: https://charts.longhorn.io
-
----
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: longhorn
-  namespace: longhorn-system
-spec:
-  interval: 10m
-  chart:
-    spec:
-      chart: longhorn
-      sourceRef:
-        kind: HelmRepository
-        name: longhorn
-        namespace: longhorn-system
-      interval: 10m
-

k8s/one-off/ingress-route.yaml

@@ -0,0 +1,29 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: jellyfin
+  namespace: default
+spec:
+  type: ExternalName
+  externalName: 10.0.0.250
+
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: jellyfin
+  namespace: default
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`jellyfin.mnke.org`)
+      kind: Rule
+      services:
+        - name: jellyfin
+          kind: Service
+          passHostHeader: false
+          port: 8096
+  # tls:
+    # certResolver: le-cf-issuer

k8s/one-off/traefik-dashboard-service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: traefik-metrics-custom
+  namespace: traefik
+  labels:
+    app.kubernetes.io/instance: traefik
+    app.kubernetes.io/name: traefik-metrics-custom
+spec:
+  type: ClusterIP
+  ports:
+    - name: traefik-metrics
+      port: 9100
+      targetPort: metrics
+      protocol: TCP
+  selector:
+    app.kubernetes.io/instance: traefik

k8s/one-off/traefik-service-monitor.yaml

@@ -0,0 +1,20 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: traefik-monitor
+  namespace: traefik
+  labels:
+    app: traefik
+    release: kube-prometheus-stack
+spec:
+  jobLabel: traefik-metrics
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: traefik-traefik
+      app.kubernetes.io/component: metrics
+  namespaceSelector:
+    matchNames:
+      - traefik
+  endpoints:
+    - port: metrics
+      path: /metrics

k8s/pre-infrastructure/dns-config-map.yaml

@@ -1,25 +1,25 @@
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: coredns
-  namespace: kube-system
-  annotations:
-    fluxcd.io/ignore: "true"
-data:
-  Corefile: |
-    .:53 {
-        errors
-        health
-        ready
-        kubernetes cluster.local in-addr.arpa ip6.arpa {
-            pods insecure
-            fallthrough in-addr.arpa ip6.arpa
-        }
-        forward . 10.0.123.123
-        cache 30
-        loop
-        reload
-        loadbalance
-    }
+# ---
+# apiVersion: v1
+# kind: ConfigMap
+# metadata:
+  # name: coredns
+  # namespace: kube-system
+  # annotations:
+    # fluxcd.io/ignore: "true"
+# data:
+  # Corefile: |
+    # .:53 {
+        # errors
+        # health
+        # ready
+        # kubernetes cluster.local in-addr.arpa ip6.arpa {
+            # pods insecure
+            # fallthrough in-addr.arpa ip6.arpa
+        # }
+        # forward . 10.0.123.123
+        # cache 30
+        # loop
+        # reload
+        # loadbalance
+    # }
 

tf/modules/docker-swarm/main.tf

@@ -185,16 +185,10 @@ resource "ansible_host" "swarm_manager" {
   count = var.manager_count
   name   = "${local.managers[count.index].name}.local"
   groups = ["${var.swarm_name}_manager", var.swarm_name]
-  variables = {
-    ipv4_address = proxmox_virtual_environment_vm.swarm_manager[count.index].ipv4_addresses[1][0]
-  }
 }
 
 resource "ansible_host" "swarm_worker" {
   count = var.worker_count
   name   = "${local.workers[count.index].name}.local"
   groups = ["${var.swarm_name}_worker", var.swarm_name]
-  variables = {
-    ipv4_address = proxmox_virtual_environment_vm.swarm_worker[count.index].ipv4_addresses[1][0]
-  }
 }