feat: switch from traefik to nginx-ingress

kubernetes/common/apps/cert-manager/certs/files/wildcard-cert.yaml

@@ -2,7 +2,7 @@ apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: wildcard-main-cert
-  namespace: traefik
+  namespace: nginx
 spec:
   secretName: wildcard-main-tls
 

kubernetes/thin/apps/helm-repositories.yaml

@@ -15,3 +15,12 @@ metadata:
 spec:
   interval: 1m
   url: https://bjw-s.github.io/helm-charts
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: ingress-nginx
+  namespace: flux-system
+spec:
+  interval: 1m
+  url: https://kubernetes.github.io/ingress-nginx

kubernetes/thin/apps/kustomization.yaml

@@ -4,7 +4,7 @@ resources:
 - ./helm-repositories.yaml
 # networking
 - ./cilium
-- ./traefik
+- ./nginx
 - ../../common/apps/cert-manager
 # storage
 - ./snapshot-system/ks.yaml

kubernetes/thin/apps/nginx/external/helm-release.yaml

@@ -0,0 +1,102 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: nginx-external
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: ingress-nginx
+      version: 4.11.2
+      sourceRef:
+        kind: HelmRepository
+        name: ingress-nginx
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  values:
+    fullnameOverride: nginx-external
+    controller:
+      replicaCount: 2
+      service:
+        annotations:
+          io.cilium/lb-ipam-ips: 192.168.2.50
+        labels:
+          bgp/service-type: public
+      
+      ingressClassResource:
+        name: external
+        default: false
+        controllerValue: k8s.io/external
+      
+      admissionWebhooks:
+        objectSelector:
+          matchExpressions:
+            - key: ingress-class
+              operator: In
+              values: ["external"]
+
+      allowSnippetAnnotations: true
+      config:
+        # taken from https://github.com/superseriousbusiness/gotosocial/blob/main/internal/web/robots.go
+        block-user-agents: "GPTBot,~*GPTBot*,ChatGPT-User,~*ChatGPT-User*,Google-Extended,~*Google-Extended*,CCBot,~*CCBot*,Omgilibot,~*Omgilibot*,FacebookBot,~*FacebookBot*"
+        client-body-buffer-size: 100M
+        client-body-timeout: 120
+        client-header-timeout: 120
+        enable-brotli: "true"
+        enable-ocsp: "true"
+        enable-real-ip: "true"
+        force-ssl-redirect: "true"
+        hide-headers: Server,X-Powered-By
+        hsts-max-age: 31449600
+        keep-alive-requests: 10000
+        keep-alive: 120
+        log-format-escape-json: "true"
+        log-format-upstream: >
+          {"time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x_forwarded_for": "$proxy_add_x_forwarded_for",
+          "request_id": "$req_id", "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time,
+          "status": $status, "vhost": "$host", "request_proto": "$server_protocol", "path": "$uri", "request_query": "$args",
+          "request_length": $request_length, "duration": $request_time, "method": "$request_method", "http_referrer": "$http_referer",
+          "http_user_agent": "$http_user_agent"}
+        proxy-body-size: 0
+        proxy-buffer-size: 16k
+        ssl-protocols: TLSv1.3 TLSv1.2
+        use-forwarded-headers: "true"
+      
+      metrics:
+        enabled: false # TODO
+        serviceMonitor:
+          enabled: true
+          namespaceSelector:
+            any: true
+      
+      extraArgs:
+        default-ssl-certificate: nginx/wildcard-main-tls
+
+      terminationGracePeriodSeconds: 120
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: ingress-nginx
+              app.kubernetes.io/instance: nginx-external
+              app.kubernetes.io/component: controller
+      
+      resources:
+        requests:
+          cpu: 100m
+        limits:
+          memory: 500Mi
+    
+    defaultBackend:
+      enabled: false

kubernetes/thin/apps/nginx/external/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ./helm-release.yaml

kubernetes/thin/apps/nginx/internal/helm-release.yaml

@@ -0,0 +1,102 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: nginx-internal
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: ingress-nginx
+      version: 4.11.2
+      sourceRef:
+        kind: HelmRepository
+        name: ingress-nginx
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  values:
+    fullnameOverride: nginx-internal
+    controller:
+      replicaCount: 2
+      service:
+        annotations:
+          io.cilium/lb-ipam-ips: 192.168.2.51
+        labels:
+          bgp/service-type: public
+      
+      ingressClassResource:
+        name: internal
+        default: true
+        controllerValue: k8s.io/internal
+      
+      admissionWebhooks:
+        objectSelector:
+          matchExpressions:
+            - key: ingress-class
+              operator: In
+              values: ["internal"]
+
+      allowSnippetAnnotations: true
+      config:
+        # taken from https://github.com/superseriousbusiness/gotosocial/blob/main/internal/web/robots.go
+        block-user-agents: "GPTBot,~*GPTBot*,ChatGPT-User,~*ChatGPT-User*,Google-Extended,~*Google-Extended*,CCBot,~*CCBot*,Omgilibot,~*Omgilibot*,FacebookBot,~*FacebookBot*"
+        client-body-buffer-size: 100M
+        client-body-timeout: 120
+        client-header-timeout: 120
+        enable-brotli: "true"
+        enable-ocsp: "true"
+        enable-real-ip: "true"
+        force-ssl-redirect: "true"
+        hide-headers: Server,X-Powered-By
+        hsts-max-age: 31449600
+        keep-alive-requests: 10000
+        keep-alive: 120
+        log-format-escape-json: "true"
+        log-format-upstream: >
+          {"time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x_forwarded_for": "$proxy_add_x_forwarded_for",
+          "request_id": "$req_id", "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time,
+          "status": $status, "vhost": "$host", "request_proto": "$server_protocol", "path": "$uri", "request_query": "$args",
+          "request_length": $request_length, "duration": $request_time, "method": "$request_method", "http_referrer": "$http_referer",
+          "http_user_agent": "$http_user_agent"}
+        proxy-body-size: 0
+        proxy-buffer-size: 16k
+        ssl-protocols: TLSv1.3 TLSv1.2
+        use-forwarded-headers: "true"
+      
+      metrics:
+        enabled: false # TODO
+        serviceMonitor:
+          enabled: true
+          namespaceSelector:
+            any: true
+      
+      extraArgs:
+        default-ssl-certificate: nginx/wildcard-main-tls
+
+      terminationGracePeriodSeconds: 120
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: ingress-nginx
+              app.kubernetes.io/instance: nginx-internal
+              app.kubernetes.io/component: controller
+      
+      resources:
+        requests:
+          cpu: 100m
+        limits:
+          memory: 500Mi
+    
+    defaultBackend:
+      enabled: false

kubernetes/thin/apps/nginx/internal/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ./helm-release.yaml

kubernetes/thin/apps/nginx/ks.yaml

@@ -0,0 +1,53 @@
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: nginx-external
+  namespace: flux-system
+spec:
+  targetNamespace: nginx
+  timeout: 5m
+  interval: 10m
+  path: ./kubernetes/thin/apps/nginx/external
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-cluster
+  decryption:
+    provider: sops
+    secretRef:
+      name: sops-gpg
+  postBuild:
+    substitute: {}
+    substituteFrom:
+      - kind: ConfigMap
+        name: cluster-settings
+      - kind: Secret
+        name: cluster-secrets
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: nginx-internal
+  namespace: flux-system
+spec:
+  targetNamespace: nginx
+  timeout: 5m
+  interval: 10m
+  path: ./kubernetes/thin/apps/nginx/internal
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-cluster
+  decryption:
+    provider: sops
+    secretRef:
+      name: sops-gpg
+  postBuild:
+    substitute: {}
+    substituteFrom:
+      - kind: ConfigMap
+        name: cluster-settings
+      - kind: Secret
+        name: cluster-secrets