fix: deploy grafana with a separate helm chart instead of kube-prometheus-stack

cluster/apps/monitoring/grafana/helm-release.yaml

@@ -0,0 +1,87 @@
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: grafana
+  namespace: monitoring
+spec:
+  interval: 5m
+  chart:
+    spec:
+      chart: grafana
+      version: "6.60.1"
+      sourceRef:
+        kind: HelmRepository
+        name: grafana-charts
+        namespace: flux-system
+
+  values:
+    ingress:
+      enabled: true
+      annotations:
+        cert-manager.io/cluster-issuer: letsencrypt-production
+        traefik.ingress.kubernetes.io/router.entrypoints: websecure
+      hosts:
+        - &grafana-host grafana.${SECRET_NEW_DOMAIN}
+      path: "/"
+      tls:
+        - hosts:
+            - *grafana-host
+          secretName: wildcard-main-tls
+
+    grafana.ini:
+      server:
+        root_url: https://grafana.${SECRET_NEW_DOMAIN}/
+
+      auth:
+        disable_login_form: true
+        oauth_auto_login: true
+
+      auth.generic_oauth:
+        enabled: true
+        allow_sign_up: true # creates new users after authentik login
+        auto_login: true
+        name: Authentik
+        client_id: $__file{/etc/secrets/auth_generic_oauth/client_id}
+        client_secret: $__file{/etc/secrets/auth_generic_oauth/client_secret}
+        scopes: openid profile email offline_access
+        auth_url: https://auth.${SECRET_NEW_DOMAIN}/application/o/authorize/
+        token_url: https://auth.${SECRET_NEW_DOMAIN}/application/o/token/
+        api_url: https://auth.${SECRET_NEW_DOMAIN}/application/o/userinfo/
+        use_pkce: true
+        use_refresh_token: true
+
+    # Provide oauth creds
+    extraSecretMounts:
+    - name: grafana-secrets-mount
+      secretName: grafana-secrets
+      defaultMode: 0440
+      mountPath: /etc/secrets/auth_generic_oauth
+      readOnly: true
+
+    # Add Victoria Metrics as the default datasource
+    datasources:
+      victoria.yaml:
+        apiVersion: 1
+        datasources:
+        - name: Victoria
+          type: prometheus
+          editable: false
+          url: http://victoria-metrics-server.monitoring.svc:8428
+          isDefault: true
+
+#    datasources:
+#    - name: Victoria
+#      uid: victoria-metrics-server
+#      type: prometheus
+#      jsonData:
+#        tlsSkipVerify: "true"
+#      editable: false"
+#      url: http://victoria-metrics-server.monitoring.svc:8428
+#      version: "1"
+#      isDefault: "true"
+
+    sidecar:
+      dashboards:
+        enabled: true
+        label: grafana_dashboard
+        labelValue: "1"

cluster/apps/monitoring/grafana/helm-repository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: grafana-charts
+  namespace: flux-system
+spec:
+  interval: 1m
+  url: https://grafana.github.io/helm-charts

cluster/apps/monitoring/grafana/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ./secret.sops.yaml
+- ./helm-repository.yaml
+- ./helm-release.yaml

cluster/apps/monitoring/grafana/secret.sops.yaml

@@ -0,0 +1,59 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: grafana-secrets
+    namespace: monitoring
+stringData:
+    client_id: ENC[AES256_GCM,data:9nDR+Mx3xCDEe/3n2pdfWWihTLPj4/TqoaqbM7+uBzqAlu2oPeEF8A==,iv:xh+GOONaVbExUdJCna0HpmUvBvV1TcV5BizUaVy7Jfs=,tag:N/jkW7ZCiiei6M7Bbv5j4g==,type:str]
+    client_secret: ENC[AES256_GCM,data:v6DMkzI+wD/7lQh8fR+GZl0l1cGKxQ3jy++H1U92U9JGA9uHYf7c1pgCZAb26eaUVou90oeTsh5pc98tbUnwsnq8WWYskKsfEy8W32dZSIm3VRs7uKAyOcRT6Ink2UXlH6wvMkTobqUEt/Quwlr5YIDmeGviEVQUuBk2JwYQE/E=,iv:AiYcuKy6MXsYGSa/S0Fdu+8Zxof4vKZAhxYB/pVFigM=,tag:eya2Xj5Q2YLHOASZN/2gyQ==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2023-10-01T23:25:40Z"
+    mac: ENC[AES256_GCM,data:LqK/sMdQpT+EJQyJIAJe9GpSefMAdDO57RDOoikLAvhA/CZxtlIFfKQ9D0v+P1T6nogCybhgc2CqvtXF2pLSLdjej9V17wmBZGn/kA3vO3GKmUoJSmKUHSf/CbUcJNE92f+6HUOTI1yWrdZNqLJdDk9FrVUhOiLKGocx2V6PCEk=,iv:YP3z8US1CfGeZliCDfQAIEiGCDz5TxRvYNxLMOdTSB4=,tag:TdNFISNgjNSVHy1EDKOwtQ==,type:str]
+    pgp:
+        - created_at: "2023-10-01T23:25:40Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMAzKleRwoSoixAQ//bkp2YBFG0TAICxbXBaPhOD5CBdhCUqGcBTVH+qWZnKY7
+            ftsTk4Y1g/2aUC+0fL7GbKgGAU3SNXiq9wYaNziihP0o7pQpDIipOXblEyB/VaR0
+            duBGSVL1dPj7ZTLDyGDnfXlyEFgpv5u/Ss9q4S6pmnEVNThtaBO6GCOs16TfYz9b
+            cW/y0eGWSm7rLzL51nklgg2pddOzCqdiylK4R5L2ngoke9M22TkzsojENM92/c8w
+            trvsvG92fJ+0XX/rVyI4LavJV6wT9vaLX+jJs8ysTHSpel6H7wr/7UXHtPRH7SU+
+            AUgLjxBujI7MQjtBwR57R4KzhD7k5+6coFo9E/oJHafx1RrgEPBoRFB2V8btuNZs
+            8H0lxPdOFNmC6bfQ8E7/Cwv+TbQMiW4T7M0W665pwXBlMBfG4xoeVvGVB409SpwP
+            Lr1aNYWuk8NS0riwU8jUerX+YCXVK5kGBEhvZXlotwoJsgo/CdA/wPaDRVxBuH9m
+            qnBKr7Er0dbQjEsPk36/fMOVRa8LPjcU2550zsUwpk9/7IIWo+zjV3Urav2fSmGX
+            njYhy90NlIT+kkC6eLU8tIqCekYTjSOoyqRJbeivLszQsj8lT3xF4hW5JyTZ+g4Y
+            8V8Uao5cQNl+JVBzfZpzzrZDXyaDdiUt7TFtRg0h7aSMx17V993F2KNsQa5fDKnS
+            XgGEAMUZ69PL1HWVMxhwfmjfZkY8tDyvBT6Wtm6zUWybOlJsurZiFserW4o4LHNQ
+            1zPrZYdFMHkq7fIJn5PJrCH7Yx/DahRyb3h/VxiZlgpl594nf73ekr4e2bhi+98=
+            =CcmB
+            -----END PGP MESSAGE-----
+          fp: 2CC2B3631D5C3393901335DB68F95C5D753EE1E5
+        - created_at: "2023-10-01T23:25:40Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMAy5t8IMoPu4VAQ/+NNma0ciWbimhjGxVRyPpnmnvaV6cowm+5k+29LOp4EyG
+            BZVm0pS1uIuQBls5BAdqi0cNbWyF6rQPE1r26Q1/jx4XS1M+LAadoufC4OcIDpOK
+            p8/UqVdAeMrS0Xn+kUvEdbKVAwGRBVRnd067+3QAY6jYcCy6iWLRCdwI3kOu1B69
+            ZcfUJ+BJcXomqrSKa9H+iOUqcaxZUvGDpa+MBGfgY8Gdxtz0idW282P3hMrPZ+pP
+            Y63/Eik6Uf4DmyQQbI6gsnERL2jtCDSoAfyYkqpAg9R2EWLjf50G7I9r2YHXb+c1
+            FrNSCmGFuYPwW6WMXLayi1hdw18ySYliYA92dMhq84bCx78K/9RPByTVuh7YVu3J
+            QEThL8nPAEqVVyW0qht5NA6NTcN6XbiDd/CvUXPgMSJ/xE9QKInvQ249g8lBD+Pb
+            kcBieDr/jzUd6lDmy+CgNKcNQetyczkC9XeFKJJEiDcNn5al7iuYGI2LqNzxJmEY
+            NOBornabsW6E7psEFK0wxyo6ePYqsBSPtwYXvyvF/jO3DJvHCXrdG86BvnqR19im
+            T2UiUB984MSYSic3y4+8zNPkOFlzwmew2Q5cyRsY/UuKqmTKeMcYvKl9CKCMS44p
+            enrWB3vIEfCmQluwIn30kyX7F7hGgHl2QkhwIcsFHYQavwwXqhHsPVULNqbb7WjS
+            XgGKHjVfDDhoMzZDYEqiJX5sAEWy1qyP7tJinnBfK1RN8pEF5nKAQr7hJLDil5O5
+            rUzQzJIVeVzB7WIGR6jMY68tHkonV/D6YwgqdXsnAjwY2hKz4JOrFgGx5CDtQ0E=
+            =WHrr
+            -----END PGP MESSAGE-----
+          fp: 687802D4DFD8AA82EA55666CF7DADAC782D7663D
+    encrypted_regex: ^(data|stringData)$
+    version: 3.8.0

cluster/apps/monitoring/kube-prometheus-stack/helm-release.yaml

@@ -38,35 +38,7 @@ spec:
           name: alertmanager-config
                 
     grafana:
-      ingress:
+      enabled: false
-        enabled: true
-        annotations:
-          cert-manager.io/cluster-issuer: letsencrypt-production
-          traefik.ingress.kubernetes.io/router.entrypoints: websecure
-        hosts:
-          - &grafana-host grafana.${SECRET_NEW_DOMAIN}
-        path: "/"
-        tls:
-          - hosts:
-              - *grafana-host
-            secretName: wildcard-main-tls
-
-      sidecar:
-        datasources:
-          defaultDatasourceEnabled: false
-          isDefaultDatasource: false
-
-      # Add Victoria Metrics as the default datasource
-      additionalDataSources:
-      - name: Victoria
-        uid: victoria-metrics-server
-        type: prometheus
-        jsonData:
-          tlsSkipVerify: true
-        editable: false
-        url: http://victoria-metrics-server.monitoring.svc:8428
-        version: 1
-        isDefault: true
 
     prometheus:
       ingress:

cluster/apps/monitoring/kustomization.yaml

@@ -4,6 +4,7 @@ resources:
 - ./namespace.yaml
 - ./network_policy.yaml
 - ./kube-prometheus-stack
+- ./grafana
 - ./zfs-exporter
 - ./alertmanager-gotify-bridge
 - ./victoria-metrics