chore: Add authentik ServiceMonitor and dashboard, fix proxmox dashboard, monitor hpwork

cluster/apps/authentik/helm-release.yaml

@@ -46,6 +46,16 @@ spec:
         host: "redis-master.database"
 #        password: "${SECRET_DATABASE_REDIS_PASS}"
     
+    prometheus:
+      serviceMonitor:
+        create: true
+        labels: 
+          release: kube-prometheus-stack
+      rules:
+        create: true
+        labels: 
+          release: kube-prometheus-stack
+
     env:
       AUTHENTIK_HOST: &host "auth.${SECRET_NEW_DOMAIN}"
       AUTHENTIK_HOST_BROWSER: *host

cluster/apps/monitoring/proxmoxve-exporter/dashboard.yaml

@@ -36,7 +36,7 @@ data:
           "type": "grafana",
           "id": "grafana",
           "name": "Grafana",
-          "version": "9.4.7"
+          "version": "9.5.3"
         },
         {
           "type": "datasource",
@@ -66,7 +66,7 @@ data:
       "annotations": {
         "list": [
           {
-            "$$hashKey": "object:215",
+            "$hashKey": "object:215",
             "builtIn": 1,
             "datasource": {
               "type": "datasource",
@@ -385,6 +385,7 @@ data:
           },
           "id": 19,
           "options": {
+            "cellHeight": "sm",
             "footer": {
               "countRows": false,
               "fields": "",
@@ -402,7 +403,7 @@ data:
               }
             ]
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "9.5.3",
           "targets": [
             {
               "datasource": {
@@ -746,17 +747,19 @@ data:
             "showThresholdMarkers": true,
             "text": {}
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "9.5.3",
           "targets": [
             {
               "datasource": {
                 "type": "prometheus",
                 "uid": "${DS_VICTORIA}"
               },
+              "editorMode": "code",
               "expr": "pve_cpu_usage_ratio{instance=\"$instance\"} / pve_cpu_usage_limit and on(id) pve_node_info",
-              "format": "time_series",
+              "format": "table",
               "intervalFactor": 1,
               "legendFormat": "",
+              "range": true,
               "refId": "A"
             }
           ],
@@ -825,7 +828,7 @@ data:
             },
             "textMode": "value_and_name"
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "9.5.3",
           "targets": [
             {
               "datasource": {
@@ -1039,17 +1042,19 @@ data:
             "showThresholdMarkers": true,
             "text": {}
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "9.5.3",
           "targets": [
             {
               "datasource": {
                 "type": "prometheus",
                 "uid": "${DS_VICTORIA}"
               },
+              "editorMode": "code",
               "expr": "pve_memory_usage_bytes{instance=\"$instance\"} / pve_memory_size_bytes and on(id) pve_node_info",
-              "format": "time_series",
+              "format": "table",
               "intervalFactor": 1,
               "legendFormat": "",
+              "range": true,
               "refId": "A"
             }
           ],
@@ -1146,7 +1151,7 @@ data:
             },
             "textMode": "value_and_name"
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "9.5.3",
           "targets": [
             {
               "datasource": {
@@ -1483,7 +1488,7 @@ data:
             "showThresholdMarkers": true,
             "text": {}
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "9.5.3",
           "targets": [
             {
               "datasource": {
@@ -1557,9 +1562,10 @@ data:
             "showUnfilled": false,
             "text": {
               "titleSize": 20
-            }
+            },
+            "valueMode": "color"
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "9.5.3",
           "targets": [
             {
               "datasource": {
@@ -2010,6 +2016,6 @@ data:
       "timezone": "",
       "title": "Proxmox VE",
       "uid": "Dp7Cd57Zza",
-      "version": 2,
+      "version": 1,
       "weekStart": ""
     }

cluster/apps/monitoring/proxmoxve-exporter/proxmoxve-config.sops.yaml

@@ -4,56 +4,54 @@ metadata:
     name: proxmoxve-configmap
     namespace: monitoring
 data:
-    pve.yml: ENC[AES256_GCM,data:tDanahhT5SLgUEtX/7T3kX891J6IscZuBDVeRsvdLn8zsCdPc97p5OwaFfShEGYftl8TJyd38k5GR+A97oKXYW8vxGHsjRQ86EykIVV++Is=,iv:/GHJg25DX4tIJXNXriar+x4pCaKS/0i0itTdleYBTCk=,tag:n+k91YXWSIiaKr0DPGW24Q==,type:str]
+    pve.yml: ENC[AES256_GCM,data:FHTW/zzQYBvL37om2hLTycAjGPHIwQTezh8kjDVm3z7F9BphetH9huchX0l5pbS3ebwHW7goeigem/6suvMjz+RQmCr/pSTftDCjCyn7ajw=,iv:9qXW9Fgc5yig21WX7azinaZiTKKYMqjJ/S1GBIeKY7E=,tag:hEd3C8ozcX1kfRM+jxCCoA==,type:str]
 sops:
     kms: []
     gcp_kms: []
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2023-05-11T15:59:29Z"
-    mac: ENC[AES256_GCM,data:8sVPIIQOdp30rvkuHErg1ionxjQnOWVCcHAK+e7u4T0tYfB95oYV+Pe2AVFyEuPrAPnAeAVGrIPGjJWvkmoAiP0e5uYXi3GKRdEtoB+10ai2nzOtlQM3cRpdd+KqkP0O33YxRix5y8ZxOn9nen4gtdkEmy/8Et9a1mxc8BWpv78=,iv:Ja3u4ASHnDtsvy6YKYJgykQfaR7FwWr1T2EmRBnBpyw=,tag:sn4gm4I8eBxSe+hGGJ9enw==,type:str]
+    lastmodified: "2023-08-06T00:43:19Z"
+    mac: ENC[AES256_GCM,data:zMBKVc4Xp4IMfZilziPaJncnBzpNyqS/SfHa+QqJiITPYiIwH82AjULJBYz213s+GQJ370tgez2FVZrDcKcsoDDplj/jKPaDX5kVPBmJi5Q0N2OnlKrDW0y/E5eXIFbUxLss8ecDvBcWl7FF5cXTLV6ktLwxQBm81jB3DRNaZvQ=,iv:SiXOBvzTFxkGB1WPabJe7jzULPfKnLOQXUgfzjTDGP4=,tag:O5jKhU7xWySwjho0U+vseA==,type:str]
     pgp:
-        - created_at: "2023-06-19T18:35:56Z"
+        - created_at: "2023-08-06T00:43:19Z"
           enc: |
             -----BEGIN PGP MESSAGE-----
 
-            hQIMAzKleRwoSoixAQ//dYOk/xUMs4kRObeW5Cmw8X1h/yF1ZiCRsFXKgqN7452n
-            LbFmJJvP5jDXgg5qNPmNT0LL2a4cP9l7fJVIVqfYM87LctKEl6yc7h1oS9JVRUt8
-            trc2aqsG+6OCeXgomwH9lB8HRxDeUEXWDWK3hS1noKEvKTE8hXtMrJJA7xfosc3L
-            caNLfVMvOiVSnxwFMniDKqZW+TupQDVTCCb5RxChx6o76TycdEfUnk8CkPTp7+1m
-            UDaG1NYUVWot444QA0pcaNAny5GJX0e5NFCN5/MoaCjUMgIvIyjaJbDzUw1405b1
-            NTpmjbRSPexmtrxPsJdXsfCEdkFmp+7YD2ifa/yCctXNmyx7wCJ8KUhMkldc6EXU
-            qN0a+ppRPUA6e/r1MgdHjJq5U3TxN9Vw4UNwcfhUGf/VR2d7VWbheLVoYOW9nwyR
-            6ru5NvGw0aUv0lVOpqnrAglY20p8RONqINNY9LMuIruaUoLD420IQaUf9cccTlhg
-            wArXFPVmwoOsRpJI6JCYqWNNJaugFjx8/4TRbFf7c8QqcPNRYwq5tJQQpTMqr0Lw
-            SzyP4ATpCKwndsLmjJA4tiGPACHbU2dHOaymqNZEoBK8eKzhi+p09GkDERP+lNje
-            3YeS0ZqmJgwax2iCh3pKFthkmHYMw17InkWHIJHM14pvEKpwUNiOVrlc52tyVKHU
-            aAEJAhCVKtEw90VF3gR+T6XptJ6DvVbWy2BlS+WZAJpHjTEJrkLKjk6flhTnPtpY
-            ZzZ97f9kvR/VQWnseF0dYmoa4qHwDt+wT9XM8fJxI4wsGboSws6Ig1Sn/ZbF3fyd
-            wT8NtFiHQ6da
-            =+VIK
+            hQIMAzKleRwoSoixAQ//cca+ABl3tzwI9pn43SjfRbOPyjRN+qbJkv07+NpiHibY
+            78qFJ9arkc6VZt/tX3YkzSIcHAjHME13XJ8kFjRHsLQoFt2mT0qUSlRroGuAOF65
+            YN9iNfbCAe/ahWUCKVk8XRVD/sNng5Z8DrvAOy040BsjGmL0PJhRGBkBdtJeM+lT
+            U9MRouPzMX8CAxCQolrryi6j+qXLlMZbpktc/zpJ6Jb582N1lA3qaCUmiX+c9Xn1
+            BTqogVrU1IIw3yV62oKUWbXSNKU409pA3NRWxPdJJgfGcH1y7LZ2xafSSFg+Vd5O
+            hItlQ37sMGn9zmGOo9eJ/EKlr7tRU1qUaT6qJVQ36YngIM3VhPfgCNibBZ/TJNEF
+            63r7XcOuLy1SVXCZJQhBiMESoYWWgtDuAwDxhwhKqvepplKr3K55Otw6TsA17uIG
+            nVYGet21ereq5jR4xwZVPcHv3uveuYN1azjb9HTpPAIxVK7sN85zzyneCmQAE+Hp
+            GJ6DMDe+jig6DJKyHVRpXcFt+sRXxhJ0hT6giBarA8jvmZCxyKowgjSQeV6RawEu
+            8salqVM0OUAProfnHKxgkteLMfGl0QVAYkO3WPepvi75N6Fb/HWa579b2HS8t9JF
+            9mIF5RNT/mRcx2tRHu0Kuz+LB8vIqsnzR/L2Orhox/9fNaKQqfQV2+1TAFob6dTS
+            XgG6lknz4TwZ6V8BIuYuacFkG3q7qHLIM0iJ86BUev21skYl45z+KDPhof2gmuF2
+            O8uC/TT36XVu3nPGLciTbPL3nB29adQi03nCHqWG2fs07AHSYP8ixArrHYRsSo0=
+            =2ius
             -----END PGP MESSAGE-----
           fp: 2CC2B3631D5C3393901335DB68F95C5D753EE1E5
-        - created_at: "2023-06-19T18:35:56Z"
+        - created_at: "2023-08-06T00:43:19Z"
           enc: |
             -----BEGIN PGP MESSAGE-----
 
-            hQIMAy5t8IMoPu4VARAAhgGzGph4IUKSVL3edGiTlcDGr642cKnnk2NNazBVdBe0
-            tjPYnwRY/P6dfTHII8KceP4W5T4cVe1DIEI2GAyyotUhQMYIDE6SGmcg03Lgd42h
-            0tJZnmppKZnceIrMXyYHtR7KDM6N0Xz6mdlm5vAULEH27/Njm1OiOsJaouEiClix
-            cZadDyCWaXKDbdTFVUOXZDPqB61ZVE+qtGzwgUIz09iViMYStQuDUhZVwQqqTUvV
-            rMxmL5Mys6BD7JnsRtpqCJgz/dmVGToQhgYKKPK0ATVRksT5oPgzgwt7Y9OpfY5B
-            APMdwEtWYWTpMivFcFADKXjzGd7uk+r6ObRMkCq4iU6ShoZ5YxxpY8BuCp5e8RdC
-            woMJPUg+X4vy4m7ZSPcFOFkFgoGDYDKm32f474zi0aSughqpwSTBK7BSYvHhIK35
-            nxIY/7F9Crw7b+6w3+lv33MjfD6ut2+7VkKfehv9Ms5ZBzUB1vFp1Jw5JMjbwmUl
-            LJpqxHx+f05h08xpSidOdtx9ntDUxhaXn3v7xx7dlzlmhFx0dZQV4cAQTqzf42FZ
-            hax/+GvCXcmi9w0Afw8oyi38hR2G0r8NKQm8OzMkHIERXdWNCr52p4EoaVzqAWi+
-            YqcVJgLy499+ebbfdf3qFMk9LCD0ewigibTuvtq9GNVhkUeTBI4FuXJM+9Dtg3jU
-            aAEJAhAvP/jo7B/0kuhnT/FS/s3/0Mte+QbyI7nZ97zqAQqvHJw4hUvSkvZRzkTZ
-            55siLJFYGHXhn9ujGxeVIwGb9bC2IYtL6nMO44dTUfKiRDlC+8YEZIeTjjuz3VRD
-            4AfQC36lIR2i
-            =nxmK
+            hQIMAy5t8IMoPu4VARAAgUoO8TG30YAKZmv9ZHecOAj2E7uNVoTUnlxDmjLrcrpu
+            SvuYbxyABHKpPmzUv6uMStOgeFTn46DSL7u0mM5tAzzqzZZEmbYJhXp7GdpibQYb
+            tJyhiYMAmrfSZ7v3hgNnJL5LeyxF9LihJlvbcGqhdzTLz7S8qq/CDKHbiU9f5xyD
+            zbXw/DT+7b5VZaSmCJ77lWOvzJEGA5wxBEARjmNiuwKVbYtZCywLJJmLYtmvjejP
+            gdxhjzmaFvTWLk4M4oIu/I15Yuv+uHQF27lZbmo8tMMI75fQiPO8TmK4p+cdRzN9
+            nptyOFXVXrrXwYNGDmGAuxI21WFN+z6yfQS5ZfFXYE7HxYL2Dqqx2iTSPZCzbMsx
+            ysyey/7xqURdCuEpYTVPebP7Oe8kiKQWjvBvDY4jYTUBUv3pGSevtkv4ZAJYgn7K
+            ClNjKm5aZH5wjra3vYhrxthAcjRzUA8P0ZoNwZmtMWA0VpFfN3tWQWaf8NdzDjjb
+            1UAD7lLyQPvcdZEtWCQlxa4dTsc+QeaIvc25Q38d/TPN2Ya5yoo13HKJ7wvSHhFW
+            BA0H9j5a00Fu2GH5bpLYvDWQpqt5ws4d35S121rwDmEqTFhoTZ1rlXFnX/4uss3A
+            /cJLECJp91ioLFerRdVDGnJ+dz6rTz2Q7cWAj8SvtlZqXp8SYXH7XYySJaPVBHjS
+            XgGulu19p94uvJQG4twWoPNbA0t7jm+VeALZdV5tAtzWTHPjKE4/hvS7ZwaLBda2
+            6DLCO3wfsPaWAPm+pOIp4BFpbzlP8UJMwMRm+W6fHcf0VfnltocsqYfTF/SY7nU=
+            =KHkb
             -----END PGP MESSAGE-----
           fp: 687802D4DFD8AA82EA55666CF7DADAC782D7663D
     encrypted_regex: ^(data|stringData)$

cluster/apps/monitoring/proxmoxve-exporter/service-monitor.yaml

@@ -19,3 +19,12 @@ spec:
       - "192.168.87.29"
       module:
       - default
+  - port: http
+    interval: 2m
+    scrapeTimeout: 1m
+    path: /pve
+    params:
+      target:
+      - "192.168.87.20"
+      module:
+      - default

docs/setup.md

@@ -2,12 +2,16 @@
 This document goes over the process of installing the GitOps files into a cluster
 
 
-Install [calico](https://docs.tigera.io/calico/latest/getting-started/kubernetes/k3s/quickstart#install-calico)
+Install [cilium](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/#install-the-cilium-cli)
 
 Now install the FluxCD stuff:\
 https://github.com/larivierec/home-cluster#installation
 
-## Uninstalling and removing all k3s data in NixOS
+1. Bootstrap FluxCD, this will likely fail
+2. After it fails, create the sops secret in the `flux-system` namespace
+3. Now trigger a reconcilation, or resume the fluxcd bootstrap
+
+# Uninstalling and removing all k3s data in NixOS
 ```shell
 sudo systemctl stop k3s
 sudo rm -rf /etc/rancher/k3s
@@ -19,7 +23,6 @@ sudo chown $USER ~/.kube/config
 sudo chmod 600 ~/.kube/config
 ```
 
-
 ## Finishing Service Setup
 
 ### SSO (Authentik)

docs/todo.md

@@ -17,8 +17,8 @@ TODO:
   - [ ] uptime-kuma
   - [x] gotify
   - [ ] Services to monitor
-    - [ ] Authentik
-    - [ ] r720xd proxmox
+    - [x] Authentik
+    - [x] r720xd proxmox
     - [x] *arr
     - [ ] nextcloud
     - [ ] onlyoffice