fix(woodpecker): fix reconcile failures

cluster/apps/dev/woodpecker/helm-release.yaml

@@ -20,7 +20,10 @@ spec:
       replicaCount: 4
 
       extraSecretNamesForEnvFrom:
-      - woodpecker-secret
+      - woodpecker
+
+      # https://github.com/woodpecker-ci/helm/issues/154
+      secrets: {}
 
       env:
         WOODPECKER_BACKEND: kubernetes
@@ -41,7 +44,10 @@ spec:
         WOODPECKER_BACKEND_K8S_NAMESPACE: dev
 
       extraSecretNamesForEnvFrom:
-      - woodpecker-secret
+      - woodpecker
+
+      # https://github.com/woodpecker-ci/helm/issues/154
+      secrets: {}
 
       persistentVolume:
         enabled: true

cluster/apps/dev/woodpecker/secret.sops.yaml

@@ -1,64 +1,77 @@
 apiVersion: v1
 kind: Secret
 metadata:
-    name: woodpecker-secret
+    name: woodpecker
     namespace: dev
 stringData:
-    WOODPECKER_ADMIN: ENC[AES256_GCM,data:TXqVhenIfPfwv1WZs3w=,iv:SxyUU9W+V/NBVqyjuMdWgrMpekp8apg7zV5XQUv8/Uk=,tag:TE4vesnF4ptKHumu9Rt69w==,type:str]
+    WOODPECKER_ADMIN: ENC[AES256_GCM,data:xWrKbuFDdfI5AzEOqck=,iv:YgZku7F+agkzv7omfBdusHzK7hhwhsI3t6nuDxoz/i4=,tag:z/w/iBphuCwAEaKhQU22/Q==,type:str]
-    WOODPECKER_OPEN: ENC[AES256_GCM,data:GOD8OA==,iv:39gfTyWXmsUoRg8GuMP73bM16cN0CkY7AhJ7mKX35LI=,tag:j/L+C/O9WnWgexL29HznjA==,type:str]
+    WOODPECKER_OPEN: ENC[AES256_GCM,data:qcSw8w==,iv:weQQxyIXn13EiVQmmm1wRBh9n5JQsPQAtGUVlaeGp3M=,tag:h20PUnyRXFMRw+duA70r9A==,type:str]
-    WOODPECKER_AGENT_SECRET: ENC[AES256_GCM,data:8jjlVX8rqX4YVUNj7pbwPCo+hYMGEwSl/KQ68yRi4R2aj8p5yddxQ8UH8hvRXEEhU3DX5lFW5mXfN+8EROsQuw==,iv:6y6ygCjT+stF3ObFZzNS8Qi2grJJCUSPzus/ebf0qcw=,tag:vCF4jrkT/h4PoOt5IGy6dw==,type:str]
+    WOODPECKER_AGENT_SECRET: ENC[AES256_GCM,data:pEKvYUdSIMHevkYWua4MEmK5wZcvR83heTkPLaEoUmR/TMz1uPSUGiHMFIlvI4SPjjK2ydM2zBK2YA/Qo50Xjg==,iv:B05Sc5/f/+pTeK07ch3aNi/B3JmKB6lett8EA1l0qpA=,tag:6rGKCPI5+D72sl40iZQKPA==,type:str]
-    WOODPECKER_GITEA: ENC[AES256_GCM,data:dLlVKw==,iv:wASS5ZoS0sBGuVB6pjygbPwp+33rDwV2x1intu2uguA=,tag:HumjgcBce0rXT377OhA3bA==,type:str]
+    WOODPECKER_GITEA: ENC[AES256_GCM,data:U+Xbdg==,iv:J/HN/XNJJhI3MLqSmlOS/TQ+vK55BuMY733WegKGV5g=,tag:WshRFIEQXBqY/wPHu+EvIg==,type:str]
-    WOODPECKER_GITEA_URL: ENC[AES256_GCM,data:cxw/whpjJ+FtymS6lT+M6Ayiz8T1ZZ6Bsw==,iv:6C3oTJ0FOXSKV5Sh5YHI5mFMH3azqBgCARWyKijnDdE=,tag:yt17NnyDlgyfrPJ6VLHjaQ==,type:str]
+    WOODPECKER_GITEA_URL: ENC[AES256_GCM,data:wzf1qN/3bkBg/hP+b9Mz0AgaPrd2NmMpAA==,iv:MVxeEf8bPdP3LAwf5KUXAsIo0eKckQ9q1PkFBcNR3oc=,tag:F2YGLkMrDw34DlG/K0QX9Q==,type:str]
-    WOODPECKER_GITEA_CLIENT: ENC[AES256_GCM,data:OlwxTSfjtmk5MuHZ4qo+o3OkDfUOspBgS99q26doMrqC5Nj9,iv:mTH7jclMu47GoTlJstwTWXh+6ZnFwUSy3JBn1Y0dusU=,tag:KgHThNxIj8wRQdimtlv7EA==,type:str]
+    WOODPECKER_GITEA_CLIENT: ENC[AES256_GCM,data:mbIyNWL3Cqm0Cj7Zuu47r2HWe38bwMrl0domJC0jlG4Lp+7b,iv:hbRRhqSDfU1lb43VXb2hh+Gvdc5kZfl2BFVzgb8wjaM=,tag:RQl84DALRWy6WdioNyFEAg==,type:str]
-    WOODPECKER_GITEA_SECRET: ENC[AES256_GCM,data:Six91NP9wP3HFFIHP4o2Q8k+JLEDbrhRKC520giheJbcFlpwWwN3FBwAApKDCIuV7/UrjKPs++E=,iv:yv8b5+PQ/eiLOBNV+OJ+WlCVIrZz+oPAcsNu8wfFU94=,tag:joSpLIzSAx79v/1kobMnyQ==,type:str]
+    WOODPECKER_GITEA_SECRET: ENC[AES256_GCM,data:oQieT/aEaZTVyBkj4nVmpfUSqcJ2wwCH6B3+Gz8TcyCWYRDLZ0jtEO3XnEzQTS6k9nmSd0YoJqo=,iv:p0y+qp4592k2+UKTIeuC2l9+JdjciCkYZXGghoBK9Fo=,tag:xYSNxS2m1+L3ynpTSlQIDg==,type:str]
 sops:
     kms: []
     gcp_kms: []
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2023-09-07T16:43:19Z"
+    lastmodified: "2024-05-23T00:59:04Z"
-    mac: ENC[AES256_GCM,data:JreywI1w6bDLlCntqpTfX+kNGO3GN4FT0egf6zi4dUwhIUOXRmVH0AXs3pYm8/1fMUrovG0W7uRXfHu5C28k+scEdLjuMhJbjz1yz6Gc9ewByN/nFGxS4vWdMkMQmPzi3Xnt7teVNkRXvlVODE9Qsn0pD627B+2sBboHar92kmI=,iv:hZ8B/oeuhT3DM7YENXbdYXI5toSG4xtp2iEyCSTxJQA=,tag:pHAE5CVr9xEMuAByANrrbA==,type:str]
+    mac: ENC[AES256_GCM,data:YSuq3l6X98ck/VD6jFyuL/wKxpTj5m7nDcRGMov0TNVVo6nIHHwnIAoOUvP1nnDwbgszJO+DDC0Db+33mNpZ6YqUiZCdJ6iwIqeldherUIjwqBkGCZLtNZ76SlJQnMulqcFt3nMG5UiVyw5TUbLDxPEGIau1bXzCRF2j6Q51Fu4=,iv:lnJTWIqbh0l2POWkv773mUUy5sMIunvfs+ztBwus3Lk=,tag:p5KkaHWgRHDG1LVZxbxx8Q==,type:str]
     pgp:
-        - created_at: "2023-09-07T16:27:38Z"
+        - created_at: "2024-05-23T00:59:04Z"
-          enc: |
+          enc: |-
             -----BEGIN PGP MESSAGE-----
 
-            hQIMAzKleRwoSoixARAAhfLs8yfYdUsm2H1rpnJ+s7IDvmPJfz1Zw6/IGxHKkXhJ
+            hQIMAyqlIeyoxYovAQ//Qi52py5A39dcBIHoz9VvM8mjUiRPbRroyURpG3KE+aT6
-            jeciJZX3D/Xkit5TTL8Rn8dj897bCORWd7Cr9hNj7sae5ZANiIQJAfraF4vsZEV4
+            HbGyzo7OT2xTRKM+RLrV/nGBJHkdykvyTAv22M1qnH1GqeUzPKcPE9RKKppMmW07
-            GUBrxNOp+pdCqOc3NhE3p5jceZ4COgeMbIR0MFuOip6KOJR9Xi3y9UmdSpQn8T1V
+            EDMkyJtbXSZrO2/gRQ9XuLN9NRzniboetZPqIWx9iTuYZCUH7Vehm5i5KNHZO2ID
-            SH5RpBpgmNk9P2q2I/m9+XtIBcvN1nXWRbblsrZ5g6JTacatBcjXInmvmEeMoABd
+            i/JkHJ4K6uJ3PwCEbxrYgE+ERj0IWCt1slythfrRuGx9zHPhYeHuNZBRGpqtjpeq
-            SE0BT2HHa+0i71MlkJl8qNcg5YWQAOPPa2NS0uJWBGwjn7M13eaiDIgmFLGXWd+k
+            eMld88EZYNG8uwuuvYbArxdFTNg0RUBgi+sOYjH7ZiDfs1GKZgGZXWxq2JvpgzA9
-            UrAfAatrG7psg3bwGztpsTKoA6TvwJxfm3hba7FwsfhsWLlikmh3cOvaoS9w0Edk
+            vr4ssbASKY3NYcwBc9k/g8/n4QsRoYPzIwtTtOIStgfwz0COHXgBLhITZLEfs0TB
-            WuIFrIjHnoHTGfTz63fUg+zyQHwCgxi17FKyVaz/zt4JgXQRw9OLwz1xurRjb2NG
+            BzH0JJ++KZBCW+I4NUm6RDNdU2Ps80vxN2JGdRmXYvqtM8XPHSaKhZrbQkudAK7p
-            INbkLShkQHNSvqG8kQNfjs5q34EE33N0C2TY9Fl4O243UdAd/jRM+3+QsDVQaEAZ
+            QdwFx5xJ/kIuGkepEwwLrZHhT9XTN6yGnkmb2yNiWHO2EfD3IcByNi2/DpDpvWL2
-            KYF/ZA/Df31dk637fQZABKdpmXqU32Gk/ZFGJhYe8GDLafI/tdUZQTeL/UA7NuQ8
+            dQjfRGlTJC+XG3u2drSivYjfJT4JqEaukk3OJV+4Kt/zjPh46W/oQVfub/zv12cJ
-            D65at6vbpIgmOIfIAxuGPtNRksj0KEnJnqVj2U7dw/7rqNoZHPue36vQt/ic/dYO
+            R+Y4ynQyrlMkroYLDB4A9MzHRWxyvWxpuP59KlU4nVIKgQnQKiGLbn+xxo/TuiIV
-            f6jsX5fXExvSmI1w67sr2u/PPIPBn+ZSzRlUgUe38QZdDwxWq4GC7PcsDWVuMxHS
+            KA4e9FnYKxXciOJ3egvX2gd4uv/XG/FmyRr+I+vjU4p60KYmP+tIODS8cZusTY+F
-            XgF/ki/LVMK6bI6FVJR/Cdrj5I9CjBwdQsmrMHoEoJXgpKY36E3hKK1Ryqs0qLE5
+            AgwDXjg0p2IN1X8BD/99maUVaC+WIw28AZ535oOkA/ommMGM0+y0E3pkqhGyp4fU
-            l/SpCEF8p6UBYd4Qlpx6Vc79PC7LbyWFENZvdPK9dIP1d8ssXsul1fUVHBPD4g8=
+            Yra0WsVMcGXVFokrs0lUMagu9htV/S6BtIERw6V3MappAm/+COVwZwuXGavNah1u
-            =wLQl
+            telD2URybiaB39DGx4tH+YSXtv5ejIj86j7xLsYqZlyPwRQikX8YxGG/lBJoVAy3
+            6l6Z3wDGNAZWMynebJGCAhTEpMO3FvSI/M5vyoupHT4MY580VDitK7379HlfSniu
+            a1sLsc8TxiC7zOkBy08Rix/7NYO1/PKXHxNX+3l7gtpncH87D58c9iLnIH1ozIst
+            CLgvV15BiiHHoHOIIvhYlZKu86fiLNgGQNx1fAS6ZWG8dLwU1+NPDcVJt36j2rcd
+            JQwCNsDloL5s2cdzIC6dtehNbK/zP1wRIL1+W3CCcZRjMPSHFrBb7Du1whhagcxI
+            /j9cV9kGpzSVyEp5xai+IAaIxasiM0Yk2pjgBC1PfoFjvCa0ZMe/L7dq/CTi5t29
+            YJBQNFhJosFuQIQk3zrwI7o41zsvH5aBAc3qL5L+57qLWB7urnzoDO1TSVCS0lBm
+            hH/e/1wNcX5/zlhe8D5Wj0Pd/culrrifLbNGTnk+pPDrPrF2qX7SIJphezurMWhl
+            bCxWpCMTKgR4hY8OXisNQkJNK23QD3Yv/d7Iyp1fgrKFARNEsUV6sbiaZtlN2NRo
+            AQkCEHhPePf6np/Pt9fcpwBhbXWxr2lX2aG2ZKXyo30KEhb4XMeDtrNfS2wCFm5G
+            hF7YAk2yjg3oO2+efBN+mU6B4j4PiCQBeoW8mg5kIT/8QBUv1DISw9x6Ogx1YWO3
+            Wl0MU3lk4hU=
+            =llh6
             -----END PGP MESSAGE-----
-          fp: 2CC2B3631D5C3393901335DB68F95C5D753EE1E5
+          fp: BD1AAF9D8170F4BEE437365FF6F0933799CFEBCD
-        - created_at: "2023-09-07T16:27:38Z"
+        - created_at: "2024-05-23T00:59:04Z"
-          enc: |
+          enc: |-
             -----BEGIN PGP MESSAGE-----
 
-            hQIMAy5t8IMoPu4VAQ//QYrbS6xfuWOrZ+NRwfqXJsDC0mVoDujFHUL8clhIB2dT
+            hQIMAy5t8IMoPu4VARAAhVkZCOiiicIA2RcjkF6ar4pzUeHSMes3fjn5GhEGILbQ
-            Kh1DGcqthw9cKRgkEc7Bz46GFak01XNP3EC/tl+B4Zwye6FSoGVabzC1jB8PwP2k
+            mzlEJ6P2Y6yJOd7oViE0BGx3UJVj/KtWUf8UCLVOjO8R9VpP63JX5PSTMmQjq/g1
-            MeFtVFGQa9iBC/w+67wng2ELypLCVVye9Fd91Dzm7ezk6FuU/PprXRrCcW9UHyXj
+            oyxpbuUcBXyu3O5nk9vNv22o6RzAfAm6HKn9hPh/rJjy34XRI1SS/YkjozLA7Wg0
-            9cMP87OE3Be/lMTuN9iz2PkNVg6EgHWtz6EIjDQYH3UFE2aAWeG4FuuYy2mdES4c
+            X9AOYoCLKiafpL2g4HVWELr21W3gJIkzQCouz92iYpmvFuK0iBR5juzxUBIRl5R8
-            HO1/D2pf21LUAOkABe4ifgyeWfUfTSnshUzegvShQRohMdLohSF9lbH9l+o8Hvwx
+            cjgyRS8GIZX9hKrZuNtmFj2+MqalEyMBYDIwNG3CBWJsO+/kQJcj//J4p9Bvo4bp
-            ECYfq9jW3vkRpleZA87hsVyhBD8TCuGRByVgQbYRlrnC/kTyJllRdesYEuFN9Cqj
+            fBkl37GcZHMTO/4Q9XaxDUoXp16hz1UR1NB/RhzjaU+FJ+8WNyCwPqzOaQKY4XXh
-            QEoD1nCrhU/vdNLR0o7WWIxl6jFYmroXLW5q8SP8XX1KPYIXzupWq9tHu6CHfmtr
+            zQQ3HOVZGjKWLdVYECDCgPxzfAytANzl0OLWfG5hMm+yz9Rg7avBe8AXIvxS50dS
-            /yBq4rbZOtAFitntg+FCMld2lmw1kr2H4VDxBGXdoC+va9PkjIS/ayorSWgmogck
+            6ory/bH0dXwYAOjRRCNYnBbovHSKM0feZlZ+TIb32gAUnyp6B9+USa+Keur8hl05
-            tgR3wJzFA1Bi7slxyJfZhRxTbPCbrny54j4291ZcBbB8pzkACQErvomXTdZwaclq
+            vTZC2Qz+KAeWx+Lnlj81KeRYCpuh8WsEAiYOWUzYXJOFHYGtj7PrftakP0UafWak
-            laMQzrGoLF4MdmDnKcMQvAvgFimEd2BpPCJsxtH2wLgEfRodUzbb8dIZM+EEVbaw
+            qseyNhn8fIe/iZdT0z9vD85ab17FwHUkfW8teJruMLKAr+26riZx6mJ8oAqgvsJ7
-            XVwFzjOLnHSLp2JtkGDW6eNEJYCk7kxQtkqFZ2etTtO3E0JKiIHatX4wygMcWoLS
+            /mFY1JQ+a/k+AyHjXg3DzWEM/w+JeteD++IgYhY0g2PvfsaGn9v234zLwnetoPzU
-            XgFyASgFilpOZpv2rE8A/90Mt4lVZrQO2c7dIRMaEqAOcCKWYxjshQ+/spkKe9eB
+            aAEJAhDQvADCSGFGXCLAmDOC10eauBgqkZDCnN5AYXVOV4VUryjRT4+PWhV7w5DF
-            froZsj2oV5ZtVwfpLG7Pw8R2taPu5DQ8uZHi9XdFPmYwmqpjEj3uZA+oo/NjgEY=
+            MZrEAMsKh0EzxhNDigzzHCGbcrKoPgFSYoJ4QHIirezN9Veq6e/lMnZTVDzxYjuL
-            =XD7x
+            +Aju/WQgu2de
+            =XUMm
             -----END PGP MESSAGE-----
           fp: 687802D4DFD8AA82EA55666CF7DADAC782D7663D
     encrypted_regex: ^(data|stringData)$
-    version: 3.7.3
+    version: 3.8.1

cluster/apps/monitoring/flux/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ./podmonitor.yaml
+- ./rules.yaml

cluster/apps/monitoring/flux/podmonitor.yaml

@@ -0,0 +1,32 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/monitoring.coreos.com/podmonitor_v1.json
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: flux-system
+  namespace: flux-system
+  labels:
+    app.kubernetes.io/part-of: flux
+    app.kubernetes.io/component: monitoring
+spec:
+  namespaceSelector:
+    matchNames:
+      - flux-system
+  selector:
+    matchExpressions:
+      - key: app
+        operator: In
+        values:
+          - helm-controller
+          - source-controller
+          - kustomize-controller
+          - notification-controller
+          - image-automation-controller
+          - image-reflector-controller
+  podMetricsEndpoints:
+    - port: http-prom
+      relabelings:
+        # Ref: https://github.com/prometheus-operator/prometheus-operator/issues/4816
+        - sourceLabels: [__meta_kubernetes_pod_phase]
+          action: keep
+          regex: Running

cluster/apps/monitoring/flux/rules.yaml

@@ -0,0 +1,32 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/monitoring.coreos.com/prometheusrule_v1.json
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: flux-rules
+  namespace: monitoring
+spec:
+  groups:
+    - name: flux.rules
+      rules:
+        - alert: FluxComponentAbsent
+          annotations:
+            summary: Flux component has disappeared from Prometheus target discovery.
+          expr: |
+            absent(up{job=~".*flux-system.*"} == 1)
+          for: 15m
+          labels:
+            severity: critical
+        - alert: FluxReconciliationFailure
+          annotations:
+            summary: >-
+              {{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} reconciliation
+              has been failing for more than 15 minutes.
+          expr: |
+            max(gotk_reconcile_condition{status="False",type="Ready"}) by (namespace, name, kind)
+              +
+            on(namespace, name, kind) (max(gotk_reconcile_condition{status="Deleted"})
+              by (namespace, name, kind)) * 2 == 1
+          for: 15m
+          labels:
+            severity: critical

cluster/apps/monitoring/kustomization.yaml

@@ -10,4 +10,5 @@ resources:
 - ./victoria-metrics
 - ./varken
 - ./proxmoxve-exporter
-- ./external-monitors
+- ./external-monitors
+- ./flux