feat(thin): switch to cloudnative-pg

kubernetes/thin/apps/database/postgresql/app/helm-release.yaml

@@ -1,42 +1,35 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2.json
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
-  name: postgresql
+  name: cloudnative-pg
   namespace: database
 spec:
-  interval: 5m
+  interval: 30m
   chart:
     spec:
-      chart: postgresql
+      chart: cloudnative-pg
-      version: 14.3.x
+      version: 0.23.0
       sourceRef:
         kind: HelmRepository
-        name: bitnami-charts
+        name: cloudnative-pg
         namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  dependsOn:
+    - name: openebs
+      namespace: openebs
   values:
-    auth:
+    crds:
-      existingSecret: "pgsql-secrets"
+      create: true
-      secretKeys:
+    monitoring:
-        adminPasswordKey: "adminPassword"
+      podMonitorEnabled: false
-        replicationPasswordKey: "replicationPassword"
+      grafanaDashboard:
-
+        create: true
-    serviceMonitor:
-      enabled: true
-      labels:
-        release: kube-prometheus-stack
-
-    volumePermissions:
-      enabled: true
-
-    primary:
-      persistence:
-        existingClaim: "postgresql-pvc"
-
-      containerSecurityContext:
-        enabled: true
-        runAsUser: 655
-
-    readReplicas:
-      containerSecurityContext:
-        enabled: true
-        runAsUser: 655

kubernetes/thin/apps/database/postgresql/app/helm-repository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: cloudnative-pg
+  namespace: flux-system
+spec:
+  interval: 2h
+  url: https://cloudnative-pg.io/charts

kubernetes/thin/apps/database/postgresql/app/kustomization.yaml

@@ -1,7 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- ./pgsql-pv.yaml
+- ./secret.sops.yaml
-- ./pgsql.sops.yaml
+- ./helm-repository.yaml
-- ./helm-release.yaml
+- ./helm-release.yaml
-#- ./pgadmin4

kubernetes/thin/apps/database/postgresql/app/secret.sops.yaml

@@ -0,0 +1,76 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: cloudnative-pg-secret
+    namespace: database
+    labels:
+        cnpg.io/reload: "true"
+stringData:
+    username: ENC[AES256_GCM,data:+rcROvnLUkE=,iv:9/iJnvFrGpJVeaudylxzbopKJG/bgF3X8BOWMGGx3O8=,tag:9dSm9nOnXxT4OnYVmgMjlw==,type:str]
+    password: ENC[AES256_GCM,data:VWqq1xpYEgm7HlyqbwdU9jIb+EiE7+IU4CgW2HrpsoM=,iv:YW7XD8RCSTo2f7COYDwP+bzsbXR79TJkkr6/BtlquxQ=,tag:muNMiMnDQjSfZ+sG+gQI0Q==,type:str]
+    minioAccessKey: ENC[AES256_GCM,data:qF/qzRRETaszWb1kz8JTIg==,iv:q2gHGMu/CsjdXcZM2BohFwqVdzQ7rkfn2tDJ1YqMxcc=,tag:+2kg6bkvDNzTdUMtshadpQ==,type:str]
+    minioSecretKey: ENC[AES256_GCM,data:HhHdmxJKlkF2FEyyWPVpihN6Bcv0fViD4corxDhkTnY=,iv:IhczTpr9MlspwjciGoj4EmYDlyan5UgphhLOFLrRi0A=,tag:HC2XPxhwyCj3Dy1+tH5o3A==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2025-02-20T03:30:36Z"
+    mac: ENC[AES256_GCM,data:L68jmZTT1PiBRo8Uec7fkPgt+uMP+n37/aM1LzrZ3RNUBdvsh9ih9GbdN9Lx+YdI0fS5YzBtB6ho7X8a7QHAxDyYr6B7iUiRvBpYIOXrKwssgHSuZs3e9v8bhkHCYsMgPOfFpWmZTATYgTb5KFHlB5CE5F7JisvYqtX5N+He8as=,iv:OLZu4dj4tppvjKynqc+h6kNGqHecxuNm6KsyZpUnJFM=,tag:E9AalIgW+Kt4XoH242ldow==,type:str]
+    pgp:
+        - created_at: "2025-02-20T03:30:36Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMAyqlIeyoxYovAQ/+Nl9LC0XPqccrXVMaQSBMORQKp1eclU5TCwIb1TnMeh1b
+            4i0OiXqdxGhmr5+s5PY8ns1hswYJkkR87ArlwI3kigErQqlw+R8uXhwGjgTTbLEl
+            u9pJPwQqfVA+H82/2JOpG+FZMzMK94U+I6F6A5hF/f+OANDm9fGO56ydCjZk6lp8
+            vmYq/ufYhn7tz9vN9dXvqNbYjoTXA2tCq66k9qeng3jZOp79EFJr/jhRsc0DEaoR
+            E8oQdGeN7K0BRDSdRpxa0ZXRYherUA41aAVyCqsD+1YynHEwPkxLIK2qJ4Y9Q5HF
+            +TDCi8ujg2NbeY+Z9WjpNEYB4Q7ppOZ0Uktv60ZLFc5Sbjmr8O8ojjkVN5rkQhMh
+            l3RWrJj4JQkg2U/sWim+dqjR1BIUN4hXu2+3s4u3vQnyATA1ep2LdEgWAI/XsdB4
+            z8Rpe4Ul38HvbLpxWnkFDyavwCI57fKN/IpQazu7N6aeYnVuZZ5xR/DY0hwe7R+q
+            FRY5P3xgu3LTF/edKiyG2j2aS1HUUe06gYUilaIjSL3yS9zf/5g8H8p53uLh/qZQ
+            TGmLNuXJ2r3yF8XNbsRhz2nTKrKZKIYJvpAx0yVvkvUSOgib+4g/97r70drXutju
+            /yNJNnjicbbZqdLQ+ivqbjiPq436C2gohfL37c1S1yNJcUgXFJd2bnahtwvj2GCF
+            AgwDXjg0p2IN1X8BEAC+B45dkZQ9RM1P3+0PHre5Zg4uam89/LJX1nk2ExAtEizX
+            BKZO950ImNg52u0ptIBcg4iZRGjoWOGmDS3hK6bh1x84fZArvBOFsM6UDB6CFthM
+            Ly+sphRT72RQZvOdmi8QGIU/f4DKurpHQYFBwIHzTTa3URCYC2/Ov/8Jmzojuuqy
+            DLyIulkcnWtBMyFV1scfXEULeqQ+nqEOoTFcSqF4sYKjVFIm/GC9JTbuYR15Ou6A
+            vo6iCyMcY6pYfP0FkHhosA468OgtMSkmBqVHJ8QFbbHKR+lb4nMYcSqvvOM3hQil
+            9yn6UNOwc3QM6w0sWIPriEiO/68GGj6h5BlG0hf/W/4iYW9Qrn0HaTTpmYRshRO/
+            CZM1sOFpscChoRXbiZEmSjqzGzAhko8uHmeQQzsKJbM+RcK1ByFquMwa7qZWQX6l
+            DpbnU8pUci7gPxXy5i3guSYg/2T7ZhY1y7MXz5Qj3zBkup9nxEC1lY+TG3QPRDdR
+            r8kk2wBemk31XYbWUi1yuNB6nD4LUUSxt71XpRz5lrAKbtx0zF1Hn8cjiBG0lxVY
+            66GwE4HPkPBPiS72OMrZBeK9+psrCu6RKMUfSd2Agx+RSOfen0po3yaoOYPXxlz2
+            CGKKkp4X8nczrBvQBmzAeEwGJ2woZPKBQr9xw/6KhB6TbiT/feZNWtWhPP6tndRo
+            AQkCEK3IWRscYRHutaqYrIhamhWIianbcCxRsR9WWbhHp+1DceYMdiqFVEUfoIIa
+            wlsmeKXm09YEudhsQ04PsxzdtLWSnw8ChqasGyW1R8/s7VBe04CobdsiosQpPl6g
+            WVZjq+H94q0=
+            =OCD4
+            -----END PGP MESSAGE-----
+          fp: BD1AAF9D8170F4BEE437365FF6F0933799CFEBCD
+        - created_at: "2025-02-20T03:30:36Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMAy5t8IMoPu4VAQ/7BvkTIPU/SK3rQ/c4FLyFKSCk/UKB9he62yc1/skp71UU
+            1wRuzmJkKO67vJvBbnszgaK4KfJHbNK1613cY+KqmqmwcXX2OpOrPoYa8BWzte76
+            lVVtmvXLvsCkVgtaIIcjeH+3ByEbiv7KDTtgSrv+hWuv0r2XuC+PcOKDUQo0qwW+
+            ntZb5fr4v+t6xWMHxxQTDNfGLfYwhwHAwaMtr6kiMQyYyxH/7Grj6zwG70spggM6
+            aH5go/1onBYfArDqr0mbimbclGRh2Xw7g7Uk4+Q4Tj6K/NNA9k0g+M1ktrCQGiHi
+            y1a36POPUkcGpuSzurmRDvlpKaXPyr7HbyHk467INl8eYT/O0H6jqGrrzUF1J2sA
+            MCy09WzzE8RV4mg59Sv47UO7YKbHB9/jhyr2eNWP7qTMN2ohq/zTjw10qyOuypgi
+            DYYtCHOh0DHTuFkEkWO9XWaXMGctQQazwtvZBvkjp2Hcg1DODG6NADdKqBoxdSsd
+            SyslogaXn+nCG/7V3RNALnoDKYHJF58aeVsQb9C1XJjPqIwW9125ok/J9EvROL00
+            Ai3gtzFDF/psQr2F8Igm/NWd3L2Fed2lOrIE6iKLXGIAZIPDNTvr21gpXPeeHuQB
+            dWCTBcoxe84w370ZRp+uE2DEPMz7Ne520jQPWvnGrRRWfx+DWkJUXIRCFZo0z6bU
+            aAEJAhCH4W4w0SC3oVcCWRW9vZPqsHeqoiJ8jKVRXojoxr0Pfm3AWaWpQ8vW23RW
+            0fDXbyVSNCQYLXL+OMEHyvMcJD/e6/9aoqhJvlvINf/YS9ZihlgPKH0LU5Zh1E+n
+            lm0AcJjp4usR
+            =cLbD
+            -----END PGP MESSAGE-----
+          fp: 687802D4DFD8AA82EA55666CF7DADAC782D7663D
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.1

kubernetes/thin/apps/database/postgresql/cluster/cluster16.yaml

@@ -0,0 +1,88 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/postgresql.cnpg.io/cluster_v1.json
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: postgres16
+  namespace: database
+spec:
+  instances: 3
+  imageName: ghcr.io/tensorchord/cloudnative-pgvecto.rs:16.4 #ghcr.io/cloudnative-pg/postgresql:16.4-28
+  primaryUpdateStrategy: unsupervised
+  storage:
+    size: 20Gi
+    storageClass: openebs-zfs-mainpool
+  superuserSecret:
+    name: cloudnative-pg-secret
+  enableSuperuserAccess: true
+  postgresql:
+    shared_preload_libraries:
+      - "vectors.so"
+    parameters:
+      max_connections: "400"
+      shared_buffers: 256MB
+    enableAlterSystem: true # must be set to true to enable pgvecto.rs
+  nodeMaintenanceWindow:
+    inProgress: false
+    reusePVC: true
+  resources:
+    requests:
+      cpu: 500m
+    limits:
+      memory: 4Gi
+  monitoring:
+    enablePodMonitor: true
+  backup:
+    retentionPolicy: 30d
+    barmanObjectStore: &barmanObjectStore
+      data:
+        compression: bzip2
+      wal:
+        compression: bzip2
+        maxParallel: 8
+      destinationPath: s3://thin-cloudnative-pg/
+      endpointURL: https://s3.seanomik.net
+      # Note: serverName version needs to be inclemented
+      # when recovering from an existing cnpg cluster
+      serverName: &currentCluster postgres16-v1
+      s3Credentials:
+        accessKeyId:
+          name: cloudnative-pg-secret
+          key: minioAccessKey
+        secretAccessKey:
+          name: cloudnative-pg-secret
+          key: minioSecretKey
+
+  # Note: previousCluster needs to be set to the name of the previous
+  # cluster when recovering from an existing cnpg cluster
+  bootstrap:
+    # recovery:
+    #   source: &previousCluster postgres16-v1
+    initdb:
+      database: init
+      owner: postgres
+    #   import:
+    #     type: monolith
+    #     databases:
+    #       - "*"
+    #     roles:
+    #       - "*"
+    #     source:
+    #       externalCluster: old-cluster
+
+  # Note: externalClusters is needed when recovering from an existing cnpg cluster
+  # externalClusters:
+  #   - name: *previousCluster
+  #     barmanObjectStore:
+  #       <<: *barmanObjectStore
+  #       serverName: *previousCluster
+    # - name: old-cluster
+    #   connectionParameters:
+    #     # Use the correct IP or host name for the source database
+    #     host: postgresql.database.svc
+    #     user: postgres
+    #     dbname: postgres
+    #     #sslmode: require
+    #   password:
+    #     name: cloudnative-pg-secret
+    #     key: password

kubernetes/thin/apps/database/postgresql/cluster/kustomization.yaml

@@ -0,0 +1,8 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ./cluster16.yaml
+- ./scheduledbackup.yaml
+- ./prometheusrule.yaml

kubernetes/thin/apps/database/postgresql/cluster/prometheusrule.yaml

@@ -0,0 +1,74 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/monitoring.coreos.com/prometheusrule_v1.json
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cloudnative-pg-rules
+  namespace: database
+spec:
+  groups:
+    - name: cloudnative-pg.rules
+      rules:
+        - alert: LongRunningTransaction
+          annotations:
+            description: Pod {{ $labels.pod }} is taking more than 5 minutes (300 seconds) for a query.
+            summary: A query is taking longer than 5 minutes.
+          expr: |-
+            cnpg_backends_max_tx_duration_seconds > 300
+          for: 1m
+          labels:
+            severity: warning
+        - alert: BackendsWaiting
+          annotations:
+            description: Pod {{ $labels.pod }} has been waiting for longer than 5 minutes
+            summary: If a backend is waiting for longer than 5 minutes
+          expr: |-
+            cnpg_backends_waiting_total > 300
+          for: 1m
+          labels:
+            severity: warning
+        - alert: PGDatabase
+          annotations:
+            description: Over 300,000,000 transactions from frozen xid on pod {{ $labels.pod }}
+            summary: Number of transactions from the frozen XID to the current one
+          expr: |-
+            cnpg_pg_database_xid_age > 300000000
+          for: 1m
+          labels:
+            severity: warning
+        - alert: PGReplication
+          annotations:
+            description: Standby is lagging behind by over 300 seconds (5 minutes)
+            summary: The standby is lagging behind the primary
+          expr: |-
+            cnpg_pg_replication_lag > 300
+          for: 1m
+          labels:
+            severity: warning
+        - alert: LastFailedArchiveTime
+          annotations:
+            description: Archiving failed for {{ $labels.pod }}
+            summary: Checks the last time archiving failed. Will be < 0 when it has not failed.
+          expr: |-
+            (cnpg_pg_stat_archiver_last_failed_time - cnpg_pg_stat_archiver_last_archived_time) > 1
+          for: 1m
+          labels:
+            severity: warning
+        - alert: DatabaseDeadlockConflicts
+          annotations:
+            description: There are over 10 deadlock conflicts in {{ $labels.pod }}
+            summary: Checks the number of database conflicts
+          expr: |-
+            cnpg_pg_stat_database_deadlocks > 10
+          for: 1m
+          labels:
+            severity: warning
+        - alert: ReplicaFailingReplication
+          annotations:
+            description: Replica {{ $labels.pod }} is failing to replicate
+            summary: Checks if the replica is failing to replicate
+          expr: |-
+            cnpg_pg_replication_in_recovery > cnpg_pg_replication_is_wal_receiver_up
+          for: 1m
+          labels:
+            severity: warning

kubernetes/thin/apps/database/postgresql/cluster/scheduledbackup.yaml

@@ -0,0 +1,13 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/postgresql.cnpg.io/scheduledbackup_v1.json
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: postgres
+  namespace: database
+spec:
+  schedule: "@daily"
+  immediate: true
+  backupOwnerReference: self
+  cluster:
+    name: postgres16

kubernetes/thin/apps/database/postgresql/ks.yaml

@@ -2,10 +2,9 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: postgresql
+  name: cloudnative-pg
   namespace: flux-system
 spec:
-  targetNamespace: database
   timeout: 5m
   interval: 10m
   path: ./kubernetes/thin/apps/database/postgresql/app
@@ -18,10 +17,39 @@ spec:
     secretRef:
       name: sops-gpg
   dependsOn:
+    - name: openebs
     - name: openebs-sc
-      namespace: flux-system
   postBuild:
-    substitute: {}
+    substituteFrom:
+      - kind: ConfigMap
+        name: cluster-settings
+      - kind: Secret
+        name: cluster-secrets
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cloudnative-pg-cluster
+  namespace: flux-system
+spec:
+  timeout: 5m
+  interval: 10m
+  path: ./kubernetes/thin/apps/database/postgresql/cluster
+  prune: true
+  targetNamespace: database
+  sourceRef:
+    kind: GitRepository
+    name: home-cluster
+  decryption:
+    provider: sops
+    secretRef:
+      name: sops-gpg
+  dependsOn:
+    - name: openebs
+    - name: openebs-sc
+    - name: cloudnative-pg
+  postBuild:
     substituteFrom:
       - kind: ConfigMap
         name: cluster-settings

kubernetes/thin/apps/database/postgresql_old/app/helm-release.yaml

@@ -0,0 +1,42 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: postgresql
+  namespace: database
+spec:
+  interval: 5m
+  chart:
+    spec:
+      chart: postgresql
+      version: 14.3.x
+      sourceRef:
+        kind: HelmRepository
+        name: bitnami-charts
+        namespace: flux-system
+  values:
+    auth:
+      existingSecret: "pgsql-secrets"
+      secretKeys:
+        adminPasswordKey: "adminPassword"
+        replicationPasswordKey: "replicationPassword"
+
+    serviceMonitor:
+      enabled: true
+      labels:
+        release: kube-prometheus-stack
+
+    volumePermissions:
+      enabled: true
+
+    primary:
+      persistence:
+        existingClaim: "postgresql-pvc"
+
+      containerSecurityContext:
+        enabled: true
+        runAsUser: 655
+
+    readReplicas:
+      containerSecurityContext:
+        enabled: true
+        runAsUser: 655

kubernetes/thin/apps/database/postgresql_old/app/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ./pgsql-pv.yaml
+- ./pgsql.sops.yaml
+- ./helm-release.yaml
+#- ./pgadmin4

kubernetes/thin/apps/database/postgresql_old/ks.yaml

@@ -0,0 +1,29 @@
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: postgresql
+  namespace: flux-system
+spec:
+  targetNamespace: database
+  timeout: 5m
+  interval: 10m
+  path: ./kubernetes/thin/apps/database/postgresql/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-cluster
+  decryption:
+    provider: sops
+    secretRef:
+      name: sops-gpg
+  dependsOn:
+    - name: openebs-sc
+      namespace: flux-system
+  postBuild:
+    substitute: {}
+    substituteFrom:
+      - kind: ConfigMap
+        name: cluster-settings
+      - kind: Secret
+        name: cluster-secrets