From e0fc99a695c0b613c776eb12856d975be6430fe9 Mon Sep 17 00:00:00 2001
From: SeanOMik <seanomik@gmail.com>
Date: Sun, 30 Apr 2023 01:27:12 -0400
Subject: [PATCH] Fix alertmanager alerts for kubeProxy, kubeScheduler, and
 kubeControllerManager

---
 .../alertmanager-config.yaml                  | 56 ++++++++++++++++++-
 .../kube-prometheus-stack/helm-release.yaml   | 48 ++++++++++++++--
 2 files changed, 98 insertions(+), 6 deletions(-)

diff --git a/cluster/apps/monitoring/kube-prometheus-stack/alertmanager-config.yaml b/cluster/apps/monitoring/kube-prometheus-stack/alertmanager-config.yaml
index e21e8be..66faaaa 100644
--- a/cluster/apps/monitoring/kube-prometheus-stack/alertmanager-config.yaml
+++ b/cluster/apps/monitoring/kube-prometheus-stack/alertmanager-config.yaml
@@ -8,7 +8,59 @@ spec:
     - name: gotify-bridge
       webhookConfigs:
         - url: http://alertmanager-gotify-bridge.monitoring:8080/gotify_webhook
-          sendResolved: true
+          sendResolved: false
 
   route:
-    receiver: 'gotify-bridge'
\ No newline at end of file
+    receiver: 'gotify-bridge'
+
+    groupBy:
+    - namespace
+    
+    routes:
+    - receiver: null
+      matchers:
+      - matchType: "=~"
+        name: "alertname"
+        value: "InfoInhibitor|Watchdog"
+      continue: false
+
+    continue: false
+    groupWait: 30s
+    groupInterval: 5m
+    repeatInterval: 12h
+
+  inhibitRules:
+  - sourceMatch:
+    - matchType: "="
+      name: "severity"
+      value: "critical"
+    targetMatch:
+    - matchType: "=~"
+      name: "severity"
+      value: "warning|info"
+    equal:
+    - namespace
+    - alertname
+
+  - sourceMatch:
+    - matchType: "="
+      name: "severity"
+      value: "warning"
+    targetMatch:
+    - matchType: "="
+      name: "severity"
+      value: "info"
+    equal:
+    - namespace
+    - alertname
+
+  - sourceMatch:
+    - matchType: "="
+      name: "alertname"
+      value: "InfoInhibitor"
+    targetMatch:
+    - matchType: "="
+      name: "severity"
+      value: "info"
+    equal:
+    - namespace
\ No newline at end of file
diff --git a/cluster/apps/monitoring/kube-prometheus-stack/helm-release.yaml b/cluster/apps/monitoring/kube-prometheus-stack/helm-release.yaml
index 23696aa..67aa7ce 100644
--- a/cluster/apps/monitoring/kube-prometheus-stack/helm-release.yaml
+++ b/cluster/apps/monitoring/kube-prometheus-stack/helm-release.yaml
@@ -51,10 +51,6 @@ spec:
               - *grafana-host
             secretName: wildcard-main-tls
 
-    # Enable this when the cluster expands to have more than a single node
-    kubeEtcd:
-      enabled: false
-
     prometheus:
       ingress:
         enabled: true
@@ -70,3 +66,47 @@ spec:
           - hosts:
               - *prometheus-host
             secretName: wildcard-main-tls
+
+      prometheusSpec:
+        enableAdminAPI: false
+
+    kubeControllerManager:
+      enabled: true
+      endpoints:
+      - 192.168.87.29
+      service:
+        enabled: true
+        port: 10257
+        targetPort: 10257
+      serviceMonitor:
+        enabled: true
+        https: true
+        insecureSkipVerify: true
+
+    kubeScheduler:
+      enabled: true
+      endpoints:
+      - 192.168.87.29
+      service:
+        enabled: true
+        port: 10259
+        targetPort: 10259
+      serviceMonitor:
+        enabled: true
+        https: true
+        insecureSkipVerify: true
+
+    kubeProxy:
+      enabled: true
+      endpoints:
+      - 192.168.87.29
+      service:
+        enabled: true
+        port: 10249
+        targetPort: 10249
+      serviceMonitor:
+        enabled: true
+        https: false
+
+    kubeEtcd:
+      enabled: false