diff --git a/cluster/apps/database/kustomization.yaml b/cluster/apps/database/kustomization.yaml index 50d9f2e..f1e1f8a 100644 --- a/cluster/apps/database/kustomization.yaml +++ b/cluster/apps/database/kustomization.yaml @@ -6,4 +6,4 @@ resources: - ./postgresql - ./redis - ./minio -- ./mariadb \ No newline at end of file +#- ./mariadb \ No newline at end of file diff --git a/cluster/apps/monitoring/kube-prometheus-stack/helm-release.yaml b/cluster/apps/monitoring/kube-prometheus-stack/helm-release.yaml index 67aa7ce..dae62c6 100644 --- a/cluster/apps/monitoring/kube-prometheus-stack/helm-release.yaml +++ b/cluster/apps/monitoring/kube-prometheus-stack/helm-release.yaml @@ -51,6 +51,23 @@ spec: - *grafana-host secretName: wildcard-main-tls + sidecar: + datasources: + defaultDatasourceEnabled: false + isDefaultDatasource: false + + # Add Victoria Metrics as the default datasource + additionalDataSources: + - name: Victoria + uid: victoria-metrics-server + type: prometheus + jsonData: + tlsSkipVerify: true + editable: false + url: http://victoria-metrics-server.monitoring.svc:8428 + version: 1 + isDefault: true + prometheus: ingress: enabled: true @@ -70,6 +87,22 @@ spec: prometheusSpec: enableAdminAPI: false + retention: 1d + + remoteWrite: + - url: http://victoria-metrics-server.monitoring.svc:8428/api/v1/write + + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: hostpath + selector: + matchLabels: + app.kubernetes.io/name: kube-prometheus-stack-pv + resources: + requests: + storage: 30Gi + kubeControllerManager: enabled: true endpoints: diff --git a/cluster/apps/monitoring/kube-prometheus-stack/kustomization.yaml b/cluster/apps/monitoring/kube-prometheus-stack/kustomization.yaml index ebc0bda..d616c15 100644 --- a/cluster/apps/monitoring/kube-prometheus-stack/kustomization.yaml +++ b/cluster/apps/monitoring/kube-prometheus-stack/kustomization.yaml @@ -1,6 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: +- ./prom-pv.yaml - ./helm-repository.yaml - ./helm-release.yaml - ./alertmanager-config.yaml \ No newline at end of file diff --git a/cluster/apps/monitoring/kube-prometheus-stack/prom-pv.yaml b/cluster/apps/monitoring/kube-prometheus-stack/prom-pv.yaml new file mode 100644 index 0000000..58cf94c --- /dev/null +++ b/cluster/apps/monitoring/kube-prometheus-stack/prom-pv.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: prometheus-pv + namespace: monitoring + labels: + app.kubernetes.io/name: kube-prometheus-stack-pv +spec: + storageClassName: hostpath + persistentVolumeReclaimPolicy: Retain + capacity: + storage: 30Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/mnt/MainPool/Kubernetes/prometheus" \ No newline at end of file diff --git a/cluster/apps/monitoring/kustomization.yaml b/cluster/apps/monitoring/kustomization.yaml index e8cbc88..1b65376 100644 --- a/cluster/apps/monitoring/kustomization.yaml +++ b/cluster/apps/monitoring/kustomization.yaml @@ -5,4 +5,5 @@ resources: - ./network_policy.yaml - ./kube-prometheus-stack - ./zfs-exporter -- ./alertmanager-gotify-bridge \ No newline at end of file +- ./alertmanager-gotify-bridge +- ./victoria-metrics \ No newline at end of file diff --git a/cluster/apps/monitoring/mimir/helm-release.yaml b/cluster/apps/monitoring/mimir/helm-release.yaml deleted file mode 100644 index afdbf51..0000000 --- a/cluster/apps/monitoring/mimir/helm-release.yaml +++ /dev/null @@ -1,188 +0,0 @@ -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: mimir - namespace: monitoring -spec: - interval: 5m - chart: - spec: - chart: mimir-distributed - version: 2.8.x - sourceRef: - kind: HelmRepository - name: grafana-charts - namespace: flux-system - - values: - global: - extraEnvFrom: - - secretRef: - name: mimir-secret - - mimir: - structuredConfig: - common: - storage: - backend: s3 - s3: - endpoint: minio.database:9000 - access_key_id: $${S3_ACCESS_KEY} - secret_access_key: $${S3_SECRET_KEY} - insecure: true - bucket_name: mimir - - compactor: - persistentVolume: - size: 20Gi - resources: - limits: - memory: 2.1Gi - requests: - cpu: 1 - memory: 1.5Gi - - distributor: - replicas: 2 - resources: - limits: - memory: 5.7Gi - requests: - cpu: 2 - memory: 4Gi - - ingester: - persistentVolume: - size: 50Gi - replicas: 3 - resources: - limits: - memory: 12Gi - requests: - cpu: 3.5 - memory: 8Gi - topologySpreadConstraints: {} - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: target # support for enterprise.legacyLabels - operator: In - values: - - ingester - topologyKey: 'kubernetes.io/hostname' - - - labelSelector: - matchExpressions: - - key: app.kubernetes.io/component - operator: In - values: - - ingester - topologyKey: 'kubernetes.io/hostname' - - zoneAwareReplication: - topologyKey: 'kubernetes.io/hostname' - - admin-cache: - enabled: true - replicas: 2 - - chunks-cache: - enabled: true - replicas: 2 - - index-cache: - enabled: true - replicas: 3 - - metadata-cache: - enabled: true - - results-cache: - enabled: true - replicas: 2 - - minio: - enabled: false - - # Deployed by kube-prometheus-stack - alertmanager: - enabled: false - - overrides_exporter: - replicas: 1 - resources: - limits: - memory: 128Mi - requests: - cpu: 100m - memory: 128Mi - - querier: - replicas: 1 - resources: - limits: - memory: 5.6Gi - requests: - cpu: 2 - memory: 4Gi - - query_frontend: - replicas: 1 - resources: - limits: - memory: 2.8Gi - requests: - cpu: 2 - memory: 2Gi - - ruler: - replicas: 1 - resources: - limits: - memory: 2.8Gi - requests: - cpu: 1 - memory: 2Gi - - store_gateway: - persistentVolume: - size: 10Gi - replicas: 3 - resources: - limits: - memory: 2.1Gi - requests: - cpu: 1 - memory: 1.5Gi - topologySpreadConstraints: {} - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: target # support for enterprise.legacyLabels - operator: In - values: - - store-gateway - topologyKey: 'kubernetes.io/hostname' - - - labelSelector: - matchExpressions: - - key: app.kubernetes.io/component - operator: In - values: - - store-gateway - topologyKey: 'kubernetes.io/hostname' - zoneAwareReplication: - topologyKey: 'kubernetes.io/hostname' - - nginx: - replicas: 1 - resources: - limits: - memory: 731Mi - requests: - cpu: 1 - memory: 512Mi \ No newline at end of file diff --git a/cluster/apps/monitoring/mimir/mimir.sops.yaml b/cluster/apps/monitoring/mimir/mimir.sops.yaml deleted file mode 100644 index 6428203..0000000 --- a/cluster/apps/monitoring/mimir/mimir.sops.yaml +++ /dev/null @@ -1,61 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: mimir-secret - namespace: monitoring -stringData: - S3_ACCESS_KEY: ENC[AES256_GCM,data:jfnHq3DE,iv:Ft3d/tbvCKuTDHmCXZJgYl5xVBOwIj0Zkc9+JgILDAI=,tag:5bcZBsODsA9Pi2vf5OGsHg==,type:str] - S3_SECRET_KEY: ENC[AES256_GCM,data:3WpNKx1d,iv:M5xewbvJm+U8td7kIpkPImd2gDIFfVTGVIR5BJtfoB8=,tag:X78jSBvcHbSIu6S8W8yZNA==,type:str] -sops: - kms: [] - gcp_kms: [] - azure_kv: [] - hc_vault: [] - age: [] - lastmodified: "2023-05-01T22:12:24Z" - mac: ENC[AES256_GCM,data:SywFZE0Kj1lx1X1f5chgW7qycPwQvHkRz/35F/hKBLjr0UXI1T9D3IIQeNZlTrxJwSiCvm/+FxMxbF4hJBfZ61Z2jfgwDINghPkoNJothgV0dlPtFTfApgK2BfNqWffhPc3Qj4cmuQZV6kG0h05CbKL4PN89DQ/aEDPPbKI01lo=,iv:x1ZGglUJM/PT5gZgvxRR411pSFmlDkEADrd3arCqFdY=,tag:0xlalnODXYns3CpuDxt9vQ==,type:str] - pgp: - - created_at: "2023-04-07T01:57:22Z" - enc: | - -----BEGIN PGP MESSAGE----- - - hQIMAzKleRwoSoixAQ/9Hi4VyrUXV7LvbCFiLbyfv314lMGwrAf+2po/4Lr1hANe - KiwpfthiNheAjNaGCG6v2C1rx2Wrr5G3+rMik/1TLWbg2u9zZU4mWO8bwJUGXKDo - /T1nl47f09UPDtQ6KiG0nPf3M0Ovmk3d63R3zpY4Q7uE4uhLNDr0KD9mp7MmRCbZ - PO++tdiZa67z9owNDh/NSnQr9Y6JwjlxlkJl5SJ76vaK/SaOi/j86mOm9CV6SQmk - cLOwiO7JxV8I4gD9jlLdYEPS+nqztX5eHLRoaXsAQrX4DdWNnOF0C2sk9nMHwQTb - W8/SVmg7TiVVL6qVCXgUCgFRXllrlGlXlfv+W6ruuZIBv2MAA1V+afl5A3/KVvE6 - FDq9YrJ4XfZPCD2ZByM2386L8MiUwkfF/3uge38MT/WDU2DTT+g7jV3UQs+Awi8f - N4YBVBcp5jGTkMD0347GPfPF7kdiN/YFZ/Ws1jf/EsS6vOpKNlPn64fVJfTSfdie - rvNxksi8Y4vpwEngy38t7JRfpJniDo9iK9EwhXMChYXnWkiz/B3vMoii496B7TzO - 9gKd4v7kFA6iXI+wqbYrZfOGeLZlMI99pwTatNL4fo9ABJ7JScISzTvS7p/xB6Ae - JPdlA0Tf8wP4RYz8YYRcNlfEQPZYb4kHj5r9Ei59InHzwKfq9GyKKvluS0/k3NHU - aAEJAhCVkPuIHluRLHsjVEbKbFzSJUG8p/hSSmQnfk3CT36/dJhgv3jzoL+1/Sx1 - o8OwWPmNq8TuX9SaXfhfy/EGMulWgRaztxt9D+0+wgc8IOAPp+0SYUsaOa0T9+Pl - pjU1GRaK5AlT - =mItp - -----END PGP MESSAGE----- - fp: 2CC2B3631D5C3393901335DB68F95C5D753EE1E5 - - created_at: "2023-04-07T01:57:22Z" - enc: | - -----BEGIN PGP MESSAGE----- - - hQIMA4WLYkVpP8xtAQ/9FQGyKS1wEodU9ZVZ8kxijp6aFtMCmL/I5HBEhbSLj0P9 - TVD0QwnUPZqf7zlWrAh6TspyLQdRMt9JAYZCPyLgu//FdKfBJNYeU3+aWj/lMtJ4 - Twgs7NPtGbRJcpF+a4NmAOIqzKfJI+h714BLFoWrGtUmTE9/dBHh2yxADSgprY1o - /4J8aHQfaqg5JwijP3PhtRMxla4YQfhqf0JRAcmQPKUDuxT2QG/wp59Fq/665aaO - JFWiCOPBqTtEhY4ML4EYNUV+Cd7UT7LOXC+Xzuj1eEGMV1Pmqd1u1UyQKvHOOXhT - AfGeCub+ZONGfmcDcY5gEMnbSCGcQEvipA3dBIIFklgnxM00jmcJ1Ojo1+MYynpl - E1XLOaolRWinlDNXA62k8iWG33hcxHGSzkHrsQjtqrrD2PdHS1RmTJ8Hn+iuRUn6 - /fGk8ZQJ7oMPsZNyfiM0OdwSXxJ4rQUtGkHHd727S4K6nXC6OLxXCzl7lYG7QKcP - RVrbFMNv01aToyNGhLmcSxUYdQ4oc+nv65rNZDsdbi34T+dlULboJDkwV6JrJ5dz - hlu3ySgijZuRD5bfpfKB2RScu2ixEijOIyk1oXBB2Dhyh1ezc3qnAw8xkGr9W2SE - roBuu95mZsIZEtfMS5hxwGyWzSCENnbkSukQhUoIjRXryly7MQgNZ5FMX+f5n3DU - aAEJAhBJcIEidIhFVqDkezzMcofKl3MlXWqkfTUV3vsjz6EpN1FwhpZ3prTexUcM - 9XCx9Wq1kMpjkphWETh2lSAafyIz6R/d4zWV5IWIeDh+USYT9z0Rprp4URka4Wjx - fux0T5xDbgq5 - =eiXM - -----END PGP MESSAGE----- - fp: 8DF31C9F48A24F525FFB1815FC96C52B59328E95 - encrypted_regex: ^(data|stringData)$ - version: 3.7.3 diff --git a/cluster/apps/monitoring/victoria-metrics/helm-release.yaml b/cluster/apps/monitoring/victoria-metrics/helm-release.yaml new file mode 100644 index 0000000..bfec8d3 --- /dev/null +++ b/cluster/apps/monitoring/victoria-metrics/helm-release.yaml @@ -0,0 +1,53 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: victoria-metrics + namespace: monitoring +spec: + interval: 5m + chart: + spec: + chart: victoria-metrics-single + version: 0.8.59 + sourceRef: + kind: HelmRepository + name: victoria-metrics-charts + namespace: flux-system + + values: + server: + retentionPeriod: 6 # months + fullnameOverride: victoria-metrics-server + persistentVolume: + enabled: true + storageClass: "hostpath" + matchLabels: + app.kubernetes.io/name: victoria-metrics-pv + size: 32Gi + +# resources: +# limits: +# cpu: 500m +# memory: 512Mi +# requests: +# cpu: 500m +# memory: 512Mi + + serviceMonitor: # TODO + # -- Enable deployment of Service Monitor for server component. This is Prometheus operator object + enabled: false + # -- Service Monitor labels + extraLabels: {} + # -- Service Monitor annotations + annotations: {} + # -- Commented. Prometheus scrape interval for server component + interval: 15s + # -- Commented. Prometheus pre-scrape timeout for server component + scrapeTimeout: 5s + # -- Commented. HTTP scheme to use for scraping. + scheme: https + # -- Commented. TLS configuration to use when scraping the endpoint + tlsConfig: + insecureSkipVerify: true + # -- Service Monitor relabelings + relabelings: [] diff --git a/cluster/apps/monitoring/victoria-metrics/helm-repository.yaml b/cluster/apps/monitoring/victoria-metrics/helm-repository.yaml new file mode 100644 index 0000000..5f18397 --- /dev/null +++ b/cluster/apps/monitoring/victoria-metrics/helm-repository.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: victoria-metrics-charts + namespace: flux-system +spec: + interval: 1m + url: https://victoriametrics.github.io/helm-charts \ No newline at end of file diff --git a/cluster/apps/monitoring/mimir/kustomization.yaml b/cluster/apps/monitoring/victoria-metrics/kustomization.yaml similarity index 52% rename from cluster/apps/monitoring/mimir/kustomization.yaml rename to cluster/apps/monitoring/victoria-metrics/kustomization.yaml index 67f50b9..3f10e2a 100644 --- a/cluster/apps/monitoring/mimir/kustomization.yaml +++ b/cluster/apps/monitoring/victoria-metrics/kustomization.yaml @@ -1,5 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ./mimir.sops.yaml +- ./victoria-pv.yaml +- ./helm-repository.yaml - ./helm-release.yaml \ No newline at end of file diff --git a/cluster/apps/monitoring/victoria-metrics/victoria-pv.yaml b/cluster/apps/monitoring/victoria-metrics/victoria-pv.yaml new file mode 100644 index 0000000..d233828 --- /dev/null +++ b/cluster/apps/monitoring/victoria-metrics/victoria-pv.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: victoria-pv + namespace: monitoring + labels: + app.kubernetes.io/name: victoria-metrics-pv +spec: + storageClassName: hostpath + persistentVolumeReclaimPolicy: Retain + capacity: + storage: 32Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/mnt/MainPool/Kubernetes/victoria-metrics" \ No newline at end of file diff --git a/cluster/apps/monitoring/zfs-exporter/dashboard.yaml b/cluster/apps/monitoring/zfs-exporter/dashboard.yaml new file mode 100644 index 0000000..ae09d1d --- /dev/null +++ b/cluster/apps/monitoring/zfs-exporter/dashboard.yaml @@ -0,0 +1,1330 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: zfs-exporter-grafana-dashboard + namespace: monitoring + labels: + grafana_dashboard: "1" +data: + zfs-exporter.json: |- + { + "__inputs": [ + { + "name": "DS_VICTORIA", + "label": "Victoria", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.4.7" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "$viewingPool", + "mappings": [], + "max": 21770000000000, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 80 + }, + { + "color": "semi-dark-red", + "value": 90 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 19, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "editorMode": "code", + "expr": "zfs_capacity{device_name=\"$viewingPool\"} - zfs_available{device_name=\"$viewingPool\"}", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Used Disk Space", + "transformations": [], + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(zfs_read_errors{device_type=\"pool\",device_name=~\"$viewingPool\"}) by (device_name)", + "format": "time_series", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Pool Read Errors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 6, + "y": 0 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "editorMode": "builder", + "expr": "zfs_write_errors{device_name=\"$viewingPool\", device_type=\"pool\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Pool Write Errors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 9, + "y": 0 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "editorMode": "builder", + "expr": "zfs_checksum_errors{device_name=\"$viewingPool\", device_type=\"pool\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Pool Checksum Errors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Unhealthy" + }, + "1": { + "color": "green", + "index": 0, + "text": "Healthy" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "min(zfs_health{pool=\"$viewingPool\", device_name=\"$viewingPool\", state=\"online\"})", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Pool Health", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 6 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "editorMode": "builder", + "expr": "zfs_read_bandwidth{device_type=\"pool\", device_name=\"$viewingPool\"}", + "legendFormat": "{{device_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pool Read Bandwidth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 6 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "editorMode": "builder", + "expr": "zfs_write_bandwidth{device_type=\"pool\", device_name=\"$viewingPool\"}", + "legendFormat": "{{device_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pool Write Bandwidth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "state" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "available": { + "color": "blue", + "index": 4 + }, + "degraded": { + "color": "orange", + "index": 1 + }, + "online": { + "color": "green", + "index": 0 + }, + "removed": { + "color": "transparent", + "index": 3 + }, + "unavailable": { + "color": "transparent", + "index": 2 + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pool" + }, + "properties": [ + { + "id": "custom.width", + "value": 118 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 2, + "options": { + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zfs_health{device_type=\"disk\", pool=\"$viewingPool\"}", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Disk Status", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "equal", + "options": { + "value": 1 + } + }, + "fieldName": "Value" + } + ], + "match": "any", + "type": "include" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "container": true, + "device_type": true, + "endpoint": true, + "field_type": true, + "instance": true, + "job": true, + "namespace": true, + "pod": true, + "pool": false, + "service": true, + "vdev": true + }, + "indexByName": { + "Time": 0, + "Value": 10, + "__name__": 1, + "device_name": 3, + "device_type": 4, + "field_type": 5, + "instance": 6, + "job": 7, + "pool": 2, + "state": 8, + "vdev": 9 + }, + "renameByName": { + "State": "St", + "device_name": "Disk Name", + "pool": "Pool", + "state": "State" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 14 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "editorMode": "builder", + "expr": "zfs_read_bandwidth{pool=\"$viewingPool\", device_type=\"disk\"}", + "legendFormat": "{{device_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Read Bandwidth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 14 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "editorMode": "builder", + "expr": "zfs_write_bandwidth{pool=\"$viewingPool\", device_type=\"disk\"}", + "legendFormat": "{{device_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Write Bandwidth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 12, + "y": 14 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zfs_read_errors{device_type=\"disk\", pool=\"$viewingPool\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{device_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Per drive read errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 16, + "y": 14 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zfs_write_errors{device_type=\"disk\", pool=\"$viewingPool\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{device_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Per drive write errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 14 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zfs_checksum_errors{device_type=\"disk\", pool=\"$viewingPool\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{device_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Per drive check sum errors", + "type": "timeseries" + } + ], + "refresh": "30s", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_VICTORIA}" + }, + "definition": "label_values(zfs_health{device_type=\"pool\"}, pool)", + "hide": 0, + "includeAll": true, + "label": "ZFS Pool", + "multi": false, + "name": "viewingPool", + "options": [], + "query": { + "query": "label_values(zfs_health{device_type=\"pool\"}, pool)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "ZFS Dashboard", + "uid": "2GaZi3hVk", + "version": 2, + "weekStart": "" + } \ No newline at end of file diff --git a/cluster/apps/monitoring/zfs-exporter/kustomization.yaml b/cluster/apps/monitoring/zfs-exporter/kustomization.yaml index ea3145d..951b965 100644 --- a/cluster/apps/monitoring/zfs-exporter/kustomization.yaml +++ b/cluster/apps/monitoring/zfs-exporter/kustomization.yaml @@ -1,4 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ./helm-release.yaml \ No newline at end of file +- ./helm-release.yaml +- ./dashboard.yaml \ No newline at end of file diff --git a/docs/todo.md b/docs/todo.md index af34fcb..5f9dda7 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -15,6 +15,9 @@ TODO: - [ ] Grafana Loki - [ ] uptime-kuma - [x] gotify + - [ ] Services to monitor + - [ ] Authentik + - [ ] - [x] IRC - [ ] Harbor - [x] CDN (nginx filelist)