From bb16a89a255198db31713df2cb8b89049dc5212a Mon Sep 17 00:00:00 2001 From: SeanOMik Date: Fri, 27 Sep 2024 16:00:51 -0400 Subject: [PATCH] feat: add grafana and kube-prometheus-stack for kubernetes and node exporters --- kubernetes/thin/apps/cilium/bgp.yaml | 54 ------ kubernetes/thin/apps/cilium/main-ip-pool.yaml | 8 - .../monitoring/grafana/app/helm-release.yaml | 168 ++++++++++++++++++ .../monitoring/grafana/app/kustomization.yaml | 5 + .../monitoring/grafana/app/secret.sops.yaml | 72 ++++++++ .../thin/apps/monitoring/grafana/ks.yaml | 30 ++++ .../app/helm-release.yaml | 85 +++++++++ .../app}/kustomization.yaml | 3 +- .../monitoring/kube-prometheus-stack/ks.yaml | 29 +++ .../thin/apps/monitoring/kustomization.yaml | 4 +- 10 files changed, 393 insertions(+), 65 deletions(-) delete mode 100644 kubernetes/thin/apps/cilium/bgp.yaml delete mode 100644 kubernetes/thin/apps/cilium/main-ip-pool.yaml create mode 100644 kubernetes/thin/apps/monitoring/grafana/app/helm-release.yaml create mode 100644 kubernetes/thin/apps/monitoring/grafana/app/kustomization.yaml create mode 100644 kubernetes/thin/apps/monitoring/grafana/app/secret.sops.yaml create mode 100644 kubernetes/thin/apps/monitoring/grafana/ks.yaml create mode 100644 kubernetes/thin/apps/monitoring/kube-prometheus-stack/app/helm-release.yaml rename kubernetes/thin/apps/{cilium => monitoring/kube-prometheus-stack/app}/kustomization.yaml (68%) create mode 100644 kubernetes/thin/apps/monitoring/kube-prometheus-stack/ks.yaml diff --git a/kubernetes/thin/apps/cilium/bgp.yaml b/kubernetes/thin/apps/cilium/bgp.yaml deleted file mode 100644 index 4bc9b5c..0000000 --- a/kubernetes/thin/apps/cilium/bgp.yaml +++ /dev/null @@ -1,54 +0,0 @@ -apiVersion: cilium.io/v2alpha1 -kind: CiliumBGPClusterConfig -metadata: - name: cilium-bgp -spec: - nodeSelector: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: DoesNotExist - bgpInstances: - - name: "bgp-public" - localASN: 65552 - peers: - - name: "router" - peerASN: 65551 - peerAddress: 192.168.1.1 - peerConfigRef: - name: "cilium-peer-router" ---- -apiVersion: cilium.io/v2alpha1 -kind: CiliumBGPPeerConfig -metadata: - name: cilium-peer-router -spec: - timers: - holdTimeSeconds: 9 - keepAliveTimeSeconds: 3 - gracefulRestart: - enabled: true - restartTimeSeconds: 15 - families: - - afi: ipv4 - safi: unicast - advertisements: - matchLabels: - advertise: "bgp-public" ---- -apiVersion: cilium.io/v2alpha1 -kind: CiliumBGPAdvertisement -metadata: - name: bgp-public-ad - labels: - advertise: "bgp-public" -spec: - advertisements: - - advertisementType: "Service" - service: - addresses: - #- ClusterIP - - ExternalIP - - LoadBalancerIP - selector: - matchLabels: - bgp/service-type: public \ No newline at end of file diff --git a/kubernetes/thin/apps/cilium/main-ip-pool.yaml b/kubernetes/thin/apps/cilium/main-ip-pool.yaml deleted file mode 100644 index 2b9ae95..0000000 --- a/kubernetes/thin/apps/cilium/main-ip-pool.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: "cilium.io/v2alpha1" -kind: CiliumLoadBalancerIPPool -metadata: - name: "main-pool" -spec: - blocks: - - start: "192.168.2.50" - stop: "192.168.2.59" \ No newline at end of file diff --git a/kubernetes/thin/apps/monitoring/grafana/app/helm-release.yaml b/kubernetes/thin/apps/monitoring/grafana/app/helm-release.yaml new file mode 100644 index 0000000..03e335e --- /dev/null +++ b/kubernetes/thin/apps/monitoring/grafana/app/helm-release.yaml @@ -0,0 +1,168 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: grafana + namespace: monitoring +spec: + interval: 5m + chart: + spec: + chart: grafana + version: "8.5.1" + sourceRef: + kind: HelmRepository + name: grafana + namespace: flux-system + values: + ingress: + enabled: true + annotations: + cert-manager.io/cluster-issuer: letsencrypt-production + traefik.ingress.kubernetes.io/router.entrypoints: websecure + className: internal + hosts: + - &grafana-host grafana.internal.${SECRET_NEW_DOMAIN} + path: "/" + tls: + - hosts: + - *grafana-host + + grafana.ini: + server: + root_url: https://grafana.internal.${SECRET_NEW_DOMAIN} + + auth: + disable_login_form: true + oauth_auto_login: true + + auth.generic_oauth: + enabled: true + allow_sign_up: true # creates new users after authentik login + auto_login: true + name: Authentik + client_id: $__file{/etc/secrets/auth_generic_oauth/client_id} + client_secret: $__file{/etc/secrets/auth_generic_oauth/client_secret} + scopes: openid profile email offline_access + auth_url: https://auth.${SECRET_BASE_DOMAIN}/application/o/authorize/ + token_url: https://auth.${SECRET_BASE_DOMAIN}/application/o/token/ + api_url: https://auth.${SECRET_BASE_DOMAIN}/application/o/userinfo/ + role_attribute_path: contains(groups[*], 'authentik Admins') && 'GrafanaAdmin' #|| contains(info.groups[*], 'editor') && 'Editor' || 'Viewer' + groups_attribute_path: groups + name_attribute_path: preferred_username + login_attribute_path: email + allow_assign_grafana_admin: true + use_pkce: true + use_refresh_token: true + + # Provide oauth creds + extraSecretMounts: + - name: grafana-secrets-mount + secretName: grafana-oauth + defaultMode: 0440 + mountPath: /etc/secrets/auth_generic_oauth + readOnly: true + + # Add Victoria Metrics as the default datasource + datasources: + victoria.yaml: + apiVersion: 1 + datasources: + - name: Victoria + type: prometheus + jsonData: + tlsSkipVerify: true + url: http://vmsingle-primary.monitoring.svc:8429 + editable: false + isDefault: true + +# datasources: +# - name: Victoria +# uid: victoria-metrics-server +# type: prometheus +# jsonData: +# tlsSkipVerify: "true" +# editable: false" +# url: http://victoria-metrics-server.monitoring.svc:8428 +# version: "1" +# isDefault: "true" + + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + labelValue: "1" + folderAnnotation: grafana_folder + provider: + foldersFromFilesStructure: true + + serviceMonitor: + enabled: true + + dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: default + orgId: 1 + folder: "" + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/default + - name: kubernetes + orgId: 1 + folder: Kubernetes + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/kubernetes + + dashboards: + default: + node-exporter-full: + # renovate: depName="Node Exporter Full" + gnetId: 1860 + revision: 33 + datasource: Victoria + cert-manager: + url: https://raw.githubusercontent.com/monitoring-mixins/website/master/assets/cert-manager/dashboards/cert-manager.json + datasource: Victoria +# minio: +# # renovate: depName="MinIO Dashboard" +# gnetId: 13502 +# revision: 24 +# datasource: +# - { name: DS_PROMETHEUS, value: Victoria } + kubernetes: + kubernetes-api-server: + # renovate: depName="Kubernetes / System / API Server" + gnetId: 15761 + revision: 16 + datasource: Victoria + kubernetes-coredns: + # renovate: depName="Kubernetes / System / CoreDNS" + gnetId: 15762 + revision: 17 + datasource: Victoria + kubernetes-global: + # renovate: depName="Kubernetes / Views / Global" + gnetId: 15757 + revision: 36 + datasource: Victoria + kubernetes-namespaces: + # renovate: depName="Kubernetes / Views / Namespaces" + gnetId: 15758 + revision: 32 + datasource: Victoria + kubernetes-nodes: + # renovate: depName="Kubernetes / Views / Nodes" + gnetId: 15759 + revision: 28 + datasource: Victoria + kubernetes-pods: + # renovate: depName="Kubernetes / Views / Pods" + gnetId: 15760 + revision: 21 + datasource: Prometheus \ No newline at end of file diff --git a/kubernetes/thin/apps/monitoring/grafana/app/kustomization.yaml b/kubernetes/thin/apps/monitoring/grafana/app/kustomization.yaml new file mode 100644 index 0000000..63c59f7 --- /dev/null +++ b/kubernetes/thin/apps/monitoring/grafana/app/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ./secret.sops.yaml +- ./helm-release.yaml \ No newline at end of file diff --git a/kubernetes/thin/apps/monitoring/grafana/app/secret.sops.yaml b/kubernetes/thin/apps/monitoring/grafana/app/secret.sops.yaml new file mode 100644 index 0000000..8c66ab8 --- /dev/null +++ b/kubernetes/thin/apps/monitoring/grafana/app/secret.sops.yaml @@ -0,0 +1,72 @@ +apiVersion: v1 +kind: Secret +metadata: + name: grafana-oauth + namespace: monitoring +stringData: + client_id: ENC[AES256_GCM,data:bmrhYD+cLeKC7sPsGYlLSXBDkqr6capZwP3CY/2T2ca9su46TnrZPg==,iv:BC7SS6B390c2ZazTpQVLuzo93YqNYMneTox0kjOIsfs=,tag:um7jM/endC6e86NfJsXpow==,type:str] + client_secret: ENC[AES256_GCM,data:H/nqW0f8tE8MfQ1wtloU7XWOmBnjImcXyP4GJVO7KJ6Q5JrUZbFAcS1khqe6nBUJCf+/9M+8BEhxPhFR5/IiuzV7xk7a71jw4jMEZs4YizkpUhw4JZ5+YhGcpcGFqB1egsd4sQLWIdRpYy2E45EiuUvldNs2qvBtYrCO2o2MnL8=,iv:UEhm5zywHwkTQO12qg0/tOkfqpOndlKIy75cydMcmvA=,tag:EkzTHSXKYpEutGUa7lZQ5Q==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2024-09-27T19:58:26Z" + mac: ENC[AES256_GCM,data:ftw1bowXQNeyQy55BugSr31VakDVp9aGnq8nNBIvpdMxS0vd/l2DVjhCy9BF/iWQU7T1PR4TErEDCfeJaMpXOt8HzDlDy7IxIuHSSMHK9rFR/L+CwWu0aIwxGB8RpcWlt5JDCWkuk3nCtv61jV6KuZ7fFX/8YXuIVx6Ev6HW7yI=,iv:t5cuWCr53rO7g6BiyLoISmQNE+TBPpdaBebY4Jy+7tQ=,tag:CQEPser8jQbcMlbXN82ANg==,type:str] + pgp: + - created_at: "2024-09-27T19:58:26Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAwAAAAAAAAAAAQ/+PDDA2u/3tQA5D7RBGPBHcKCuraO13Ogb32gxt5ZvrBgA + SpABi32A/zZ9FB8kg+T/VWHXh1NK7aDB8TvAQN6W9l/d+OIRvvqd3EdZLax9EhXu + DFSOXP0ZnqzMSLILP6e7ahEHdlvno+gaCAGJnMyPjNYhvd5dFiVKZYufsYIVXUq4 + 6UsFxWTzs5cDtphZfuGNgZeHWxyalbrl+XagIY0KX9+pFN5muufo3tnSbqjDOui/ + UAhpEE2V/6gZPXJLWggSfKQq/QTO24bxl5Csbb1GIOnLnz48eVpU5oZpiwXWbZo8 + mTS9QKzs/Z+rDooTLE/R73agxEq1EvfGOXlHg2RqV7wBPu3PoHWmId3mM+PykvDD + HyzLzIHtzd1d7kDqH4j/SMi3xP3vZN3H7P2SVBoPX9d0lPmNtILX0N1Rh+sTrF9A + sRSOmHNw8eNLoQojF+VcglA9LiQSuzUn1XKu/QjLkHYMZeRNXfF78XZNj/X6CNf5 + U5i8AuhIZTlRRYKFD+Yw5zMu/UNe0R04z/sBpn7Kdaf8vszihRB/0DLXi33bCltG + 788DZVc3Sx9lFqb9avq6c79Fg6GxIM/FIkpiRm9inC/06c1TwqeZI8VI+s4RwDCb + yVBleNKBAB9hDaO1U/yEOwm9WH8TBnh3sOBwmMaRiTL5z3/PjFFuhXMiVqEAY6+F + AgwDAAAAAAAAAAABD/0QueC44PlfxiFEu+3bj4Lwe3RD03FxEealrb/eZsN/eOZR + gKCZzk3TXXg/2Uc4S7p3ebiNCgs+9MYlN5PvxcheNtWv2rz/n3EdVeZIqZ/YDnv4 + C/eWgyp0YFVthcZtm/ZAetzK17TFG44vpayh+v0GlsLTzdgnhuwDSARcHOZgiCC0 + vcerM9+r+l3UZ+xmhnphoWAaYBfeRHFSGUZUjjQnmBjheB5/ru/mltPMrTX6nlEM + Ngw8cgOj9WL2JNfGIutE7nfDB9Lapyng8eFL2IlAMralFgq6psO2ZdeApLdTy5zV + 38+zYbGRQLTACfKVej6dRdH0dHSxzP0uJe2GVXZE6V7C8CfrDTPDrA9fwdpJLWmR + t148KzFUMQ5twva9CBzfrGNS4Fs41EV1BjhwqlhqWNjPrr/BnK45wKrmPbna+tyJ + EUsu1Ztrxn/J055ugN7vaDuRvSjdSsSyTbhdZPFhgk95Btn8uqNgpp/bk7l/i01w + e/u6qqzqbNez4EpNLReqfSbI6dMxsXqgY1a8WTbGtgzp7d0sZAFzMGdT4aUdAkV4 + SWkexxsZiIqxv26reBtdfq7aKGyg+hQuOVe7D20cRjjuB9Enbbs+OHGbITpt15Oj + X1I2UgfOW1fzMHLmeg1qIF7q2Kd4vLyQ3BiBQPClhRdq+MdZPLPl2bKSFD0ZntRm + AQkCEHAlUhkXndiJatN+uW5/N5xZ9GQ7FEszGVwbdGMlF60VGPfkXqDC45J42wLL + 0zHM02DFd0T8stj3SD0PRJVQWJXwE2qrHdzO4MPBqb0W2vjO+9xBdj0uVYq8vcHI + JaVyt6TK + =3Jlu + -----END PGP MESSAGE----- + fp: BD1AAF9D8170F4BEE437365FF6F0933799CFEBCD + - created_at: "2024-09-27T19:58:26Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAwAAAAAAAAAAAQ//Wu7UO5G0QcWMKZ+0vwEiMSW5HAuUoGJkbrthRtw6yzuu + XCBEbQKZRhuAm3BS7dx0LXYYiE7bEQX4s5ZnqwuwWpRzNCzbfOSxSgibpCqPMocM + RrRKHDi+yma96PY3Uruj3X8kWlL3I+8N9R1Fd7UiTqnm4pti/DaKfeWLIp8by6FE + E2N+BY58GZo8NnkHM1Muud8JaecH8CJxtkHEpFjT/EMu+AgUacmM/OYpRBJyR8Ma + r1gc64Nem2PTwevQj79iGuM3OJWcybeWUh3ssl8QypdOLiMG1ywUCfuwJ4Qqo81q + G+zSbY8AqfaX7ykx329vaWtz3GZrqQAeRSPtVpDd3lFdG4dGTvOUW89OTDvbBgXa + Hi38y18N3gBMspCe+AlH7a2WT8zz1GegMLvaIL7sFHWcmPfBj57wAyoosi85Wy83 + adkgOB7DNO/LMX11XWPha+5Pab1ofqn0XUN+xklfkZ1766pMWCnBCKpK6LrAvxWk + 6kChg8lW+flq6T3MtoloQmWiKY875pd7ZuEihAthgCDjdbOoPV24dPkj0ICFMATN + pZVXd0TzT5efhIb16oxe/+OPdphNIkKl0KCdM4/sSiG8h+FKKjPYNRNsrkPdx/gl + uCIxA7AF0lH3tvi4yUIgZEd+23RTxNd+5IzMVFwntG7Kj9IeCQ0N5fIeoz4BvB7U + ZgEJAhBCtL8btKWlS9/RUTUDBUhdHZ7lxwQFnM2f6ki4RqO5cbb9QphNriEAxTvm + BXhne3sMAVi9KKf/ZsmJ5EOvBicf4qQeN2NfIGJ+HxiQAIyHj9PhVJNJK5OP+t+Q + HnIbXMw9fQ== + =Ls8l + -----END PGP MESSAGE----- + fp: 687802D4DFD8AA82EA55666CF7DADAC782D7663D + encrypted_regex: ^(data|stringData)$ + version: 3.9.0 diff --git a/kubernetes/thin/apps/monitoring/grafana/ks.yaml b/kubernetes/thin/apps/monitoring/grafana/ks.yaml new file mode 100644 index 0000000..4440027 --- /dev/null +++ b/kubernetes/thin/apps/monitoring/grafana/ks.yaml @@ -0,0 +1,30 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: grafana + namespace: flux-system +spec: + targetNamespace: monitoring + timeout: 5m + interval: 10m + path: ./kubernetes/thin/apps/monitoring/grafana/app + prune: true + sourceRef: + kind: GitRepository + name: home-cluster + decryption: + provider: sops + secretRef: + name: sops-gpg + dependsOn: + - name: victoria-metrics-deploys + - name: kube-prometheus-stack + postBuild: + substitute: {} + substituteFrom: + - kind: ConfigMap + name: cluster-settings + - kind: Secret + name: cluster-secrets diff --git a/kubernetes/thin/apps/monitoring/kube-prometheus-stack/app/helm-release.yaml b/kubernetes/thin/apps/monitoring/kube-prometheus-stack/app/helm-release.yaml new file mode 100644 index 0000000..6e7623d --- /dev/null +++ b/kubernetes/thin/apps/monitoring/kube-prometheus-stack/app/helm-release.yaml @@ -0,0 +1,85 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: kube-prometheus-stack + namespace: monitoring +spec: + interval: 5m + chart: + spec: + chart: kube-prometheus-stack + version: "62.7.0" + sourceRef: + kind: HelmRepository + name: prometheus-community + namespace: flux-system + + values: + alertmanager: + enabled: false + + grafana: + enabled: false + # enabled dashboards even with grafana being disabled + #forceDeployDashboards: true + #defaultDashboardsTimezone: America/New_York + + prometheus: + enabled: false + + defaultRule: + create: false + rules: + general: false + + kubelet: + enabled: true + serviceMonitor: + metricRelabelings: + # Remove duplicate metrics + - sourceLabels: ["__name__"] + regex: "(apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|authentication_token|cadvisor_version|container_blkio|container_cpu|container_fs|container_last|container_memory|container_network|container_oom|container_processes|container|csi_operations|disabled_metric|get_token|go|hidden_metric|kubelet_certificate|kubelet_cgroup|kubelet_container|kubelet_containers|kubelet_cpu|kubelet_device|kubelet_graceful|kubelet_http|kubelet_lifecycle|kubelet_managed|kubelet_node|kubelet_pleg|kubelet_pod|kubelet_run|kubelet_running|kubelet_runtime|kubelet_server|kubelet_started|kubelet_volume|kubernetes_build|kubernetes_feature|machine_cpu|machine_memory|machine_nvm|machine_scrape|node_namespace|plugin_manager|prober_probe|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scrape_duration|scrape_samples|scrape_series|storage_operation|volume_manager|volume_operation|workqueue)_(.+)" + action: keep + - sourceLabels: ["node"] + targetLabel: instance + action: replace + kubeApiServer: + serviceMonitor: + selector: + k8s-app: kube-apiserver + kubeScheduler: + service: + selector: + k8s-app: kube-scheduler + kubeControllerManager: &kubeControllerManager + service: + selector: + k8s-app: kube-controller-manager + kubeEtcd: + <<: *kubeControllerManager # etcd runs on control plane nodes + prometheus-node-exporter: + fullnameOverride: node-exporter + prometheus: + monitor: + enabled: true + relabelings: + - action: replace + regex: (.*) + replacement: $1 + sourceLabels: ["__meta_kubernetes_pod_node_name"] + targetLabel: kubernetes_node + kube-state-metrics: + fullnameOverride: kube-state-metrics + metricLabelsAllowlist: + - pods=[*] + - deployments=[*] + - persistentvolumeclaims=[*] + prometheus: + monitor: + enabled: true + relabelings: + - action: replace + regex: (.*) + replacement: $1 + sourceLabels: ["__meta_kubernetes_pod_node_name"] + targetLabel: kubernetes_node \ No newline at end of file diff --git a/kubernetes/thin/apps/cilium/kustomization.yaml b/kubernetes/thin/apps/monitoring/kube-prometheus-stack/app/kustomization.yaml similarity index 68% rename from kubernetes/thin/apps/cilium/kustomization.yaml rename to kubernetes/thin/apps/monitoring/kube-prometheus-stack/app/kustomization.yaml index b436c3f..ea3145d 100644 --- a/kubernetes/thin/apps/cilium/kustomization.yaml +++ b/kubernetes/thin/apps/monitoring/kube-prometheus-stack/app/kustomization.yaml @@ -1,5 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ./main-ip-pool.yaml -- ./bgp.yaml \ No newline at end of file +- ./helm-release.yaml \ No newline at end of file diff --git a/kubernetes/thin/apps/monitoring/kube-prometheus-stack/ks.yaml b/kubernetes/thin/apps/monitoring/kube-prometheus-stack/ks.yaml new file mode 100644 index 0000000..f60c285 --- /dev/null +++ b/kubernetes/thin/apps/monitoring/kube-prometheus-stack/ks.yaml @@ -0,0 +1,29 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: kube-prometheus-stack + namespace: flux-system +spec: + targetNamespace: monitoring + timeout: 5m + interval: 10m + path: ./kubernetes/thin/apps/monitoring/kube-prometheus-stack/app + prune: true + sourceRef: + kind: GitRepository + name: home-cluster + decryption: + provider: sops + secretRef: + name: sops-gpg + dependsOn: + - name: victoria-metrics-deploys + postBuild: + substitute: {} + substituteFrom: + - kind: ConfigMap + name: cluster-settings + - kind: Secret + name: cluster-secrets diff --git a/kubernetes/thin/apps/monitoring/kustomization.yaml b/kubernetes/thin/apps/monitoring/kustomization.yaml index 016c50a..4caa6d0 100644 --- a/kubernetes/thin/apps/monitoring/kustomization.yaml +++ b/kubernetes/thin/apps/monitoring/kustomization.yaml @@ -1,4 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ./victoria-metrics/ks.yaml \ No newline at end of file +- ./victoria-metrics/ks.yaml +- ./kube-prometheus-stack/ks.yaml +- ./grafana/ks.yaml \ No newline at end of file