From 82cfa8f79e47dd1b3afd3f22e1c3df70db8b96a5 Mon Sep 17 00:00:00 2001
From: SeanOMik <seanomik@gmail.com>
Date: Fri, 20 Oct 2023 21:03:39 -0400
Subject: [PATCH] fix(victoria-metrics): update grafana dashboard

---
 .../victoria-metrics/dashboard.yaml           | 1043 ++++++++++-------
 1 file changed, 647 insertions(+), 396 deletions(-)

diff --git a/cluster/apps/monitoring/victoria-metrics/dashboard.yaml b/cluster/apps/monitoring/victoria-metrics/dashboard.yaml
index 1fb03c1..aedd3e9 100644
--- a/cluster/apps/monitoring/victoria-metrics/dashboard.yaml
+++ b/cluster/apps/monitoring/victoria-metrics/dashboard.yaml
@@ -15,7 +15,7 @@ data:
           "type": "grafana",
           "id": "grafana",
           "name": "Grafana",
-          "version": "9.4.7"
+          "version": "10.1.2"
         },
         {
           "type": "datasource",
@@ -98,7 +98,7 @@ data:
       "editable": true,
       "fiscalYearStartMonth": 0,
       "gnetId": 10229,
-      "graphTooltip": 0,
+      "graphTooltip": 1,
       "id": null,
       "links": [
         {
@@ -177,7 +177,7 @@ data:
             "content": "<div style=\"text-align: center;\">$version</div>",
             "mode": "markdown"
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "10.1.2",
           "targets": [
             {
               "datasource": {
@@ -239,7 +239,7 @@ data:
             "text": {},
             "textMode": "auto"
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "10.1.2",
           "targets": [
             {
               "datasource": {
@@ -308,7 +308,7 @@ data:
             "text": {},
             "textMode": "auto"
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "10.1.2",
           "targets": [
             {
               "datasource": {
@@ -378,7 +378,7 @@ data:
             "text": {},
             "textMode": "auto"
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "10.1.2",
           "targets": [
             {
               "datasource": {
@@ -452,7 +452,7 @@ data:
             "text": {},
             "textMode": "auto"
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "10.1.2",
           "targets": [
             {
               "datasource": {
@@ -522,7 +522,7 @@ data:
             "text": {},
             "textMode": "auto"
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "10.1.2",
           "targets": [
             {
               "datasource": {
@@ -590,7 +590,7 @@ data:
             "text": {},
             "textMode": "auto"
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "10.1.2",
           "targets": [
             {
               "datasource": {
@@ -659,7 +659,7 @@ data:
             "text": {},
             "textMode": "auto"
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "10.1.2",
           "targets": [
             {
               "datasource": {
@@ -729,7 +729,7 @@ data:
             "text": {},
             "textMode": "auto"
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "10.1.2",
           "targets": [
             {
               "datasource": {
@@ -799,7 +799,7 @@ data:
             "text": {},
             "textMode": "auto"
           },
-          "pluginVersion": "9.4.7",
+          "pluginVersion": "10.1.2",
           "targets": [
             {
               "datasource": {
@@ -869,6 +869,7 @@ data:
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -974,6 +975,7 @@ data:
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -1081,6 +1083,7 @@ data:
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -1189,6 +1192,7 @@ data:
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -1463,9 +1467,9 @@ data:
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, level) > 0",
+              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (level, location) > 0",
               "interval": "5m",
-              "legendFormat": "{{job}} - {{level}}",
+              "legendFormat": "{{level}}: {{location}}",
               "range": true,
               "refId": "A"
             }
@@ -1582,7 +1586,7 @@ data:
                   },
                   "editorMode": "code",
                   "exemplar": false,
-                  "expr": "max(\n    max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
+                  "expr": "max(\n    max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
                   "interval": "",
                   "legendFormat": "__auto",
                   "range": true,
@@ -1592,112 +1596,6 @@ data:
               "title": "RSS memory % usage ($instance)",
               "type": "timeseries"
             },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "$ds"
-              },
-              "fieldConfig": {
-                "defaults": {
-                  "color": {
-                    "mode": "palette-classic"
-                  },
-                  "custom": {
-                    "axisCenteredZero": false,
-                    "axisColorMode": "text",
-                    "axisLabel": "",
-                    "axisPlacement": "auto",
-                    "barAlignment": 0,
-                    "drawStyle": "line",
-                    "fillOpacity": 0,
-                    "gradientMode": "none",
-                    "hideFrom": {
-                      "legend": false,
-                      "tooltip": false,
-                      "viz": false
-                    },
-                    "lineInterpolation": "linear",
-                    "lineWidth": 1,
-                    "pointSize": 5,
-                    "scaleDistribution": {
-                      "type": "linear"
-                    },
-                    "showPoints": "never",
-                    "spanNulls": false,
-                    "stacking": {
-                      "group": "A",
-                      "mode": "none"
-                    },
-                    "thresholdsStyle": {
-                      "mode": "off"
-                    }
-                  },
-                  "links": [],
-                  "mappings": [],
-                  "min": 0,
-                  "thresholds": {
-                    "mode": "absolute",
-                    "steps": [
-                      {
-                        "color": "green"
-                      },
-                      {
-                        "color": "red",
-                        "value": 80
-                      }
-                    ]
-                  },
-                  "unit": "percentunit"
-                },
-                "overrides": []
-              },
-              "gridPos": {
-                "h": 8,
-                "w": 12,
-                "x": 12,
-                "y": 31
-              },
-              "id": 114,
-              "links": [],
-              "options": {
-                "legend": {
-                  "calcs": [
-                    "mean",
-                    "lastNotNull",
-                    "max"
-                  ],
-                  "displayMode": "table",
-                  "placement": "bottom",
-                  "showLegend": true,
-                  "sortBy": "Last *",
-                  "sortDesc": true
-                },
-                "tooltip": {
-                  "mode": "multi",
-                  "sort": "none"
-                }
-              },
-              "pluginVersion": "9.1.0",
-              "targets": [
-                {
-                  "datasource": {
-                    "type": "prometheus",
-                    "uid": "$ds"
-                  },
-                  "editorMode": "code",
-                  "exemplar": false,
-                  "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
-                  "format": "time_series",
-                  "interval": "",
-                  "intervalFactor": 1,
-                  "legendFormat": "__auto",
-                  "range": true,
-                  "refId": "A"
-                }
-              ],
-              "title": "CPU % usage ($instance)",
-              "type": "timeseries"
-            },
             {
               "datasource": {
                 "type": "prometheus",
@@ -1761,8 +1659,8 @@ data:
               "gridPos": {
                 "h": 8,
                 "w": 12,
-                "x": 0,
-                "y": 39
+                "x": 12,
+                "y": 31
               },
               "id": 44,
               "links": [],
@@ -1858,7 +1756,7 @@ data:
                 "type": "prometheus",
                 "uid": "$ds"
               },
-              "description": "",
+              "description": "Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%",
               "fieldConfig": {
                 "defaults": {
                   "color": {
@@ -1909,33 +1807,17 @@ data:
                       }
                     ]
                   },
-                  "unit": "short"
+                  "unit": "percentunit"
                 },
-                "overrides": [
-                  {
-                    "matcher": {
-                      "id": "byName",
-                      "options": "Limit"
-                    },
-                    "properties": [
-                      {
-                        "id": "color",
-                        "value": {
-                          "fixedColor": "#F2495C",
-                          "mode": "fixed"
-                        }
-                      }
-                    ]
-                  }
-                ]
+                "overrides": []
               },
               "gridPos": {
                 "h": 8,
                 "w": 12,
-                "x": 12,
+                "x": 0,
                 "y": 39
               },
-              "id": 57,
+              "id": 123,
               "links": [],
               "options": {
                 "legend": {
@@ -1962,29 +1844,122 @@ data:
                     "type": "prometheus",
                     "uid": "$ds"
                   },
-                  "expr": "rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
-                  "format": "time_series",
+                  "editorMode": "code",
+                  "exemplar": false,
+                  "expr": "max(\n    max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
                   "interval": "",
-                  "intervalFactor": 1,
-                  "legendFormat": "CPU cores used",
+                  "legendFormat": "__auto",
+                  "range": true,
                   "refId": "A"
+                }
+              ],
+              "title": "RSS anonymous memory % usage ($instance)",
+              "type": "timeseries"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "fieldConfig": {
+                "defaults": {
+                  "color": {
+                    "mode": "palette-classic"
+                  },
+                  "custom": {
+                    "axisCenteredZero": false,
+                    "axisColorMode": "text",
+                    "axisLabel": "",
+                    "axisPlacement": "auto",
+                    "barAlignment": 0,
+                    "drawStyle": "line",
+                    "fillOpacity": 0,
+                    "gradientMode": "none",
+                    "hideFrom": {
+                      "legend": false,
+                      "tooltip": false,
+                      "viz": false
+                    },
+                    "lineInterpolation": "linear",
+                    "lineWidth": 1,
+                    "pointSize": 5,
+                    "scaleDistribution": {
+                      "type": "linear"
+                    },
+                    "showPoints": "never",
+                    "spanNulls": false,
+                    "stacking": {
+                      "group": "A",
+                      "mode": "none"
+                    },
+                    "thresholdsStyle": {
+                      "mode": "off"
+                    }
+                  },
+                  "links": [],
+                  "mappings": [],
+                  "min": 0,
+                  "thresholds": {
+                    "mode": "absolute",
+                    "steps": [
+                      {
+                        "color": "green"
+                      },
+                      {
+                        "color": "red",
+                        "value": 80
+                      }
+                    ]
+                  },
+                  "unit": "percentunit"
                 },
+                "overrides": []
+              },
+              "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 39
+              },
+              "id": 114,
+              "links": [],
+              "options": {
+                "legend": {
+                  "calcs": [
+                    "mean",
+                    "lastNotNull",
+                    "max"
+                  ],
+                  "displayMode": "table",
+                  "placement": "bottom",
+                  "showLegend": true,
+                  "sortBy": "Last *",
+                  "sortDesc": true
+                },
+                "tooltip": {
+                  "mode": "multi",
+                  "sort": "none"
+                }
+              },
+              "pluginVersion": "9.1.0",
+              "targets": [
                 {
                   "datasource": {
                     "type": "prometheus",
                     "uid": "$ds"
                   },
+                  "editorMode": "code",
                   "exemplar": false,
-                  "expr": "process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}",
+                  "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
                   "format": "time_series",
-                  "hide": false,
                   "interval": "",
                   "intervalFactor": 1,
-                  "legendFormat": "Limit",
-                  "refId": "B"
+                  "legendFormat": "__auto",
+                  "range": true,
+                  "refId": "A"
                 }
               ],
-              "title": "CPU ($instance)",
+              "title": "CPU % usage ($instance)",
               "type": "timeseries"
             },
             {
@@ -2103,7 +2078,7 @@ data:
                   "hide": false,
                   "interval": "",
                   "intervalFactor": 2,
-                  "legendFormat": "{{job}}",
+                  "legendFormat": "{{instance}}",
                   "range": true,
                   "refId": "A"
                 }
@@ -2111,6 +2086,245 @@ data:
               "title": "Open FDs ($instance)",
               "type": "timeseries"
             },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "description": "",
+              "fieldConfig": {
+                "defaults": {
+                  "color": {
+                    "mode": "palette-classic"
+                  },
+                  "custom": {
+                    "axisCenteredZero": false,
+                    "axisColorMode": "text",
+                    "axisLabel": "",
+                    "axisPlacement": "auto",
+                    "barAlignment": 0,
+                    "drawStyle": "line",
+                    "fillOpacity": 0,
+                    "gradientMode": "none",
+                    "hideFrom": {
+                      "legend": false,
+                      "tooltip": false,
+                      "viz": false
+                    },
+                    "lineInterpolation": "linear",
+                    "lineWidth": 1,
+                    "pointSize": 5,
+                    "scaleDistribution": {
+                      "type": "linear"
+                    },
+                    "showPoints": "never",
+                    "spanNulls": false,
+                    "stacking": {
+                      "group": "A",
+                      "mode": "none"
+                    },
+                    "thresholdsStyle": {
+                      "mode": "off"
+                    }
+                  },
+                  "links": [],
+                  "mappings": [],
+                  "min": 0,
+                  "thresholds": {
+                    "mode": "absolute",
+                    "steps": [
+                      {
+                        "color": "green"
+                      },
+                      {
+                        "color": "red",
+                        "value": 80
+                      }
+                    ]
+                  },
+                  "unit": "short"
+                },
+                "overrides": [
+                  {
+                    "matcher": {
+                      "id": "byName",
+                      "options": "Limit"
+                    },
+                    "properties": [
+                      {
+                        "id": "color",
+                        "value": {
+                          "fixedColor": "#F2495C",
+                          "mode": "fixed"
+                        }
+                      }
+                    ]
+                  }
+                ]
+              },
+              "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 47
+              },
+              "id": 57,
+              "links": [],
+              "options": {
+                "legend": {
+                  "calcs": [
+                    "mean",
+                    "lastNotNull",
+                    "max"
+                  ],
+                  "displayMode": "table",
+                  "placement": "bottom",
+                  "showLegend": true,
+                  "sortBy": "Last *",
+                  "sortDesc": true
+                },
+                "tooltip": {
+                  "mode": "multi",
+                  "sort": "desc"
+                }
+              },
+              "pluginVersion": "9.1.0",
+              "targets": [
+                {
+                  "datasource": {
+                    "type": "prometheus",
+                    "uid": "$ds"
+                  },
+                  "expr": "rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
+                  "format": "time_series",
+                  "interval": "",
+                  "intervalFactor": 1,
+                  "legendFormat": "CPU cores used",
+                  "refId": "A"
+                },
+                {
+                  "datasource": {
+                    "type": "prometheus",
+                    "uid": "$ds"
+                  },
+                  "exemplar": false,
+                  "expr": "process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}",
+                  "format": "time_series",
+                  "hide": false,
+                  "interval": "",
+                  "intervalFactor": 1,
+                  "legendFormat": "Limit",
+                  "refId": "B"
+                }
+              ],
+              "title": "CPU ($instance)",
+              "type": "timeseries"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "fieldConfig": {
+                "defaults": {
+                  "color": {
+                    "mode": "palette-classic"
+                  },
+                  "custom": {
+                    "axisCenteredZero": false,
+                    "axisColorMode": "text",
+                    "axisLabel": "",
+                    "axisPlacement": "auto",
+                    "barAlignment": 0,
+                    "drawStyle": "line",
+                    "fillOpacity": 10,
+                    "gradientMode": "none",
+                    "hideFrom": {
+                      "legend": false,
+                      "tooltip": false,
+                      "viz": false
+                    },
+                    "lineInterpolation": "linear",
+                    "lineWidth": 1,
+                    "pointSize": 5,
+                    "scaleDistribution": {
+                      "type": "linear"
+                    },
+                    "showPoints": "never",
+                    "spanNulls": false,
+                    "stacking": {
+                      "group": "A",
+                      "mode": "none"
+                    },
+                    "thresholdsStyle": {
+                      "mode": "off"
+                    }
+                  },
+                  "decimals": 0,
+                  "links": [],
+                  "mappings": [],
+                  "min": 0,
+                  "thresholds": {
+                    "mode": "absolute",
+                    "steps": [
+                      {
+                        "color": "green"
+                      },
+                      {
+                        "color": "red",
+                        "value": 80
+                      }
+                    ]
+                  },
+                  "unit": "short"
+                },
+                "overrides": []
+              },
+              "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 55
+              },
+              "id": 47,
+              "links": [],
+              "options": {
+                "legend": {
+                  "calcs": [
+                    "mean",
+                    "lastNotNull",
+                    "max"
+                  ],
+                  "displayMode": "table",
+                  "placement": "bottom",
+                  "showLegend": true,
+                  "sortBy": "Last *",
+                  "sortDesc": true
+                },
+                "tooltip": {
+                  "mode": "multi",
+                  "sort": "none"
+                }
+              },
+              "pluginVersion": "9.1.0",
+              "targets": [
+                {
+                  "datasource": {
+                    "type": "prometheus",
+                    "uid": "$ds"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(go_goroutines{job=~\"$job\", instance=~\"$instance\"}) by(instance)",
+                  "format": "time_series",
+                  "intervalFactor": 2,
+                  "legendFormat": "{{instance}}",
+                  "range": true,
+                  "refId": "A"
+                }
+              ],
+              "title": "Goroutines ($instance)",
+              "type": "timeseries"
+            },
             {
               "datasource": {
                 "type": "prometheus",
@@ -2187,7 +2401,7 @@ data:
                 "h": 8,
                 "w": 12,
                 "x": 12,
-                "y": 47
+                "y": 55
               },
               "id": 76,
               "links": [],
@@ -2241,213 +2455,6 @@ data:
               "title": "Disk writes/reads ($instance)",
               "type": "timeseries"
             },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "$ds"
-              },
-              "fieldConfig": {
-                "defaults": {
-                  "color": {
-                    "mode": "palette-classic"
-                  },
-                  "custom": {
-                    "axisCenteredZero": false,
-                    "axisColorMode": "text",
-                    "axisLabel": "",
-                    "axisPlacement": "auto",
-                    "barAlignment": 0,
-                    "drawStyle": "line",
-                    "fillOpacity": 10,
-                    "gradientMode": "none",
-                    "hideFrom": {
-                      "legend": false,
-                      "tooltip": false,
-                      "viz": false
-                    },
-                    "lineInterpolation": "linear",
-                    "lineWidth": 1,
-                    "pointSize": 5,
-                    "scaleDistribution": {
-                      "type": "linear"
-                    },
-                    "showPoints": "never",
-                    "spanNulls": false,
-                    "stacking": {
-                      "group": "A",
-                      "mode": "none"
-                    },
-                    "thresholdsStyle": {
-                      "mode": "off"
-                    }
-                  },
-                  "decimals": 0,
-                  "links": [],
-                  "mappings": [],
-                  "min": 0,
-                  "thresholds": {
-                    "mode": "absolute",
-                    "steps": [
-                      {
-                        "color": "green"
-                      },
-                      {
-                        "color": "red",
-                        "value": 80
-                      }
-                    ]
-                  },
-                  "unit": "short"
-                },
-                "overrides": []
-              },
-              "gridPos": {
-                "h": 8,
-                "w": 12,
-                "x": 0,
-                "y": 55
-              },
-              "id": 47,
-              "links": [],
-              "options": {
-                "legend": {
-                  "calcs": [
-                    "mean",
-                    "lastNotNull",
-                    "max"
-                  ],
-                  "displayMode": "table",
-                  "placement": "bottom",
-                  "showLegend": true,
-                  "sortBy": "Last *",
-                  "sortDesc": true
-                },
-                "tooltip": {
-                  "mode": "multi",
-                  "sort": "none"
-                }
-              },
-              "pluginVersion": "9.1.0",
-              "targets": [
-                {
-                  "datasource": {
-                    "type": "prometheus",
-                    "uid": "$ds"
-                  },
-                  "expr": "sum(go_goroutines{job=~\"$job\", instance=~\"$instance\"})",
-                  "format": "time_series",
-                  "intervalFactor": 2,
-                  "legendFormat": "gc duration",
-                  "refId": "A"
-                }
-              ],
-              "title": "Goroutines ($instance)",
-              "type": "timeseries"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "$ds"
-              },
-              "description": "",
-              "fieldConfig": {
-                "defaults": {
-                  "color": {
-                    "mode": "palette-classic"
-                  },
-                  "custom": {
-                    "axisCenteredZero": false,
-                    "axisColorMode": "text",
-                    "axisLabel": "",
-                    "axisPlacement": "auto",
-                    "barAlignment": 0,
-                    "drawStyle": "line",
-                    "fillOpacity": 10,
-                    "gradientMode": "none",
-                    "hideFrom": {
-                      "legend": false,
-                      "tooltip": false,
-                      "viz": false
-                    },
-                    "lineInterpolation": "linear",
-                    "lineWidth": 1,
-                    "pointSize": 5,
-                    "scaleDistribution": {
-                      "type": "linear"
-                    },
-                    "showPoints": "never",
-                    "spanNulls": false,
-                    "stacking": {
-                      "group": "A",
-                      "mode": "none"
-                    },
-                    "thresholdsStyle": {
-                      "mode": "off"
-                    }
-                  },
-                  "links": [],
-                  "mappings": [],
-                  "min": 0,
-                  "thresholds": {
-                    "mode": "absolute",
-                    "steps": [
-                      {
-                        "color": "green"
-                      },
-                      {
-                        "color": "red",
-                        "value": 80
-                      }
-                    ]
-                  },
-                  "unit": "short"
-                },
-                "overrides": []
-              },
-              "gridPos": {
-                "h": 8,
-                "w": 12,
-                "x": 12,
-                "y": 55
-              },
-              "id": 37,
-              "links": [],
-              "options": {
-                "legend": {
-                  "calcs": [
-                    "mean",
-                    "lastNotNull",
-                    "max"
-                  ],
-                  "displayMode": "table",
-                  "placement": "bottom",
-                  "showLegend": true,
-                  "sortBy": "Last *",
-                  "sortDesc": true
-                },
-                "tooltip": {
-                  "mode": "multi",
-                  "sort": "none"
-                }
-              },
-              "pluginVersion": "9.1.0",
-              "targets": [
-                {
-                  "datasource": {
-                    "type": "prometheus",
-                    "uid": "$ds"
-                  },
-                  "expr": "sum(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"})",
-                  "format": "time_series",
-                  "hide": false,
-                  "intervalFactor": 1,
-                  "legendFormat": "connections",
-                  "refId": "A"
-                }
-              ],
-              "title": "TCP connections ($instance)",
-              "type": "timeseries"
-            },
             {
               "datasource": {
                 "type": "prometheus",
@@ -2541,16 +2548,258 @@ data:
                     "type": "prometheus",
                     "uid": "$ds"
                   },
-                  "expr": "sum(process_num_threads{job=~\"$job\", instance=~\"$instance\"})",
+                  "editorMode": "code",
+                  "expr": "sum(process_num_threads{job=~\"$job\", instance=~\"$instance\"}) by(instance)",
                   "format": "time_series",
                   "intervalFactor": 2,
-                  "legendFormat": "threads",
+                  "legendFormat": "{{instance}}",
+                  "range": true,
                   "refId": "A"
                 }
               ],
               "title": "Threads ($instance)",
               "type": "timeseries"
             },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "description": "Shows the number of read/write syscalls such as read, pread, write, pwrite.",
+              "fieldConfig": {
+                "defaults": {
+                  "color": {
+                    "mode": "palette-classic"
+                  },
+                  "custom": {
+                    "axisCenteredZero": false,
+                    "axisColorMode": "text",
+                    "axisLabel": "",
+                    "axisPlacement": "auto",
+                    "barAlignment": 0,
+                    "drawStyle": "line",
+                    "fillOpacity": 10,
+                    "gradientMode": "none",
+                    "hideFrom": {
+                      "legend": false,
+                      "tooltip": false,
+                      "viz": false
+                    },
+                    "lineInterpolation": "linear",
+                    "lineWidth": 1,
+                    "pointSize": 5,
+                    "scaleDistribution": {
+                      "type": "linear"
+                    },
+                    "showPoints": "never",
+                    "spanNulls": false,
+                    "stacking": {
+                      "group": "A",
+                      "mode": "none"
+                    },
+                    "thresholdsStyle": {
+                      "mode": "off"
+                    }
+                  },
+                  "links": [],
+                  "mappings": [],
+                  "thresholds": {
+                    "mode": "absolute",
+                    "steps": [
+                      {
+                        "color": "green"
+                      },
+                      {
+                        "color": "red",
+                        "value": 80
+                      }
+                    ]
+                  },
+                  "unit": "short"
+                },
+                "overrides": [
+                  {
+                    "matcher": {
+                      "id": "byName",
+                      "options": "read calls"
+                    },
+                    "properties": [
+                      {
+                        "id": "custom.transform",
+                        "value": "negative-Y"
+                      }
+                    ]
+                  }
+                ]
+              },
+              "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 63
+              },
+              "id": 124,
+              "links": [],
+              "options": {
+                "legend": {
+                  "calcs": [
+                    "mean",
+                    "lastNotNull",
+                    "max"
+                  ],
+                  "displayMode": "table",
+                  "placement": "bottom",
+                  "showLegend": true,
+                  "sortBy": "Last *",
+                  "sortDesc": true
+                },
+                "tooltip": {
+                  "mode": "multi",
+                  "sort": "none"
+                }
+              },
+              "pluginVersion": "9.1.0",
+              "targets": [
+                {
+                  "datasource": {
+                    "type": "prometheus",
+                    "uid": "$ds"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(rate(process_io_read_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
+                  "format": "time_series",
+                  "hide": false,
+                  "interval": "",
+                  "intervalFactor": 1,
+                  "legendFormat": "read calls",
+                  "range": true,
+                  "refId": "A"
+                },
+                {
+                  "datasource": {
+                    "type": "prometheus",
+                    "uid": "$ds"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(rate(process_io_write_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
+                  "format": "time_series",
+                  "hide": false,
+                  "interval": "",
+                  "intervalFactor": 1,
+                  "legendFormat": "write calls",
+                  "range": true,
+                  "refId": "B"
+                }
+              ],
+              "title": "Disk write/read calls ($instance)",
+              "type": "timeseries"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "description": "",
+              "fieldConfig": {
+                "defaults": {
+                  "color": {
+                    "mode": "palette-classic"
+                  },
+                  "custom": {
+                    "axisCenteredZero": false,
+                    "axisColorMode": "text",
+                    "axisLabel": "",
+                    "axisPlacement": "auto",
+                    "barAlignment": 0,
+                    "drawStyle": "line",
+                    "fillOpacity": 10,
+                    "gradientMode": "none",
+                    "hideFrom": {
+                      "legend": false,
+                      "tooltip": false,
+                      "viz": false
+                    },
+                    "lineInterpolation": "linear",
+                    "lineWidth": 1,
+                    "pointSize": 5,
+                    "scaleDistribution": {
+                      "type": "linear"
+                    },
+                    "showPoints": "never",
+                    "spanNulls": false,
+                    "stacking": {
+                      "group": "A",
+                      "mode": "none"
+                    },
+                    "thresholdsStyle": {
+                      "mode": "off"
+                    }
+                  },
+                  "links": [],
+                  "mappings": [],
+                  "min": 0,
+                  "thresholds": {
+                    "mode": "absolute",
+                    "steps": [
+                      {
+                        "color": "green"
+                      },
+                      {
+                        "color": "red",
+                        "value": 80
+                      }
+                    ]
+                  },
+                  "unit": "short"
+                },
+                "overrides": []
+              },
+              "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 71
+              },
+              "id": 49,
+              "links": [],
+              "options": {
+                "legend": {
+                  "calcs": [
+                    "mean",
+                    "lastNotNull",
+                    "max"
+                  ],
+                  "displayMode": "table",
+                  "placement": "bottom",
+                  "showLegend": true,
+                  "sortBy": "Last *",
+                  "sortDesc": true
+                },
+                "tooltip": {
+                  "mode": "multi",
+                  "sort": "none"
+                }
+              },
+              "pluginVersion": "9.1.0",
+              "targets": [
+                {
+                  "datasource": {
+                    "type": "prometheus",
+                    "uid": "$ds"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(instance)",
+                  "format": "time_series",
+                  "hide": false,
+                  "intervalFactor": 1,
+                  "legendFormat": "{{instance}}",
+                  "range": true,
+                  "refId": "A"
+                }
+              ],
+              "title": "TCP connections rate ($instance)",
+              "type": "timeseries"
+            },
             {
               "datasource": {
                 "type": "prometheus",
@@ -2615,9 +2864,9 @@ data:
                 "h": 8,
                 "w": 12,
                 "x": 12,
-                "y": 63
+                "y": 71
               },
-              "id": 49,
+              "id": 37,
               "links": [],
               "options": {
                 "legend": {
@@ -2645,16 +2894,16 @@ data:
                     "uid": "$ds"
                   },
                   "editorMode": "code",
-                  "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
+                  "expr": "sum(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"}) by(instance)",
                   "format": "time_series",
                   "hide": false,
                   "intervalFactor": 1,
-                  "legendFormat": "connections",
+                  "legendFormat": "{{instance}}",
                   "range": true,
                   "refId": "A"
                 }
               ],
-              "title": "TCP connections rate ($instance)",
+              "title": "TCP connections ($instance)",
               "type": "timeseries"
             }
           ],
@@ -2748,7 +2997,7 @@ data:
                 "h": 8,
                 "w": 12,
                 "x": 0,
-                "y": 32
+                "y": 8
               },
               "id": 66,
               "options": {
@@ -2800,7 +3049,7 @@ data:
                 "type": "prometheus",
                 "uid": "$ds"
               },
-              "description": "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. \n\nThe less value is better. If percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of active time series. \n\nIn general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on may other factors such as the number of labels per time series and the length of label values.",
+              "description": "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. \n\nThe less value is better. If percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of active time series. \n\nIn general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on many other factors such as the number of labels per time series and the length of label values. See also https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183",
               "fieldConfig": {
                 "defaults": {
                   "color": {
@@ -2860,7 +3109,7 @@ data:
                 "h": 8,
                 "w": 12,
                 "x": 12,
-                "y": 32
+                "y": 8
               },
               "id": 68,
               "links": [],
@@ -2967,7 +3216,7 @@ data:
                 "h": 8,
                 "w": 12,
                 "x": 0,
-                "y": 40
+                "y": 16
               },
               "id": 116,
               "links": [],
@@ -3072,7 +3321,7 @@ data:
                 "h": 8,
                 "w": 12,
                 "x": 12,
-                "y": 40
+                "y": 16
               },
               "id": 60,
               "links": [],
@@ -3174,7 +3423,7 @@ data:
                 "h": 9,
                 "w": 12,
                 "x": 0,
-                "y": 48
+                "y": 24
               },
               "id": 90,
               "options": {
@@ -3279,7 +3528,7 @@ data:
                 "h": 9,
                 "w": 12,
                 "x": 12,
-                "y": 48
+                "y": 24
               },
               "id": 118,
               "links": [],
@@ -3385,12 +3634,11 @@ data:
                 "h": 8,
                 "w": 12,
                 "x": 0,
-                "y": 57
+                "y": 33
               },
               "id": 120,
               "options": {
                 "footer": {
-                  "countRows": false,
                   "fields": "",
                   "reducer": [
                     "sum"
@@ -3405,7 +3653,7 @@ data:
                   }
                 ]
               },
-              "pluginVersion": "9.4.7",
+              "pluginVersion": "9.1.0",
               "targets": [
                 {
                   "datasource": {
@@ -3513,7 +3761,7 @@ data:
                 "h": 8,
                 "w": 12,
                 "x": 12,
-                "y": 57
+                "y": 33
               },
               "id": 74,
               "links": [],
@@ -4182,7 +4430,7 @@ data:
                     "uid": "$ds"
                   },
                   "editorMode": "code",
-                  "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) \n/ sum(vm_rows{job=~\"$job\", instance=~\"$instance\"})",
+                  "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) \n/ sum(vm_rows{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"})",
                   "format": "time_series",
                   "interval": "",
                   "intervalFactor": 1,
@@ -4609,7 +4857,7 @@ data:
                     "uid": "$ds"
                   },
                   "editorMode": "code",
-                  "expr": "max_over_time(vm_concurrent_addrows_capacity{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
+                  "expr": "max_over_time(vm_concurrent_insert_capacity{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
                   "format": "time_series",
                   "interval": "",
                   "intervalFactor": 1,
@@ -4622,7 +4870,7 @@ data:
                     "type": "prometheus",
                     "uid": "$ds"
                   },
-                  "expr": "sum(vm_concurrent_addrows_current{job=~\"$job\", instance=~\"$instance\"})",
+                  "expr": "sum(vm_concurrent_insert_current{job=~\"$job\", instance=~\"$instance\"})",
                   "format": "time_series",
                   "intervalFactor": 1,
                   "legendFormat": "current",
@@ -4827,11 +5075,13 @@ data:
                     "type": "prometheus",
                     "uid": "$ds"
                   },
-                  "expr": "histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (vmrange))",
+                  "editorMode": "code",
+                  "expr": "histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))",
                   "format": "time_series",
                   "interval": "",
                   "intervalFactor": 1,
                   "legendFormat": "{{instance}}",
+                  "range": true,
                   "refId": "A"
                 }
               ],
@@ -4933,11 +5183,11 @@ data:
                     "uid": "$ds"
                   },
                   "editorMode": "code",
-                  "expr": "histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (vmrange))",
+                  "expr": "histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))",
                   "format": "time_series",
                   "interval": "",
                   "intervalFactor": 1,
-                  "legendFormat": "{{label_name}}",
+                  "legendFormat": "{{instance}}",
                   "range": true,
                   "refId": "A"
                 }
@@ -5040,11 +5290,11 @@ data:
                     "uid": "$ds"
                   },
                   "editorMode": "code",
-                  "expr": "sum(histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))",
+                  "expr": "histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))",
                   "format": "time_series",
                   "interval": "",
                   "intervalFactor": 1,
-                  "legendFormat": "datapoints",
+                  "legendFormat": "{{instance}}",
                   "range": true,
                   "refId": "A"
                 }
@@ -5146,15 +5396,17 @@ data:
                     "type": "prometheus",
                     "uid": "$ds"
                   },
-                  "expr": "histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (vmrange))",
+                  "editorMode": "code",
+                  "expr": "histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))",
                   "format": "time_series",
                   "interval": "",
                   "intervalFactor": 1,
                   "legendFormat": "{{instance}}",
+                  "range": true,
                   "refId": "A"
                 }
               ],
-              "title": "Datapoints scanned per series ($instance)",
+              "title": "Datapoints scanned per query ($instance)",
               "type": "timeseries"
             }
           ],
@@ -5171,8 +5423,7 @@ data:
           "type": "row"
         }
       ],
-      "refresh": "30s",
-      "revision": 1,
+      "refresh": "",
       "schemaVersion": 38,
       "style": "dark",
       "tags": [
@@ -5184,8 +5435,8 @@ data:
           {
             "current": {
               "selected": false,
-              "text": "Victoria",
-              "value": "Victoria"
+              "text": "VictoriaMetrics",
+              "value": "VictoriaMetrics"
             },
             "hide": 0,
             "includeAll": false,
@@ -5241,7 +5492,7 @@ data:
               "refId": "VictoriaMetrics-version-Variable-Query"
             },
             "refresh": 1,
-            "regex": "/.*-tags-(v\\d+\\.\\d+\\.\\d+)/",
+            "regex": "/.*-tags-(v\\\\d+\\\\.\\\\d+\\\\.\\\\d+)/",
             "skipUrlSync": false,
             "sort": 2,
             "tagValuesQuery": "",
@@ -5318,6 +5569,6 @@ data:
       "timezone": "",
       "title": "VictoriaMetrics",
       "uid": "wNf0q_kZk",
-      "version": 2,
+      "version": 1,
       "weekStart": ""
     }
\ No newline at end of file