2023-05-04 20:17:01 +00:00
apiVersion : v1
kind : ConfigMap
metadata :
name : victoria-metrics-grafana-dashboard
namespace : monitoring
labels :
grafana_dashboard : "1"
data :
victoria-metrics.json : |-
{
"__inputs": [ ] ,
"__elements": {},
"__requires": [
{
"type": "grafana" ,
"id": "grafana" ,
"name": "Grafana" ,
2023-10-21 01:03:39 +00:00
"version": "10.1.2"
2023-05-04 20:17:01 +00:00
},
{
"type": "datasource" ,
"id": "prometheus" ,
"name": "Prometheus" ,
"version": "1.0.0"
},
{
"type": "panel" ,
"id": "stat" ,
"name": "Stat" ,
"version": ""
},
{
"type": "panel" ,
"id": "table" ,
"name": "Table" ,
"version": ""
},
{
"type": "panel" ,
"id": "text" ,
"name": "Text" ,
"version": ""
},
{
"type": "panel" ,
"id": "timeseries" ,
"name": "Time series" ,
"version": ""
}
] ,
"annotations": {
"list": [
{
"builtIn": 1 ,
"datasource": {
"type": "datasource" ,
"uid": "grafana"
},
"enable": true ,
"hide": true ,
"iconColor": "rgba(0, 211, 255, 1)" ,
"name": "Annotations & Alerts" ,
"target": {
"limit": 100 ,
"matchAny": false ,
"tags": [ ] ,
"type": "dashboard"
},
"type": "dashboard"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"enable": true ,
"expr": "sum(ALERTS{alertgroup=\"vmsingle\",alertstate=\"firing\",show_at=\"dashboard\"}) by(alertname)" ,
"iconColor": "red" ,
"name": "alerts" ,
"titleFormat": "{{alertname}}"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"enable": true ,
"expr": "sum(vm_app_version{job=~\"$job\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\"} offset 20m) by(short_version))" ,
"hide": true ,
"iconColor": "dark-blue" ,
"name": "version" ,
"textFormat": "{{short_version}}" ,
"titleFormat": "Version change"
}
]
},
"description": "Overview for single node VictoriaMetrics v1.83.0 or higher" ,
"editable": true ,
"fiscalYearStartMonth": 0 ,
"gnetId": 10229 ,
2023-10-21 01:03:39 +00:00
"graphTooltip": 1 ,
2023-05-04 20:17:01 +00:00
"id": null ,
"links": [
{
"icon": "doc" ,
"tags": [ ] ,
"targetBlank": true ,
"title": "Single server Wiki" ,
"type": "link" ,
"url": "https://docs.victoriametrics.com/"
},
{
"icon": "external link" ,
"tags": [ ] ,
"targetBlank": true ,
"title": "Found a bug?" ,
"type": "link" ,
"url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues"
},
{
"icon": "external link" ,
"tags": [ ] ,
"targetBlank": true ,
"title": "New releases" ,
"tooltip": "" ,
"type": "link" ,
"url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases"
}
] ,
"liveNow": false ,
"panels": [
{
"collapsed": false ,
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"gridPos": {
"h": 1 ,
"w": 24 ,
"x": 0 ,
"y": 0
},
"id": 6 ,
"panels": [ ] ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"refId": "A"
}
] ,
"title": "Stats" ,
"type": "row"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "" ,
"gridPos": {
"h": 2 ,
"w": 4 ,
"x": 0 ,
"y": 1
},
"id": 85 ,
"options": {
"code": {
"language": "plaintext" ,
"showLineNumbers": false ,
"showMiniMap": false
},
"content": "<div style=\"text-align: center;\">$version</div>" ,
"mode": "markdown"
},
2023-10-21 01:03:39 +00:00
"pluginVersion": "10.1.2" ,
2023-05-04 20:17:01 +00:00
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"refId": "A"
}
] ,
"title": "Version" ,
"type": "text"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "How many datapoints are in storage" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [ ] ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green" ,
"value": null
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 2 ,
"w": 5 ,
"x": 4 ,
"y": 1
},
"id": 26 ,
"links": [ ] ,
"maxDataPoints": 100 ,
"options": {
"colorMode": "value" ,
"graphMode": "area" ,
"justifyMode": "auto" ,
"orientation": "horizontal" ,
"reduceOptions": {
"calcs": [
"lastNotNull"
] ,
"fields": "" ,
"values": false
},
"text": {},
"textMode": "auto"
},
2023-10-21 01:03:39 +00:00
"pluginVersion": "10.1.2" ,
2023-05-04 20:17:01 +00:00
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"exemplar": false ,
"expr": "sum(vm_rows{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"})" ,
"format": "time_series" ,
"instant": true ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "" ,
"refId": "A"
}
] ,
"title": "Total datapoints" ,
"type": "stat"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Shows the datapoints ingestion rate." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [ ] ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green" ,
"value": null
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 2 ,
"w": 5 ,
"x": 9 ,
"y": 1
},
"id": 107 ,
"links": [ ] ,
"maxDataPoints": 100 ,
"options": {
"colorMode": "value" ,
"graphMode": "area" ,
"justifyMode": "auto" ,
"orientation": "horizontal" ,
"reduceOptions": {
"calcs": [
"lastNotNull"
] ,
"fields": "" ,
"values": false
},
"text": {},
"textMode": "auto"
},
2023-10-21 01:03:39 +00:00
"pluginVersion": "10.1.2" ,
2023-05-04 20:17:01 +00:00
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"exemplar": false ,
"expr": "sum(rate(vm_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))" ,
"format": "time_series" ,
"instant": true ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "" ,
"refId": "A"
}
] ,
"title": "Ingestion rate" ,
"type": "stat"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Shows the rate of HTTP read requests." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [ ] ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green" ,
"value": null
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 2 ,
"w": 5 ,
"x": 14 ,
"y": 1
},
"id": 108 ,
"links": [ ] ,
"maxDataPoints": 100 ,
"options": {
"colorMode": "value" ,
"graphMode": "area" ,
"justifyMode": "auto" ,
"orientation": "horizontal" ,
"reduceOptions": {
"calcs": [
"lastNotNull"
] ,
"fields": "" ,
"values": false
},
"text": {},
"textMode": "auto"
},
2023-10-21 01:03:39 +00:00
"pluginVersion": "10.1.2" ,
2023-05-04 20:17:01 +00:00
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"exemplar": false ,
"expr": "sum(rate(vm_http_requests_total{job=~\"$job\", instance=~\"$instance\", path!~\".*(/write|/metrics)\"}[$__rate_interval]))" ,
"format": "time_series" ,
"instant": true ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "" ,
"refId": "A"
}
] ,
"title": "Read requests" ,
"type": "stat"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Total number of available CPUs for VM process" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [ ] ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green" ,
"value": null
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 2 ,
"w": 5 ,
"x": 19 ,
"y": 1
},
"id": 77,
"links": [ ] ,
"maxDataPoints": 100 ,
"options": {
"colorMode": "value" ,
"graphMode": "area" ,
"justifyMode": "auto" ,
"orientation": "horizontal" ,
"reduceOptions": {
"calcs": [
"lastNotNull"
] ,
"fields": "" ,
"values": false
},
"text": {},
"textMode": "auto"
},
2023-10-21 01:03:39 +00:00
"pluginVersion": "10.1.2" ,
2023-05-04 20:17:01 +00:00
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"exemplar": false ,
"expr": "sum(vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"})" ,
"format": "time_series" ,
"instant": true ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "" ,
"refId": "A"
}
] ,
"title": "Available CPU" ,
"type": "stat"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [ ] ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "red" ,
"value": null
},
{
"color": "green" ,
"value": 1800
}
]
},
"unit": "s"
},
"overrides": [ ]
},
"gridPos": {
"h": 2 ,
"w": 4 ,
"x": 0 ,
"y": 3
},
"id": 87 ,
"options": {
"colorMode": "value" ,
"graphMode": "area" ,
"justifyMode": "auto" ,
"orientation": "auto" ,
"reduceOptions": {
"calcs": [
"lastNotNull"
] ,
"fields": "" ,
"values": false
},
"text": {},
"textMode": "auto"
},
2023-10-21 01:03:39 +00:00
"pluginVersion": "10.1.2" ,
2023-05-04 20:17:01 +00:00
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"exemplar": false ,
"expr": "vm_app_uptime_seconds{job=~\"$job\", instance=~\"$instance\"}" ,
"instant": true ,
"interval": "" ,
"legendFormat": "" ,
"refId": "A"
}
] ,
"title": "Uptime" ,
"type": "stat"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Shows the number of active time series with new data points inserted during the last hour. High value may result in ingestion slowdown. \n\nSee more details here https://docs.victoriametrics.com/FAQ.html#what-is-an-active-time-series" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [ ] ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green" ,
"value": null
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 2 ,
"w": 5 ,
"x": 4 ,
"y": 3
},
"id": 38 ,
"links": [ ] ,
"maxDataPoints": 100 ,
"options": {
"colorMode": "value" ,
"graphMode": "area" ,
"justifyMode": "auto" ,
"orientation": "horizontal" ,
"reduceOptions": {
"calcs": [
"lastNotNull"
] ,
"fields": "" ,
"values": false
},
"text": {},
"textMode": "auto"
},
2023-10-21 01:03:39 +00:00
"pluginVersion": "10.1.2" ,
2023-05-04 20:17:01 +00:00
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"exemplar": false ,
"expr": "vm_cache_entries{job=~\"$job\", instance=~\"$instance\", type=\"storage/hour_metric_ids\"}" ,
"format": "time_series" ,
"instant": true ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "" ,
"refId": "A"
}
] ,
"title": "Active series" ,
"type": "stat"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Total amount of used disk space" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [ ] ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green" ,
"value": null
}
]
},
"unit": "bytes"
},
"overrides": [ ]
},
"gridPos": {
"h": 2 ,
"w": 5 ,
"x": 9 ,
"y": 3
},
"id": 81 ,
"links": [ ] ,
"maxDataPoints": 100 ,
"options": {
"colorMode": "value" ,
"graphMode": "area" ,
"justifyMode": "auto" ,
"orientation": "horizontal" ,
"reduceOptions": {
"calcs": [
"lastNotNull"
] ,
"fields": "" ,
"values": false
},
"text": {},
"textMode": "auto"
},
2023-10-21 01:03:39 +00:00
"pluginVersion": "10.1.2" ,
2023-05-04 20:17:01 +00:00
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"exemplar": false ,
"expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"})" ,
"format": "time_series" ,
"instant": true ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "" ,
"refId": "A"
}
] ,
"title": "Disk space usage" ,
"type": "stat"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Average disk usage per datapoint." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [ ] ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green" ,
"value": null
}
]
},
"unit": "bytes"
},
"overrides": [ ]
},
"gridPos": {
"h": 2 ,
"w": 5 ,
"x": 14 ,
"y": 3
},
"id": 82 ,
"links": [ ] ,
"maxDataPoints": 100 ,
"options": {
"colorMode": "value" ,
"graphMode": "area" ,
"justifyMode": "auto" ,
"orientation": "horizontal" ,
"reduceOptions": {
"calcs": [
"lastNotNull"
] ,
"fields": "" ,
"values": false
},
"text": {},
"textMode": "auto"
},
2023-10-21 01:03:39 +00:00
"pluginVersion": "10.1.2" ,
2023-05-04 20:17:01 +00:00
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"exemplar": false ,
"expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) / sum(vm_rows{job=~\"$job\", instance=~\"$instance\"})" ,
"format": "time_series" ,
"instant": true ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "" ,
"refId": "A"
}
] ,
"title": "Bytes per point" ,
"type": "stat"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Total size of available memory for VM process" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [ ] ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green" ,
"value": null
}
]
},
"unit": "bytes"
},
"overrides": [ ]
},
"gridPos": {
"h": 2 ,
"w": 5 ,
"x": 19 ,
"y": 3
},
"id": 78 ,
"links": [ ] ,
"maxDataPoints": 100 ,
"options": {
"colorMode": "value" ,
"graphMode": "area" ,
"justifyMode": "auto" ,
"orientation": "horizontal" ,
"reduceOptions": {
"calcs": [
"lastNotNull"
] ,
"fields": "" ,
"values": false
},
"text": {},
"textMode": "auto"
},
2023-10-21 01:03:39 +00:00
"pluginVersion": "10.1.2" ,
2023-05-04 20:17:01 +00:00
"targets": [
{
"datasource": {
"uid": "$ds"
},
"exemplar": false ,
"expr": "sum(vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"})" ,
"format": "time_series" ,
"instant": true ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "" ,
"refId": "A"
}
] ,
"title": "Available memory" ,
"type": "stat"
},
{
"collapsed": false ,
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"gridPos": {
"h": 1 ,
"w": 24 ,
"x": 0 ,
"y": 5
},
"id": 24 ,
"panels": [ ] ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"refId": "A"
}
] ,
"title": "Overview" ,
"type": "row"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "How many datapoints are inserted into storage per second" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 10 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
2023-10-21 01:03:39 +00:00
"insertNulls": false ,
2023-05-04 20:17:01 +00:00
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green" ,
"value": null
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
"y": 6
},
"id": 106 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"uid": "$ds"
},
"editorMode": "code" ,
"expr": "sum(rate(vm_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (type) > 0" ,
"format": "time_series" ,
"hide": false ,
"intervalFactor": 1 ,
"legendFormat": "__auto" ,
"range": true ,
"refId": "A"
}
] ,
"title": "Datapoints ingestion rate ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
2023-10-21 01:03:39 +00:00
"insertNulls": false ,
2023-05-04 20:17:01 +00:00
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green" ,
"value": null
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
"y": 6
},
"id": 12 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"expr": "sum(rate(vm_http_requests_total{job=~\"$job\", instance=~\"$instance\", path!~\"/favicon.ico\"}[$__rate_interval])) by (path) > 0" ,
"format": "time_series" ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "{{path}}" ,
"range": true ,
"refId": "A"
}
] ,
"title": "Requests rate ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Shows the number of active time series with new data points inserted during the last hour. High value may result in ingestion slowdown. \n\nSee following link for details:" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 10 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
2023-10-21 01:03:39 +00:00
"insertNulls": false ,
2023-05-04 20:17:01 +00:00
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green" ,
"value": null
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
"y": 14
},
"id": 51 ,
"links": [
{
"targetBlank": true ,
"title": "troubleshooting" ,
"url": "https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#troubleshooting"
}
] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "vm_cache_entries{job=~\"$job\", instance=~\"$instance\", type=\"storage/hour_metric_ids\"}" ,
"format": "time_series" ,
"intervalFactor": 1 ,
"legendFormat": "Active time series" ,
"refId": "A"
}
] ,
"title": "Active time series ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "The less time it takes is better.\n* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 10 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
2023-10-21 01:03:39 +00:00
"insertNulls": false ,
2023-05-04 20:17:01 +00:00
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green" ,
"value": null
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "s"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
"y": 14
},
"id": 22 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"expr": "max(vm_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", quantile=\"0.99\"}) by (path) > 0" ,
"format": "time_series" ,
"intervalFactor": 1 ,
"legendFormat": "__auto" ,
"range": true ,
"refId": "A"
}
] ,
"title": "Query duration 0.99 quantile ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 10 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
"y": 22
},
"id": 35 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"exemplar": false ,
"expr": "sum(rate(vm_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0" ,
"format": "time_series" ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "__auto" ,
"range": true ,
"refId": "A"
}
] ,
"title": "Requests error rate ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "bars" ,
"fillOpacity": 100 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [ ] ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
}
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
"y": 22
},
"id": 110 ,
"options": {
"legend": {
"calcs": [
"lastNotNull" ,
"mean" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "single" ,
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
2023-10-21 01:03:39 +00:00
"expr": "sum(rate(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (level, location) > 0" ,
2023-05-04 20:17:01 +00:00
"interval": "5m" ,
2023-10-21 01:03:39 +00:00
"legendFormat": "{{level}}: {{location}}" ,
2023-05-04 20:17:01 +00:00
"range": true ,
"refId": "A"
}
] ,
"title": "Logging rate" ,
"type": "timeseries"
},
{
"collapsed": true ,
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"gridPos": {
"h": 1 ,
"w": 24 ,
"x": 0 ,
"y": 30
},
"id": 46 ,
"panels": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Percentage of used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "percentunit"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
"y": 31
},
"id": 112 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"exemplar": false ,
2023-10-21 01:03:39 +00:00
"expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)" ,
2023-05-04 20:17:01 +00:00
"interval": "" ,
"legendFormat": "__auto" ,
"range": true ,
"refId": "A"
}
] ,
"title": "RSS memory % usage ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-10-21 01:03:39 +00:00
"description": "" ,
2023-05-04 20:17:01 +00:00
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
2023-10-21 01:03:39 +00:00
"unit": "bytes"
2023-05-04 20:17:01 +00:00
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
"y": 31
},
2023-10-21 01:03:39 +00:00
"id": 44 ,
2023-05-04 20:17:01 +00:00
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
2023-10-21 01:03:39 +00:00
"sort": "desc"
2023-05-04 20:17:01 +00:00
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-10-21 01:03:39 +00:00
"expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})" ,
"format": "time_series" ,
"hide": false ,
"intervalFactor": 1 ,
"legendFormat": "requested from system" ,
"refId": "A"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})" ,
"format": "time_series" ,
"hide": false ,
"intervalFactor": 1 ,
"legendFormat": "heap inuse" ,
"refId": "B"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "sum(go_memstats_stack_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})" ,
"format": "time_series" ,
"hide": false ,
"intervalFactor": 1 ,
"legendFormat": "stack inuse" ,
"refId": "C"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "sum(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"})" ,
"format": "time_series" ,
"hide": false ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "resident" ,
"refId": "D"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-05-04 20:17:01 +00:00
"exemplar": false ,
2023-10-21 01:03:39 +00:00
"expr": "sum(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"})" ,
2023-05-04 20:17:01 +00:00
"format": "time_series" ,
2023-10-21 01:03:39 +00:00
"hide": false ,
2023-05-04 20:17:01 +00:00
"interval": "" ,
"intervalFactor": 1 ,
2023-10-21 01:03:39 +00:00
"legendFormat": "resident anonymous" ,
"refId": "E"
2023-05-04 20:17:01 +00:00
}
] ,
2023-10-21 01:03:39 +00:00
"title": "Memory usage ($instance)" ,
2023-05-04 20:17:01 +00:00
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-10-21 01:03:39 +00:00
"description": "Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%" ,
2023-05-04 20:17:01 +00:00
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
2023-10-21 01:03:39 +00:00
"unit": "percentunit"
2023-05-04 20:17:01 +00:00
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
"y": 39
},
2023-10-21 01:03:39 +00:00
"id": 123 ,
2023-05-04 20:17:01 +00:00
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-10-21 01:03:39 +00:00
"editorMode": "code" ,
"exemplar": false ,
"expr": "max(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)" ,
"interval": "" ,
"legendFormat": "__auto" ,
"range": true ,
2023-05-04 20:17:01 +00:00
"refId": "A"
2023-10-21 01:03:39 +00:00
}
] ,
"title": "RSS anonymous memory % usage ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
2023-05-04 20:17:01 +00:00
},
2023-10-21 01:03:39 +00:00
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
2023-05-04 20:17:01 +00:00
},
2023-10-21 01:03:39 +00:00
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "percentunit"
2023-05-04 20:17:01 +00:00
},
2023-10-21 01:03:39 +00:00
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
"y": 39
},
"id": 114 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
2023-05-04 20:17:01 +00:00
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-10-21 01:03:39 +00:00
"editorMode": "code" ,
"exemplar": false ,
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)" ,
2023-05-04 20:17:01 +00:00
"format": "time_series" ,
"interval": "" ,
"intervalFactor": 1 ,
2023-10-21 01:03:39 +00:00
"legendFormat": "__auto" ,
"range": true ,
"refId": "A"
}
] ,
"title": "CPU % usage ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Shows the percentage of open file descriptors compared to the limit set in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 2 ,
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "percentunit"
2023-05-04 20:17:01 +00:00
},
2023-10-21 01:03:39 +00:00
"overrides": [
{
"matcher": {
"id": "byName" ,
"options": "max"
},
"properties": [
{
"id": "color" ,
"value": {
"fixedColor": "#C4162A" ,
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
"y": 47
},
"id": 75 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
2023-05-04 20:17:01 +00:00
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-10-21 01:03:39 +00:00
"editorMode": "code" ,
"expr": "max_over_time(process_open_fds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n/\nprocess_max_fds{job=~\"$job\", instance=~\"$instance\"}" ,
2023-05-04 20:17:01 +00:00
"format": "time_series" ,
"hide": false ,
"interval": "" ,
2023-10-21 01:03:39 +00:00
"intervalFactor": 2 ,
"legendFormat": "{{instance}}" ,
"range": true ,
"refId": "A"
2023-05-04 20:17:01 +00:00
}
] ,
2023-10-21 01:03:39 +00:00
"title": "Open FDs ($instance)" ,
2023-05-04 20:17:01 +00:00
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName" ,
"options": "Limit"
},
"properties": [
{
"id": "color" ,
"value": {
"fixedColor": "#F2495C" ,
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
2023-10-21 01:03:39 +00:00
"y": 47
2023-05-04 20:17:01 +00:00
},
"id": 57 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])" ,
"format": "time_series" ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "CPU cores used" ,
"refId": "A"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"exemplar": false ,
"expr": "process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}" ,
"format": "time_series" ,
"hide": false ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "Limit" ,
"refId": "B"
}
] ,
"title": "CPU ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
2023-10-21 01:03:39 +00:00
"fillOpacity": 10 ,
2023-05-04 20:17:01 +00:00
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
2023-10-21 01:03:39 +00:00
"decimals": 0 ,
2023-05-04 20:17:01 +00:00
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
2023-10-21 01:03:39 +00:00
"unit": "short"
2023-05-04 20:17:01 +00:00
},
2023-10-21 01:03:39 +00:00
"overrides": [ ]
2023-05-04 20:17:01 +00:00
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
2023-10-21 01:03:39 +00:00
"y": 55
2023-05-04 20:17:01 +00:00
},
2023-10-21 01:03:39 +00:00
"id": 47 ,
2023-05-04 20:17:01 +00:00
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
2023-10-21 01:03:39 +00:00
"expr": "sum(go_goroutines{job=~\"$job\", instance=~\"$instance\"}) by(instance)" ,
2023-05-04 20:17:01 +00:00
"format": "time_series" ,
"intervalFactor": 2 ,
2023-10-21 01:03:39 +00:00
"legendFormat": "{{instance}}" ,
2023-05-04 20:17:01 +00:00
"range": true ,
"refId": "A"
}
] ,
2023-10-21 01:03:39 +00:00
"title": "Goroutines ($instance)" ,
2023-05-04 20:17:01 +00:00
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Shows the number of bytes read/write from the storage layer." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 10 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "bytes"
},
"overrides": [
{
"matcher": {
"id": "byName" ,
"options": "read"
},
"properties": [
{
"id": "custom.transform" ,
"value": "negative-Y"
}
]
}
]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
2023-10-21 01:03:39 +00:00
"y": 55
2023-05-04 20:17:01 +00:00
},
"id": 76 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))" ,
"format": "time_series" ,
"hide": false ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "read" ,
"refId": "A"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))" ,
"format": "time_series" ,
"hide": false ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "write" ,
"refId": "B"
}
] ,
"title": "Disk writes/reads ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 10 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0 ,
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
2023-10-21 01:03:39 +00:00
"y": 63
2023-05-04 20:17:01 +00:00
},
2023-10-21 01:03:39 +00:00
"id": 48 ,
2023-05-04 20:17:01 +00:00
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-10-21 01:03:39 +00:00
"editorMode": "code" ,
"expr": "sum(process_num_threads{job=~\"$job\", instance=~\"$instance\"}) by(instance)" ,
2023-05-04 20:17:01 +00:00
"format": "time_series" ,
"intervalFactor": 2 ,
2023-10-21 01:03:39 +00:00
"legendFormat": "{{instance}}" ,
"range": true ,
2023-05-04 20:17:01 +00:00
"refId": "A"
}
] ,
2023-10-21 01:03:39 +00:00
"title": "Threads ($instance)" ,
2023-05-04 20:17:01 +00:00
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-10-21 01:03:39 +00:00
"description": "Shows the number of read/write syscalls such as read, pread, write, pwrite." ,
2023-05-04 20:17:01 +00:00
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 10 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
2023-10-21 01:03:39 +00:00
"overrides": [
{
"matcher": {
"id": "byName" ,
"options": "read calls"
},
"properties": [
{
"id": "custom.transform" ,
"value": "negative-Y"
}
]
}
]
2023-05-04 20:17:01 +00:00
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
2023-10-21 01:03:39 +00:00
"y": 63
2023-05-04 20:17:01 +00:00
},
2023-10-21 01:03:39 +00:00
"id": 124 ,
2023-05-04 20:17:01 +00:00
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-10-21 01:03:39 +00:00
"editorMode": "code" ,
"expr": "sum(rate(process_io_read_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))" ,
2023-05-04 20:17:01 +00:00
"format": "time_series" ,
"hide": false ,
2023-10-21 01:03:39 +00:00
"interval": "" ,
2023-05-04 20:17:01 +00:00
"intervalFactor": 1 ,
2023-10-21 01:03:39 +00:00
"legendFormat": "read calls" ,
"range": true ,
2023-05-04 20:17:01 +00:00
"refId": "A"
2023-10-21 01:03:39 +00:00
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"expr": "sum(rate(process_io_write_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))" ,
"format": "time_series" ,
"hide": false ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "write calls" ,
"range": true ,
"refId": "B"
2023-05-04 20:17:01 +00:00
}
] ,
2023-10-21 01:03:39 +00:00
"title": "Disk write/read calls ($instance)" ,
2023-05-04 20:17:01 +00:00
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-10-21 01:03:39 +00:00
"description": "" ,
2023-05-04 20:17:01 +00:00
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 10 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
2023-10-21 01:03:39 +00:00
"y": 71
2023-05-04 20:17:01 +00:00
},
2023-10-21 01:03:39 +00:00
"id": 49 ,
2023-05-04 20:17:01 +00:00
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-10-21 01:03:39 +00:00
"editorMode": "code" ,
"expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(instance)" ,
2023-05-04 20:17:01 +00:00
"format": "time_series" ,
2023-10-21 01:03:39 +00:00
"hide": false ,
"intervalFactor": 1 ,
"legendFormat": "{{instance}}" ,
"range": true ,
2023-05-04 20:17:01 +00:00
"refId": "A"
}
] ,
2023-10-21 01:03:39 +00:00
"title": "TCP connections rate ($instance)" ,
2023-05-04 20:17:01 +00:00
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 10 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
2023-10-21 01:03:39 +00:00
"y": 71
2023-05-04 20:17:01 +00:00
},
2023-10-21 01:03:39 +00:00
"id": 37 ,
2023-05-04 20:17:01 +00:00
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
2023-10-21 01:03:39 +00:00
"expr": "sum(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"}) by(instance)" ,
2023-05-04 20:17:01 +00:00
"format": "time_series" ,
"hide": false ,
"intervalFactor": 1 ,
2023-10-21 01:03:39 +00:00
"legendFormat": "{{instance}}" ,
2023-05-04 20:17:01 +00:00
"range": true ,
"refId": "A"
}
] ,
2023-10-21 01:03:39 +00:00
"title": "TCP connections ($instance)" ,
2023-05-04 20:17:01 +00:00
"type": "timeseries"
}
] ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"refId": "A"
}
] ,
"title": "Resource usage" ,
"type": "row"
},
{
"collapsed": true ,
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"gridPos": {
"h": 1 ,
"w": 24 ,
"x": 0 ,
"y": 31
},
"id": 71 ,
"panels": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Shows the rate and total number of new series created over last 24h.\n\nHigh churn rate tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected cardinality \"explosions\".\n\nThe higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://www.robustperception.io/using-tsdb-analyze-to-investigate-churn-and-cardinality" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
2023-10-21 01:03:39 +00:00
"y": 8
2023-05-04 20:17:01 +00:00
},
"id": 66 ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "sum(rate(vm_new_timeseries_created_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))" ,
"interval": "" ,
"legendFormat": "churn rate" ,
"refId": "A"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "sum(increase(vm_new_timeseries_created_total{job=~\"$job\", instance=~\"$instance\"}[24h]))" ,
"interval": "" ,
"legendFormat": "new series over 24h" ,
"refId": "B"
}
] ,
"title": "Churn rate ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-10-21 01:03:39 +00:00
"description": "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. \n\nThe less value is better. If percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of active time series. \n\nIn general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on many other factors such as the number of labels per time series and the length of label values. See also https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183" ,
2023-05-04 20:17:01 +00:00
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 10 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "line+area"
}
},
"decimals": 2 ,
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "transparent"
},
{
"color": "red" ,
"value": 0.1
}
]
},
"unit": "percentunit"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
2023-10-21 01:03:39 +00:00
"y": 8
2023-05-04 20:17:01 +00:00
},
"id": 68 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"expr": "max(\n rate(vm_slow_row_inserts_total{job=~\"$job\"}[$__rate_interval]) \n / rate(vm_rows_added_to_storage_total{job=~\"$job\"}[$__rate_interval])\n)" ,
"format": "time_series" ,
"hide": false ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "slow inserts percentage" ,
"range": true ,
"refId": "A"
}
] ,
"title": "Slow inserts ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Merge assist happens when storage can't keep up with merging parts. This is usually a sign of overload for storage." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "bars" ,
"fillOpacity": 100 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
2023-10-21 01:03:39 +00:00
"y": 16
2023-05-04 20:17:01 +00:00
},
"id": 116 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"exemplar": false ,
"expr": "sum(increase(vm_assisted_merges_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(type) > 0" ,
"format": "time_series" ,
"interval": "5m" ,
"intervalFactor": 1 ,
"legendFormat": "__auto" ,
"range": true ,
"refId": "A"
}
] ,
"title": "Assisted merges ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Slow queries rate according to `search.logSlowQueryDuration` flag, which is `5s` by default." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 10 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
2023-10-21 01:03:39 +00:00
"y": 16
2023-05-04 20:17:01 +00:00
},
"id": 60 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "sum(rate(vm_slow_queries_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))" ,
"format": "time_series" ,
"hide": false ,
"intervalFactor": 1 ,
"legendFormat": "slow queries rate" ,
"refId": "A"
}
] ,
"title": "Slow queries rate ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [ ] ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "percentunit"
},
"overrides": [ ]
},
"gridPos": {
"h": 9 ,
"w": 12 ,
"x": 0 ,
2023-10-21 01:03:39 +00:00
"y": 24
2023-05-04 20:17:01 +00:00
},
"id": 90 ,
"options": {
"legend": {
"calcs": [
"lastNotNull" ,
"mean" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"exemplar": false ,
"expr": "vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"} / vm_cache_size_max_bytes{job=~\"$job\", instance=~\"$instance\"}" ,
"interval": "" ,
"legendFormat": "{{type}}" ,
"range": true ,
"refId": "A"
}
] ,
"title": "Cache usage % by type ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Shows cache miss ratio. Lower is better." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"max": 1 ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "percentunit"
},
"overrides": [ ]
},
"gridPos": {
"h": 9 ,
"w": 12 ,
"x": 12 ,
2023-10-21 01:03:39 +00:00
"y": 24
2023-05-04 20:17:01 +00:00
},
"id": 118 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"exemplar": false ,
"expr": "(\n rate(vm_cache_misses_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n rate(vm_cache_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n) > 0" ,
"format": "time_series" ,
"hide": false ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "{{type}}" ,
"range": true ,
"refId": "A"
}
] ,
"title": "Cache miss ratio ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto" ,
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [ ] ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName" ,
"options": "Value"
},
"properties": [
{
"id": "custom.hidden" ,
"value": true
}
]
},
{
"matcher": {
"id": "byName" ,
"options": "Time"
},
"properties": [
{
"id": "custom.hidden" ,
"value": true
}
]
}
]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
2023-10-21 01:03:39 +00:00
"y": 33
2023-05-04 20:17:01 +00:00
},
"id": 120 ,
"options": {
"footer": {
"fields": "" ,
"reducer": [
"sum"
] ,
"show": false
},
"showHeader": true ,
"sortBy": [
{
"desc": true ,
"displayName": "job"
}
]
},
2023-10-21 01:03:39 +00:00
"pluginVersion": "9.1.0" ,
2023-05-04 20:17:01 +00:00
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"exemplar": false ,
"expr": "sum(flag{is_set=\"true\", job=~\"$job\", instance=~\"$instance\"}) by(job, instance, name, value)" ,
"format": "table" ,
"instant": true ,
"legendFormat": "__auto" ,
"range": false ,
"refId": "A"
}
] ,
"title": "Non-default flags" ,
"transformations": [
{
"id": "groupBy" ,
"options": {
"fields": {
"instance": {
"aggregations": [ ]
},
"job": {
"aggregations": [ ]
},
"name": {
"aggregations": [ ] ,
"operation": "groupby"
},
"value": {
"aggregations": [ ] ,
"operation": "groupby"
}
}
}
}
] ,
"type": "table"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n\nThis prevents from ingesting metrics with too many labels. The value of `maxLabelsPerTimeseries` must be adjusted for your workload.\n\nWhen limit is exceeded (graph is > 0) - extra labels are dropped, which could result in unexpected identical time series." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 10 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 2 ,
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
2023-10-21 01:03:39 +00:00
"y": 33
2023-05-04 20:17:01 +00:00
},
"id": 74 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "list" ,
"placement": "bottom" ,
"showLegend": false
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"exemplar": false ,
"expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))" ,
"format": "time_series" ,
"hide": false ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "limit exceeded" ,
"refId": "A"
}
] ,
"title": "Labels limit exceeded ($instance)" ,
"type": "timeseries"
}
] ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"refId": "A"
}
] ,
"title": "Troubleshooting" ,
"type": "row"
},
{
"collapsed": true ,
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"gridPos": {
"h": 1 ,
"w": 24 ,
"x": 0 ,
"y": 32
},
"id": 14 ,
"panels": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "How many datapoints are inserted into storage per second" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 10 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
"y": 33
},
"id": 10 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"expr": "sum(rate(vm_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (type) > 0" ,
"format": "time_series" ,
"hide": false ,
"intervalFactor": 1 ,
"legendFormat": "{{type}}" ,
"range": true ,
"refId": "A"
}
] ,
"title": "Datapoints ingestion rate ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Shows the time needed to reach the 100% of disk capacity based on the following params:\n* free disk space;\n* row ingestion rate;\n* dedup rate;\n* compression.\n\nUse this panel for capacity planning in order to estimate the time remaining for running out of the disk space.\n\n" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "s"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
"y": 33
},
"id": 73 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"min"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"expr": "vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"} \n/ ignoring(path) (\n (\n rate(vm_rows_added_to_storage_total{job=~\"$job\", instance=~\"$instance\"}[1d]) \n - ignoring(type) rate(vm_deduplicated_samples_total{job=~\"$job\", instance=~\"$instance\", type=\"merge\"}[1d])\n ) * scalar(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"}) \n / sum(vm_rows{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"})\n )\n )" ,
"format": "time_series" ,
"hide": false ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "{{instance}}" ,
"range": true ,
"refId": "A"
}
] ,
"title": "Storage full ETA ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Shows amount of on-disk space occupied by data points and the remaining disk space at `-storageDataPath`" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "bytes"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
"y": 41
},
"id": 53 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"})" ,
"format": "time_series" ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "Used (datapoints)" ,
"range": true ,
"refId": "A"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}" ,
"format": "time_series" ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "Free" ,
"refId": "B"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type=~\"indexdb.*\"})" ,
"format": "time_series" ,
"hide": false ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "Used (index)" ,
"range": true ,
"refId": "C"
}
] ,
"title": "Disk space usage - datapoints ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `2*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every second." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 10 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName" ,
"options": "pending index entries"
},
"properties": [
{
"id": "unit" ,
"value": "none"
},
{
"id": "decimals" ,
"value": 3
}
]
}
]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
"y": 41
},
"id": 34 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "vm_pending_rows{job=~\"$job\", instance=~\"$instance\", type=\"storage\"}" ,
"format": "time_series" ,
"hide": false ,
"intervalFactor": 1 ,
"legendFormat": "pending datapoints" ,
"refId": "A"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "vm_pending_rows{job=~\"$job\", instance=~\"$instance\", type=\"indexdb\"}" ,
"format": "time_series" ,
"hide": false ,
"intervalFactor": 1 ,
"legendFormat": "pending index entries" ,
"refId": "B"
}
] ,
"title": "Pending datapoints ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Shows how many datapoints are in the storage and what is average disk usage per datapoint." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName" ,
"options": "bytes-per-datapoint"
},
"properties": [
{
"id": "unit" ,
"value": "bytes"
},
{
"id": "decimals" ,
"value": 2
}
]
}
]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
"y": 49
},
"id": 30 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "sum(vm_rows{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"})" ,
"format": "time_series" ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "total datapoints" ,
"refId": "A"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
2023-10-21 01:03:39 +00:00
"expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) \n/ sum(vm_rows{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"})" ,
2023-05-04 20:17:01 +00:00
"format": "time_series" ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "bytes-per-datapoint" ,
"range": true ,
"refId": "B"
}
] ,
"title": "Datapoints ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"description": "Data parts of LSM tree.\nHigh number of parts could be an evidence of slow merge performance - check the resource utilization.\n* `indexdb` - inverted index\n* `storage/small` - recently added parts of data ingested into storage(hot data)\n* `storage/big` - small parts gradually merged into big parts (cold data)" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
"y": 49
},
"id": 36 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "sum(vm_parts{job=~\"$job\", instance=~\"$instance\"}) by (type)" ,
"format": "time_series" ,
"intervalFactor": 1 ,
"legendFormat": "{{type}}" ,
"refId": "A"
}
] ,
"title": "LSM parts ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Shows how many rows were ignored on insertion due to corrupted or out of retention timestamps." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 10 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
"y": 57
},
"id": 58 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
"exemplar": false ,
"expr": "sum(increase(vm_rows_ignored_total{job=~\"$job\", instance=~\"$instance\"}[1h])) by (reason)" ,
"format": "time_series" ,
"hide": false ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "{{reason}}" ,
"range": true ,
"refId": "A"
}
] ,
"title": "Rows ignored for last 1h ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "The number of on-going merges in storage nodes. It is expected to have high numbers for `storage/small` metric." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0 ,
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
"y": 57
},
"id": 62 ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "sum(vm_active_merges{job=~\"$job\", instance=~\"$instance\"}) by(type)" ,
"legendFormat": "{{type}}" ,
"refId": "A"
}
] ,
"title": "Active merges ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. However, these two actions are separated and controlled via different concurrency limiters. The `max` on this panel can't be changed and always equal to number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU.\n\n" ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0 ,
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName" ,
"options": "max"
},
"properties": [
{
"id": "color" ,
"value": {
"fixedColor": "#C4162A" ,
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
"y": 65
},
"id": 59 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
2023-10-21 01:03:39 +00:00
"expr": "max_over_time(vm_concurrent_insert_capacity{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])" ,
2023-05-04 20:17:01 +00:00
"format": "time_series" ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "max" ,
"range": true ,
"refId": "A"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-10-21 01:03:39 +00:00
"expr": "sum(vm_concurrent_insert_current{job=~\"$job\", instance=~\"$instance\"})" ,
2023-05-04 20:17:01 +00:00
"format": "time_series" ,
"intervalFactor": 1 ,
"legendFormat": "current" ,
"refId": "B"
}
] ,
"title": "Concurrent flushes on disk ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "The number of rows merged per second by storage nodes." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0 ,
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
"y": 65
},
"id": 64 ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "none"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"expr": "sum(rate(vm_rows_merged_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(type)" ,
"legendFormat": "{{type}}" ,
"refId": "A"
}
] ,
"title": "Merge speed ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "99th percentile of number of series read per query." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 2 ,
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
"y": 73
},
"id": 99 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-10-21 01:03:39 +00:00
"editorMode": "code" ,
"expr": "histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))" ,
2023-05-04 20:17:01 +00:00
"format": "time_series" ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "{{instance}}" ,
2023-10-21 01:03:39 +00:00
"range": true ,
2023-05-04 20:17:01 +00:00
"refId": "A"
}
] ,
"title": "Series read per query ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "99th percentile of number of raw samples read per queried series." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 2 ,
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
"y": 73
},
"id": 103 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
2023-10-21 01:03:39 +00:00
"expr": "histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))" ,
2023-05-04 20:17:01 +00:00
"format": "time_series" ,
"interval": "" ,
"intervalFactor": 1 ,
2023-10-21 01:03:39 +00:00
"legendFormat": "{{instance}}" ,
2023-05-04 20:17:01 +00:00
"range": true ,
"refId": "A"
}
] ,
"title": "Datapoints read per series ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "99th percentile of number of raw datapoints read per query." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 2 ,
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 0 ,
"y": 81
},
"id": 122 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"editorMode": "code" ,
2023-10-21 01:03:39 +00:00
"expr": "histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))" ,
2023-05-04 20:17:01 +00:00
"format": "time_series" ,
"interval": "" ,
"intervalFactor": 1 ,
2023-10-21 01:03:39 +00:00
"legendFormat": "{{instance}}" ,
2023-05-04 20:17:01 +00:00
"range": true ,
"refId": "A"
}
] ,
"title": "Datapoints read per query ($instance)" ,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"description": "99th percentile of number of raw samples scanner per query.\n\nThis number can exceed number of RowsReadPerQuery if `step` query arg passed to [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) is smaller than the lookbehind window set in square brackets of [rollup function](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions). For example, if `increase(some_metric[1h])` is executed with the `step=5m`, then the same raw samples on a hour time range are scanned `1h/5m=12` times. See [this article](https://valyala.medium.com/how-to-optimize-promql-and-metricsql-queries-85a1b75bf986) for details." ,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false ,
"axisColorMode": "text" ,
"axisLabel": "" ,
"axisPlacement": "auto" ,
"barAlignment": 0 ,
"drawStyle": "line" ,
"fillOpacity": 0 ,
"gradientMode": "none" ,
"hideFrom": {
"legend": false ,
"tooltip": false ,
"viz": false
},
"lineInterpolation": "linear" ,
"lineWidth": 1 ,
"pointSize": 5 ,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never" ,
"spanNulls": false ,
"stacking": {
"group": "A" ,
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 2 ,
"links": [ ] ,
"mappings": [ ] ,
"min": 0 ,
"thresholds": {
"mode": "absolute" ,
"steps": [
{
"color": "green"
},
{
"color": "red" ,
"value": 80
}
]
},
"unit": "short"
},
"overrides": [ ]
},
"gridPos": {
"h": 8 ,
"w": 12 ,
"x": 12 ,
"y": 81
},
"id": 105 ,
"links": [ ] ,
"options": {
"legend": {
"calcs": [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode": "table" ,
"placement": "bottom" ,
"showLegend": true ,
"sortBy": "Last *" ,
"sortDesc": true
},
"tooltip": {
"mode": "multi" ,
"sort": "desc"
}
},
"pluginVersion": "9.1.0" ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
2023-10-21 01:03:39 +00:00
"editorMode": "code" ,
"expr": "histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))" ,
2023-05-04 20:17:01 +00:00
"format": "time_series" ,
"interval": "" ,
"intervalFactor": 1 ,
"legendFormat": "{{instance}}" ,
2023-10-21 01:03:39 +00:00
"range": true ,
2023-05-04 20:17:01 +00:00
"refId": "A"
}
] ,
2023-10-21 01:03:39 +00:00
"title": "Datapoints scanned per query ($instance)" ,
2023-05-04 20:17:01 +00:00
"type": "timeseries"
}
] ,
"targets": [
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"refId": "A"
}
] ,
"title": "Storage" ,
"type": "row"
}
] ,
2023-10-21 01:03:39 +00:00
"refresh": "" ,
2023-05-04 20:17:01 +00:00
"schemaVersion": 38 ,
"style": "dark" ,
"tags": [
"victoriametrics" ,
"vmsingle"
] ,
"templating": {
"list": [
{
"current": {
"selected": false ,
2023-10-21 01:03:39 +00:00
"text": "VictoriaMetrics" ,
"value": "VictoriaMetrics"
2023-05-04 20:17:01 +00:00
},
"hide": 0 ,
"includeAll": false ,
"multi": false ,
"name": "ds" ,
"options": [ ] ,
"query": "prometheus" ,
"queryValue": "" ,
"refresh": 1 ,
"regex": "" ,
"skipUrlSync": false ,
"type": "datasource"
},
{
"current": {},
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"definition": "label_values(vm_app_version{version=~\"victoria-metrics-.*\"}, job)" ,
"hide": 0 ,
"includeAll": false ,
"multi": false ,
"name": "job" ,
"options": [ ] ,
"query": {
"query": "label_values(vm_app_version{version=~\"victoria-metrics-.*\"}, job)" ,
"refId": "VictoriaMetrics-job-Variable-Query"
},
"refresh": 1 ,
"regex": "" ,
"skipUrlSync": false ,
"sort": 0 ,
"tagValuesQuery": "" ,
"tagsQuery": "" ,
"type": "query" ,
"useTags": false
},
{
"current": {},
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"definition": "label_values(vm_app_version{job=~\"$job\", instance=~\"$instance\"}, version)" ,
"hide": 2 ,
"includeAll": false ,
"multi": false ,
"name": "version" ,
"options": [ ] ,
"query": {
"query": "label_values(vm_app_version{job=~\"$job\", instance=~\"$instance\"}, version)" ,
"refId": "VictoriaMetrics-version-Variable-Query"
},
"refresh": 1 ,
2023-10-21 01:03:39 +00:00
"regex": "/.*-tags-(v\\\\d+\\\\.\\\\d+\\\\.\\\\d+)/" ,
2023-05-04 20:17:01 +00:00
"skipUrlSync": false ,
"sort": 2 ,
"tagValuesQuery": "" ,
"tagsQuery": "" ,
"type": "query" ,
"useTags": false
},
{
"current": {},
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"definition": "label_values(vm_app_version{job=~\"$job\"}, instance)" ,
"hide": 0 ,
"includeAll": false ,
"multi": false ,
"name": "instance" ,
"options": [ ] ,
"query": {
"query": "label_values(vm_app_version{job=~\"$job\"}, instance)" ,
"refId": "VictoriaMetrics-instance-Variable-Query"
},
"refresh": 1 ,
"regex": "" ,
"skipUrlSync": false ,
"sort": 0 ,
"tagValuesQuery": "" ,
"tagsQuery": "" ,
"type": "query" ,
"useTags": false
},
{
"datasource": {
"type": "prometheus" ,
"uid": "$ds"
},
"filters": [ ] ,
"hide": 0 ,
"name": "adhoc" ,
"skipUrlSync": false ,
"type": "adhoc"
}
]
},
"time": {
"from": "now-30m" ,
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"10s" ,
"30s" ,
"1m" ,
"5m" ,
"15m" ,
"30m" ,
"1h" ,
"2h" ,
"1d"
] ,
"time_options": [
"5m" ,
"15m" ,
"1h" ,
"6h" ,
"12h" ,
"24h" ,
"2d" ,
"7d" ,
"30d"
]
},
"timezone": "" ,
"title": "VictoriaMetrics" ,
"uid": "wNf0q_kZk" ,
2023-10-21 01:03:39 +00:00
"version": 1 ,
2023-05-04 20:17:01 +00:00
"weekStart": ""
}
