ff7fd22ca2
This is an adapted version of this dashboard:
https://grafana.com/grafana/dashboards/3681
Change-Id: I405f09f75698b940becd6994a7fc457853603756
4670 lines
109 KiB
JSON
4670 lines
109 KiB
JSON
{
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"$$hashKey": "object:456",
|
|
"builtIn": 1,
|
|
"datasource": "-- Grafana --",
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"name": "Annotations & Alerts",
|
|
"type": "dashboard"
|
|
}
|
|
]
|
|
},
|
|
"description": "Dashboard providing insight to Prometheus monitoring system status and it's runtime info.",
|
|
"editable": true,
|
|
"gnetId": 3681,
|
|
"graphTooltip": 1,
|
|
"iteration": 1585226845807,
|
|
"links": [],
|
|
"panels": [
|
|
{
|
|
"content": "<img src=\"https://cdn.worldvectorlogo.com/logos/prometheus.svg\"/ height=\"140px\">",
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 2,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"height": "",
|
|
"id": 50,
|
|
"links": [],
|
|
"mode": "html",
|
|
"title": "",
|
|
"transparent": true,
|
|
"type": "text"
|
|
},
|
|
{
|
|
"cacheTimeout": null,
|
|
"columns": [],
|
|
"datasource": "Prometheus",
|
|
"fontSize": "100%",
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 22,
|
|
"x": 2,
|
|
"y": 0
|
|
},
|
|
"id": 41,
|
|
"links": [],
|
|
"pageSize": null,
|
|
"scroll": true,
|
|
"showHeader": true,
|
|
"sort": {
|
|
"col": 3,
|
|
"desc": false
|
|
},
|
|
"styles": [
|
|
{
|
|
"$$hashKey": "object:832",
|
|
"alias": "",
|
|
"align": "auto",
|
|
"colorMode": null,
|
|
"colors": [
|
|
"rgba(245, 54, 54, 0.9)",
|
|
"rgba(237, 129, 40, 0.89)",
|
|
"rgba(50, 172, 45, 0.97)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": 2,
|
|
"mappingType": 1,
|
|
"pattern": "Time",
|
|
"thresholds": [],
|
|
"type": "hidden",
|
|
"unit": "short"
|
|
},
|
|
{
|
|
"$$hashKey": "object:833",
|
|
"alias": "Prometheus instance",
|
|
"align": "auto",
|
|
"colorMode": null,
|
|
"colors": [
|
|
"rgba(245, 54, 54, 0.9)",
|
|
"rgba(237, 129, 40, 0.89)",
|
|
"rgba(50, 172, 45, 0.97)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": 2,
|
|
"link": true,
|
|
"linkTargetBlank": true,
|
|
"linkTooltip": "Link to the prometheus instance",
|
|
"linkUrl": "${__cell:raw}/graph",
|
|
"mappingType": 1,
|
|
"pattern": "instance",
|
|
"thresholds": [],
|
|
"type": "string",
|
|
"unit": "short"
|
|
},
|
|
{
|
|
"$$hashKey": "object:834",
|
|
"alias": "Uptime",
|
|
"align": "auto",
|
|
"colorMode": null,
|
|
"colors": [
|
|
"rgba(245, 54, 54, 0.9)",
|
|
"rgba(237, 129, 40, 0.89)",
|
|
"rgba(50, 172, 45, 0.97)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": 2,
|
|
"mappingType": 1,
|
|
"pattern": "Value #A",
|
|
"thresholds": [],
|
|
"type": "number",
|
|
"unit": "s"
|
|
},
|
|
{
|
|
"$$hashKey": "object:835",
|
|
"alias": "",
|
|
"align": "auto",
|
|
"colorMode": null,
|
|
"colors": [
|
|
"rgba(245, 54, 54, 0.9)",
|
|
"rgba(237, 129, 40, 0.89)",
|
|
"rgba(50, 172, 45, 0.97)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": 2,
|
|
"link": false,
|
|
"mappingType": 1,
|
|
"pattern": "__name__",
|
|
"thresholds": [],
|
|
"type": "hidden",
|
|
"unit": "short"
|
|
},
|
|
{
|
|
"$$hashKey": "object:836",
|
|
"alias": "Build git branch",
|
|
"align": "auto",
|
|
"colorMode": null,
|
|
"colors": [
|
|
"rgba(245, 54, 54, 0.9)",
|
|
"rgba(237, 129, 40, 0.89)",
|
|
"rgba(50, 172, 45, 0.97)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": 2,
|
|
"link": true,
|
|
"linkTargetBlank": true,
|
|
"linkTooltip": "Link to git branch of a build",
|
|
"linkUrl": "https://github.com/prometheus/prometheus/tree/${__cell}",
|
|
"mappingType": 1,
|
|
"pattern": "branch",
|
|
"thresholds": [],
|
|
"type": "string",
|
|
"unit": "short"
|
|
},
|
|
{
|
|
"$$hashKey": "object:837",
|
|
"alias": "",
|
|
"align": "auto",
|
|
"colorMode": null,
|
|
"colors": [
|
|
"rgba(245, 54, 54, 0.9)",
|
|
"rgba(237, 129, 40, 0.89)",
|
|
"rgba(50, 172, 45, 0.97)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": 2,
|
|
"mappingType": 1,
|
|
"pattern": "Value #B",
|
|
"thresholds": [],
|
|
"type": "hidden",
|
|
"unit": "short"
|
|
},
|
|
{
|
|
"$$hashKey": "object:838",
|
|
"alias": "Prometheus version",
|
|
"align": "auto",
|
|
"colorMode": null,
|
|
"colors": [
|
|
"rgba(245, 54, 54, 0.9)",
|
|
"rgba(237, 129, 40, 0.89)",
|
|
"rgba(50, 172, 45, 0.97)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": 2,
|
|
"link": true,
|
|
"linkTargetBlank": true,
|
|
"linkTooltip": "Prometheus release page",
|
|
"linkUrl": "https://github.com/prometheus/prometheus/releases/tag/v${__cell}",
|
|
"mappingType": 1,
|
|
"pattern": "version",
|
|
"thresholds": [],
|
|
"type": "string",
|
|
"unit": "short"
|
|
},
|
|
{
|
|
"$$hashKey": "object:839",
|
|
"alias": "Go version",
|
|
"align": "auto",
|
|
"colorMode": null,
|
|
"colors": [
|
|
"rgba(245, 54, 54, 0.9)",
|
|
"rgba(237, 129, 40, 0.89)",
|
|
"rgba(50, 172, 45, 0.97)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": 2,
|
|
"link": true,
|
|
"linkTargetBlank": true,
|
|
"linkTooltip": "Go release page",
|
|
"linkUrl": "https://github.com/golang/go/releases/tag/${__cell}",
|
|
"mappingType": 1,
|
|
"pattern": "goversion",
|
|
"thresholds": [],
|
|
"type": "string",
|
|
"unit": "short"
|
|
},
|
|
{
|
|
"$$hashKey": "object:840",
|
|
"alias": "Build git revision",
|
|
"align": "auto",
|
|
"colorMode": null,
|
|
"colors": [
|
|
"rgba(245, 54, 54, 0.9)",
|
|
"rgba(237, 129, 40, 0.89)",
|
|
"rgba(50, 172, 45, 0.97)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": 2,
|
|
"link": true,
|
|
"linkTargetBlank": true,
|
|
"linkTooltip": "Link to git commit of a build",
|
|
"linkUrl": "https://github.com/prometheus/prometheus/commit/${__cell}",
|
|
"mappingType": 1,
|
|
"pattern": "revision",
|
|
"thresholds": [],
|
|
"type": "string",
|
|
"unit": "short"
|
|
},
|
|
{
|
|
"$$hashKey": "object:841",
|
|
"alias": "",
|
|
"align": "auto",
|
|
"colorMode": null,
|
|
"colors": [
|
|
"rgba(245, 54, 54, 0.9)",
|
|
"rgba(237, 129, 40, 0.89)",
|
|
"rgba(50, 172, 45, 0.97)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": 2,
|
|
"mappingType": 1,
|
|
"pattern": "job",
|
|
"thresholds": [],
|
|
"type": "hidden",
|
|
"unit": "short"
|
|
},
|
|
{
|
|
"$$hashKey": "object:842",
|
|
"alias": "",
|
|
"align": "auto",
|
|
"colorMode": null,
|
|
"colors": [
|
|
"rgba(245, 54, 54, 0.9)",
|
|
"rgba(237, 129, 40, 0.89)",
|
|
"rgba(50, 172, 45, 0.97)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": 2,
|
|
"mappingType": 1,
|
|
"pattern": "",
|
|
"thresholds": [],
|
|
"type": "number",
|
|
"unit": "short"
|
|
},
|
|
{
|
|
"$$hashKey": "object:843",
|
|
"alias": "Last reload success",
|
|
"align": "auto",
|
|
"colorMode": "cell",
|
|
"colors": [
|
|
"#E02F44",
|
|
"#E02F44",
|
|
"rgba(50, 172, 45, 0)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": 0,
|
|
"link": true,
|
|
"linkTargetBlank": true,
|
|
"linkTooltip": "Config page",
|
|
"linkUrl": "https://${__cell_1:raw}/config",
|
|
"mappingType": 1,
|
|
"pattern": "Value #C",
|
|
"thresholds": [
|
|
"0",
|
|
"1"
|
|
],
|
|
"type": "number",
|
|
"unit": "none"
|
|
},
|
|
{
|
|
"$$hashKey": "object:844",
|
|
"alias": "Rules",
|
|
"align": "auto",
|
|
"colorMode": null,
|
|
"colors": [
|
|
"rgba(245, 54, 54, 0.9)",
|
|
"rgba(237, 129, 40, 0.89)",
|
|
"rgba(50, 172, 45, 0.97)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": 0,
|
|
"link": true,
|
|
"linkTargetBlank": true,
|
|
"linkTooltip": "Rules page",
|
|
"linkUrl": "https://${__cell_1:raw}/rules",
|
|
"mappingType": 1,
|
|
"pattern": "Value #D",
|
|
"thresholds": [],
|
|
"type": "number",
|
|
"unit": "short"
|
|
},
|
|
{
|
|
"$$hashKey": "object:845",
|
|
"alias": "Targets",
|
|
"align": "auto",
|
|
"colorMode": null,
|
|
"colors": [
|
|
"rgba(245, 54, 54, 0.9)",
|
|
"rgba(237, 129, 40, 0.89)",
|
|
"rgba(50, 172, 45, 0.97)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": null,
|
|
"link": true,
|
|
"linkTargetBlank": true,
|
|
"linkTooltip": "Targets page",
|
|
"linkUrl": "https://${__cell_1:raw}/rules",
|
|
"mappingType": 1,
|
|
"pattern": "Value #E",
|
|
"thresholds": [],
|
|
"type": "number",
|
|
"unit": "short"
|
|
},
|
|
{
|
|
"$$hashKey": "object:846",
|
|
"alias": "# series",
|
|
"align": "auto",
|
|
"colorMode": "cell",
|
|
"colors": [
|
|
"rgba(50, 172, 45, 0)",
|
|
"#FF780A",
|
|
"#C4162A"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": null,
|
|
"mappingType": 1,
|
|
"pattern": "Value #F",
|
|
"thresholds": [
|
|
"500000",
|
|
"1000000"
|
|
],
|
|
"type": "number",
|
|
"unit": "short"
|
|
},
|
|
{
|
|
"$$hashKey": "object:847",
|
|
"alias": "Time Range",
|
|
"align": "auto",
|
|
"colorMode": null,
|
|
"colors": [
|
|
"rgba(245, 54, 54, 0.9)",
|
|
"rgba(237, 129, 40, 0.89)",
|
|
"rgba(50, 172, 45, 0.97)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": null,
|
|
"mappingType": 1,
|
|
"pattern": "Value #G",
|
|
"thresholds": [],
|
|
"type": "number",
|
|
"unit": "s"
|
|
},
|
|
{
|
|
"$$hashKey": "object:848",
|
|
"alias": "Data",
|
|
"align": "auto",
|
|
"colorMode": null,
|
|
"colors": [
|
|
"rgba(245, 54, 54, 0.9)",
|
|
"rgba(237, 129, 40, 0.89)",
|
|
"rgba(50, 172, 45, 0.97)"
|
|
],
|
|
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
|
"decimals": null,
|
|
"mappingType": 1,
|
|
"pattern": "Value #H",
|
|
"thresholds": [],
|
|
"type": "number",
|
|
"unit": "bytes"
|
|
}
|
|
],
|
|
"targets": [
|
|
{
|
|
"expr": "max(time() - process_start_time_seconds{job=\"prometheus\"}) by (instance)",
|
|
"format": "table",
|
|
"instant": true,
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "max(prometheus_build_info{job=\"prometheus\"}) by (instance, branch, goversion, version, revision)",
|
|
"format": "table",
|
|
"instant": true,
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "max(prometheus_config_last_reload_successful{job=\"prometheus\"}) by (instance)",
|
|
"format": "table",
|
|
"instant": true,
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"expr": "max(sum(prometheus_rule_group_rules{job=\"prometheus\"}) by (instance) or prometheus_build_info{job=\"prometheus\"}*0) by (instance)",
|
|
"format": "table",
|
|
"instant": true,
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "D"
|
|
},
|
|
{
|
|
"expr": "max(sum(prometheus_sd_discovered_targets{job=\"prometheus\", name=\"scrape\"}) by (instance) or prometheus_build_info{job=\"prometheus\"}*0) by (instance)",
|
|
"format": "table",
|
|
"instant": true,
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "E"
|
|
},
|
|
{
|
|
"expr": "max(prometheus_tsdb_head_series{job=\"prometheus\"}) by (instance)",
|
|
"format": "table",
|
|
"instant": true,
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "F"
|
|
},
|
|
{
|
|
"expr": "time() - max(prometheus_tsdb_lowest_timestamp_seconds{job=\"prometheus\"}) by (instance)",
|
|
"format": "table",
|
|
"instant": true,
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "G"
|
|
},
|
|
{
|
|
"expr": "min(prometheus_tsdb_storage_blocks_bytes{job=\"prometheus\"}) by (instance)",
|
|
"format": "table",
|
|
"instant": true,
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "H"
|
|
}
|
|
],
|
|
"title": "Build, uptime and runtime instance info",
|
|
"transform": "table",
|
|
"type": "table"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 5
|
|
},
|
|
"id": 56,
|
|
"panels": [],
|
|
"repeat": null,
|
|
"title": "Main info",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"fill": 0,
|
|
"fillGradient": 1,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 6
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 71,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 2,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "max(prometheus_tsdb_head_series{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "{{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": "$long_term_interval",
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Number of time series",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"description": "",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"grid": {},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 7,
|
|
"x": 8,
|
|
"y": 6
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 25,
|
|
"legend": {
|
|
"alignAsTable": true,
|
|
"avg": true,
|
|
"current": true,
|
|
"max": true,
|
|
"min": false,
|
|
"show": false,
|
|
"sort": "max",
|
|
"sortDesc": true,
|
|
"total": false,
|
|
"values": true
|
|
},
|
|
"lines": true,
|
|
"linewidth": 2,
|
|
"links": [],
|
|
"nullPointMode": "connected",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "prometheus_target_interval_length_seconds{job=\"prometheus\",quantile=\"$quantile\"}",
|
|
"interval": "",
|
|
"legendFormat": "job {{job}} should be {{interval}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "$quantile quantile of interval length between scrapes per job",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"decimals": null,
|
|
"format": "s",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833"
|
|
},
|
|
"bars": true,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 6,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 9,
|
|
"x": 15,
|
|
"y": 6
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 13,
|
|
"legend": {
|
|
"alignAsTable": true,
|
|
"avg": true,
|
|
"current": true,
|
|
"hideEmpty": true,
|
|
"hideZero": true,
|
|
"max": true,
|
|
"min": false,
|
|
"rightSide": false,
|
|
"show": false,
|
|
"sort": "total",
|
|
"sortDesc": true,
|
|
"total": true,
|
|
"values": true
|
|
},
|
|
"lines": false,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [
|
|
{
|
|
"$$hashKey": "object:1744",
|
|
"alias": "/net_conntrack/"
|
|
}
|
|
],
|
|
"spaceLength": 10,
|
|
"stack": true,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(increase(label_replace({__name__=~\"(:?prometheus|net_conntrack)_.*(:?fail|duplicate|out_of|error|not_found|corrupt|dropped|missed|exceeded).*_total\", job=\"prometheus\"}, \"metric_name\", \"$1\", \"__name__\", \"(.*)\")[$aggregation_interval:20s])) by (instance,metric_name,alertmanager,name,mechanism,queue) > 0",
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Prometheus errors in $aggregation_interval",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:1755",
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:1756",
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 13
|
|
},
|
|
"id": 58,
|
|
"panels": [],
|
|
"repeat": null,
|
|
"title": "Requests & queries",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 0,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 14
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 18,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"hideEmpty": true,
|
|
"hideZero": true,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "connected",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": true,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(increase(prometheus_http_requests_total{job=\"prometheus\"}[$aggregation_interval])) by (instance, handler)",
|
|
"interval": "",
|
|
"legendFormat": "{{ handler }} handler on {{ instance }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "HTTP request count by handler in $aggregation_interval",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:1899",
|
|
"format": "none",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:1900",
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 14
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 16,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"hideEmpty": true,
|
|
"hideZero": true,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile($quantile, sum(increase(prometheus_http_request_duration_seconds_bucket{job=\"prometheus\"}[$aggregation_interval])) by (le,instance, handler))",
|
|
"interval": "",
|
|
"legendFormat": "{{ handler }} handler on {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "$quantile quantile of HTTP request duration per handler",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "s",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 14
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 19,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile($quantile, sum(increase(prometheus_http_response_size_bytes_bucket{job=\"prometheus\"}[$aggregation_interval])) by (le,instance, handler)) > 0",
|
|
"interval": "",
|
|
"legendFormat": "{{ handler }} handler on {{ instance }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "$quantile quantile of request size by handler",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 21
|
|
},
|
|
"id": 64,
|
|
"panels": [],
|
|
"repeat": null,
|
|
"title": "Resources",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"decimals": 2,
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 22
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 9,
|
|
"legend": {
|
|
"alignAsTable": false,
|
|
"avg": false,
|
|
"current": false,
|
|
"hideEmpty": false,
|
|
"max": false,
|
|
"min": false,
|
|
"rightSide": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(process_cpu_seconds_total{job=\"prometheus\"}[$aggregation_interval])) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "CPU usage of {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "CPU usage",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": [
|
|
"avg"
|
|
]
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "none",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Allocated bytes": "#7EB26D",
|
|
"Allocated bytes - 1m max": "#BF1B00",
|
|
"Allocated bytes - 1m min": "#BF1B00",
|
|
"Allocated bytes - 5m max": "#BF1B00",
|
|
"Allocated bytes - 5m min": "#BF1B00",
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833",
|
|
"RSS": "#447EBC"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"decimals": null,
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 22
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 6,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [
|
|
{
|
|
"alias": "//",
|
|
"fill": 0
|
|
},
|
|
{
|
|
"alias": "collector heap size",
|
|
"color": "#E0752D",
|
|
"linewidth": 2
|
|
},
|
|
{
|
|
"alias": "collector kubernetes memory limit",
|
|
"color": "#BF1B00",
|
|
"linewidth": 3
|
|
}
|
|
],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(process_resident_memory_bytes{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "Resident memory of {{instance}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "sum(go_memstats_alloc_bytes{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "Total allocated bytes of {{instance}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Memory usage",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Allocated bytes": "#F9BA8F",
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833",
|
|
"RSS": "#890F02"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 22
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 7,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "rate(go_memstats_alloc_bytes_total{job=\"prometheus\"}[$aggregation_interval])",
|
|
"interval": "",
|
|
"legendFormat": "Allocated Bytes of {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Allocations per second in $aggregation_interval",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 29
|
|
},
|
|
"id": 61,
|
|
"panels": [],
|
|
"repeat": null,
|
|
"title": "TSDB stats",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 0,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 30
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 89,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 2,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "time() - min(prometheus_tsdb_lowest_timestamp_seconds{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "Oldest data for {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": "$long_term_interval",
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Oldest data",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "s",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 0,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 30
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 88,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 2,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "max(prometheus_tsdb_storage_blocks_bytes{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": "$long_term_interval",
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Size of the storage",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 37
|
|
},
|
|
"id": 75,
|
|
"panels": [],
|
|
"title": "Query engine & API",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Allocated bytes": "#F9BA8F",
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max count collector": "#bf1b00",
|
|
"Max count harvester": "#bf1b00",
|
|
"Max to persist": "#3F6833",
|
|
"RSS": "#890F02"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 38
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 8,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [
|
|
{
|
|
"$$hashKey": "object:2178",
|
|
"alias": "/Limit/",
|
|
"color": "#E02F44",
|
|
"dashLength": 3,
|
|
"dashes": true,
|
|
"fill": 0,
|
|
"nullPointMode": "connected"
|
|
}
|
|
],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(prometheus_engine_queries{job=\"prometheus\"}) by (instance)",
|
|
"instant": false,
|
|
"interval": "",
|
|
"legendFormat": "{{instance}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "sum(prometheus_engine_queries_concurrent_max{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "{{instance}} Limit",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Number of concurent queries and it's limit",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:2199",
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:2200",
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 38
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 15,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "prometheus_engine_query_duration_seconds{job=\"prometheus\", quantile=\"$quantile\"}",
|
|
"interval": "",
|
|
"legendFormat": "{{slice}} slice on {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "$quantile quantile of query engine evaluation duration per slice",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "s",
|
|
"label": "",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 38
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 95,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(prometheus_api_remote_read_queries{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "{{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Number of queries on remote read API",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": "",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 45
|
|
},
|
|
"id": 57,
|
|
"panels": [],
|
|
"repeat": null,
|
|
"title": "Rule evaluation",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 0,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 9,
|
|
"x": 0,
|
|
"y": 46
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 77,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [
|
|
{
|
|
"alias": "/LIMIT/",
|
|
"color": "#E02F44",
|
|
"dashLength": 3,
|
|
"dashes": true,
|
|
"fill": 0
|
|
},
|
|
{
|
|
"alias": "slow",
|
|
"color": "#C4162A",
|
|
"fill": 8
|
|
}
|
|
],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "topk(20,max(prometheus_rule_group_last_duration_seconds{job=\"prometheus\"}) by (rule_group, instance) / min(prometheus_rule_group_interval_seconds{job=\"prometheus\"}) by (rule_group, instance))",
|
|
"interval": "",
|
|
"legendFormat": "Percentage of {{rule_group}} group evaluation interval on {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Percentage rule group evaluation duration from the rule group evaluation interval (top 20)",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "percentunit",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 9,
|
|
"y": 46
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 14,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [
|
|
{
|
|
"alias": "Queue length",
|
|
"yaxis": 2
|
|
}
|
|
],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(prometheus_rule_evaluation_duration_seconds{job=\"prometheus\", quantile=\"$quantile\"}) by (instance, quantile)",
|
|
"interval": "",
|
|
"legendFormat": "evaluation duration of {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "$quantile quantile of rule evaulation duration",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "s",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 3,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 7,
|
|
"x": 17,
|
|
"y": 46
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 76,
|
|
"interval": "",
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"hideEmpty": true,
|
|
"hideZero": true,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [
|
|
{
|
|
"alias": "Queue length",
|
|
"yaxis": 2
|
|
}
|
|
],
|
|
"spaceLength": 10,
|
|
"stack": true,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(topk(20,prometheus_rule_group_rules{job=\"prometheus\"})) by (instance, rule_group)",
|
|
"interval": "",
|
|
"legendFormat": "number of rules in {{rule_group}} rule group on {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Number of rules per group (top 20 groups)",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 53
|
|
},
|
|
"id": 59,
|
|
"panels": [],
|
|
"repeat": null,
|
|
"title": "Alerting",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Alert queue capacity on o collector": "#bf1b00",
|
|
"Alert queue capacity on o harvester": "#bf1b00",
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 54
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 20,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [
|
|
{
|
|
"alias": "/capacity/",
|
|
"color": "#E02F44",
|
|
"dashLength": 3,
|
|
"dashes": true,
|
|
"fill": 0,
|
|
"nullPointMode": "connected"
|
|
}
|
|
],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(prometheus_notifications_queue_length{job=\"prometheus\"})by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "Queue size of {{instance}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "sum(prometheus_notifications_queue_capacity{job=\"prometheus\"})by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "Queue capacity of {{instance}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Notification queue size and capacity",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Alert queue capacity on o collector": "#bf1b00",
|
|
"Alert queue capacity on o harvester": "#bf1b00",
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 54
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 78,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": true,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(increase(prometheus_notifications_sent_total{job=\"prometheus\"}[$aggregation_interval])) by (instance, alertmanager)",
|
|
"interval": "",
|
|
"legendFormat": "Sent notifications to {{alertmanager}} alertmanager from {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Number of sent notifictions per alertmanager in $aggregation_interval",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Alert queue capacity on o collector": "#bf1b00",
|
|
"Alert queue capacity on o harvester": "#bf1b00",
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 54
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 79,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"hideEmpty": true,
|
|
"hideZero": true,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "avg(prometheus_notifications_latency_seconds{job=\"prometheus\", quantile=\"$quantile\"}) by (instance, alertmanager)",
|
|
"interval": "",
|
|
"legendFormat": "Latency of notifications sent to {{alertmanager}} from {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "$quantile of notification latency per alertmanager",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "s",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 61
|
|
},
|
|
"id": 60,
|
|
"panels": [],
|
|
"repeat": null,
|
|
"title": "Service discovery",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"decimals": 0,
|
|
"fill": 5,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 62
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 69,
|
|
"legend": {
|
|
"alignAsTable": true,
|
|
"avg": false,
|
|
"current": true,
|
|
"hideEmpty": true,
|
|
"hideZero": true,
|
|
"max": false,
|
|
"min": false,
|
|
"rightSide": true,
|
|
"show": false,
|
|
"sort": "current",
|
|
"sortDesc": true,
|
|
"total": false,
|
|
"values": true
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": true,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(prometheus_sd_discovered_targets{job=\"prometheus\", name=\"scrape\"}) by (instance, config,name)",
|
|
"interval": "",
|
|
"intervalFactor": 5,
|
|
"legendFormat": "Number of {{name}} type targets from config {{config}} on {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Number of discovered targets per Type and config",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:2495",
|
|
"decimals": 0,
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:2496",
|
|
"decimals": 0,
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 62
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 45,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(increase(prometheus_sd_refresh_duration_seconds_count{job=\"prometheus\"}[$aggregation_interval])) by (instance, mechanism)",
|
|
"interval": "",
|
|
"legendFormat": "Number of sync for {{mechanism}} SD mechanism on {{instance}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "sum(increase(prometheus_sd_consul_rpc_duration_seconds_count{job=\"prometheus\"}[$aggregation_interval])) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "Number of sync for consul SD mechanism on {{instance}}",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "sum(increase(prometheus_sd_kubernetes_http_request_duration_seconds_count{job=\"prometheus\"}[$aggregation_interval])) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "Number of sync for kubernetes SD mechanism on {{instance}}",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Service discovery sync count by mechanism in $aggregation_interval",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 62
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 43,
|
|
"legend": {
|
|
"alignAsTable": true,
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"rightSide": true,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "prometheus_sd_refresh_duration_seconds{job=\"prometheus\", quantile=\"$quantile\"}",
|
|
"interval": "",
|
|
"legendFormat": "Duration of sync for {{mechanism}} SD mechanism on {{instance}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "consul_rpc_duration_seconds{job=\"prometheus\", quantile=\"$quantile\"}",
|
|
"interval": "",
|
|
"legendFormat": "Number of sync for consul SD mechanism on {{instance}}",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "prometheus_sd_file_scan_duration_seconds{job=\"prometheus\", quantile=\"$quantile\"}",
|
|
"interval": "",
|
|
"legendFormat": "Duration of sync for file_scan SD mechanism on {{instance}}",
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"expr": "prometheus_sd_kubernetes_http_request_duration_seconds_sum{job=\"prometheus\"} / prometheus_sd_kubernetes_http_request_duration_seconds_count{job=\"prometheus\"}",
|
|
"interval": "",
|
|
"legendFormat": "AVG Duration of sync for k8s SD mechanism on {{instance}}",
|
|
"refId": "D"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "$quantile quantile of refresh duration per SD mechanism",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "s",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 69
|
|
},
|
|
"id": 73,
|
|
"panels": [],
|
|
"title": "Compaction and retention",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": true,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 70
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 80,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": false,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 1,
|
|
"points": true,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile($quantile, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{job=\"prometheus\"}[$aggregation_interval])) by (instance, le))",
|
|
"interval": "",
|
|
"legendFormat": "Compaction duration on {{ instance }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "$quantile quantile of compaction duration",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"decimals": null,
|
|
"format": "s",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"fill": 3,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 70
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 81,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": true,
|
|
"steppedLine": true,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(increase(prometheus_tsdb_compactions_total{job=\"prometheus\"}[$aggregation_interval])) by (instance) > 0",
|
|
"interval": "",
|
|
"legendFormat": "Count of horizontal compactions on {{instance}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "sum(increase(prometheus_tsdb_vertical_compactions_total{job=\"prometheus\"}[$aggregation_interval])) by (instance) > 0",
|
|
"interval": "",
|
|
"legendFormat": "Count of vertical compactions on {{instance}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Number of vertical/horizontal compactions in last $aggregation_interval",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"decimals": null,
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"fill": 3,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 70
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 82,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": true,
|
|
"steppedLine": true,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(increase(prometheus_tsdb_size_retentions_total{job=\"prometheus\"}[$aggregation_interval])) by (instance) > 0",
|
|
"interval": "",
|
|
"legendFormat": "Number of size retention cutoffs on {{ instance }}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "sum(increase(prometheus_tsdb_time_retentions_total{job=\"prometheus\"}[$aggregation_interval])) by (instance) > 0",
|
|
"interval": "",
|
|
"legendFormat": "Number of time retention cutoffs on {{ instance }}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Number of time/size retention cutoffs in $aggregation_interval",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"decimals": null,
|
|
"format": "s",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 77
|
|
},
|
|
"id": 62,
|
|
"panels": [],
|
|
"repeat": null,
|
|
"title": "Head block",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833",
|
|
"To persist": "#9AC48A"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 78
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 2,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [
|
|
{
|
|
"alias": "/Max.*/",
|
|
"fill": 0
|
|
}
|
|
],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(prometheus_tsdb_head_chunks{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "Head block chunk count on {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Head chunks count",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 78
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 1,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\"}[$aggregation_interval])) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "Appended samples/s on {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Samples Appended per second in $aggregation_interval",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": "",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 0,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 78
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 3,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 2,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "prometheus_tsdb_head_series{job=\"prometheus\"}",
|
|
"interval": "",
|
|
"legendFormat": "Time series count of {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Time series total count",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:262",
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:263",
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 85
|
|
},
|
|
"id": 84,
|
|
"panels": [],
|
|
"title": "WAL",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833",
|
|
"To persist": "#9AC48A"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 86
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 85,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(prometheus_tsdb_wal_fsync_duration_seconds{job=\"prometheus\",quantile=\"$quantile\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "Duration of WAL fsync on {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "$quantile quantile of WAL fsync duration",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "s",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833",
|
|
"To persist": "#9AC48A"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 0,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 86
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 86,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(increase(prometheus_tsdb_wal_completed_pages_total{job=\"prometheus\"}[$aggregation_interval])) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "Completed WAL pages count of {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Number of completed pages in $aggregation_interval",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833",
|
|
"To persist": "#9AC48A"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 86
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 87,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(prometheus_tsdb_wal_truncate_duration_seconds_sum{job=\"prometheus\"} /prometheus_tsdb_wal_truncate_duration_seconds_count{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "Duration of WAL truncation on {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Average duration of WAL truncation",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "s",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 93
|
|
},
|
|
"id": 99,
|
|
"panels": [],
|
|
"title": "Go stats",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833",
|
|
"To persist": "#9AC48A"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 0,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 94
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 100,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(go_goroutines{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "Gorutines count of {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Number of gorutines",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833",
|
|
"To persist": "#9AC48A"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 0,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 94
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 101,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "max(go_gc_duration_seconds{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "GC duration on {{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Duration of Go garbage collection",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "s",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833",
|
|
"To persist": "#9AC48A"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 5,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 94
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 102,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": true,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "max(go_memstats_mspan_sys_bytes{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "mspan {{instance}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "max(go_memstats_mcache_sys_bytes{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "mcache {{instance}}",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "max(go_memstats_buck_hash_sys_bytes{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "buck_hash {{instance}}",
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"expr": "max(go_memstats_gc_sys_bytes{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "gc {{instance}}",
|
|
"refId": "D"
|
|
},
|
|
{
|
|
"expr": "max(go_memstats_other_sys_bytes{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "other {{instance}}",
|
|
"refId": "E"
|
|
},
|
|
{
|
|
"expr": "max(go_memstats_stack_sys_bytes{job=\"prometheus\"}) by (instance)",
|
|
"format": "time_series",
|
|
"hide": false,
|
|
"intervalFactor": 1,
|
|
"legendFormat": "stack {{instance}}",
|
|
"metric": "prometheus_local_storage_memory_chunks",
|
|
"refId": "F"
|
|
},
|
|
{
|
|
"expr": "max(go_memstats_heap_sys_bytes{job=\"prometheus\"}) by (instance)",
|
|
"interval": "",
|
|
"legendFormat": "heap {{instance}}",
|
|
"refId": "G"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Go system memory allocations",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:2633",
|
|
"format": "bytes",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:2634",
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 101
|
|
},
|
|
"id": 68,
|
|
"panels": [],
|
|
"repeat": null,
|
|
"title": "Network",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 102
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 47,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [
|
|
{
|
|
"alias": "/closed/",
|
|
"transform": "negative-Y"
|
|
}
|
|
],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(increase(net_conntrack_listener_conn_accepted_total{job=\"prometheus\"}[$aggregation_interval])) by (instance, listener_name) > 0",
|
|
"interval": "",
|
|
"legendFormat": "{{listener_name}} accepted connections on {{instance}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "sum(increase(net_conntrack_listener_conn_closed_total{job=\"prometheus\"}[$aggregation_interval])) by (instance, listener_name) > 0",
|
|
"interval": "",
|
|
"legendFormat": "{{listener_name}} closed connections on {{instance}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Accepted/closed inbound connections per listener",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {
|
|
"Chunks": "#1F78C1",
|
|
"Chunks to persist": "#508642",
|
|
"Max chunks": "#052B51",
|
|
"Max to persist": "#3F6833"
|
|
},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "Prometheus",
|
|
"editable": true,
|
|
"error": false,
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 102
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 90,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": false,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"links": [],
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"dataLinks": []
|
|
},
|
|
"percentage": false,
|
|
"pointradius": 5,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [
|
|
{
|
|
"alias": "/closed/",
|
|
"stack": "B",
|
|
"transform": "negative-Y"
|
|
},
|
|
{
|
|
"alias": "/established/",
|
|
"stack": "A"
|
|
}
|
|
],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(increase(net_conntrack_dialer_conn_established_total{job=\"prometheus\"}[$aggregation_interval])) by (instance, dialer_name) > 0",
|
|
"interval": "",
|
|
"legendFormat": "{{dialer_name}} established connections on {{instance}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "sum(increase(net_conntrack_dialer_conn_closed_total{job=\"prometheus\"}[$aggregation_interval])) by (instance, dialer_name) > 0",
|
|
"interval": "",
|
|
"legendFormat": "{{dialer_name}} closed connections on {{instance}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Established/closed outbound connections per dialer ",
|
|
"tooltip": {
|
|
"msResolution": false,
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
}
|
|
],
|
|
"refresh": "1m",
|
|
"schemaVersion": 22,
|
|
"style": "dark",
|
|
"tags": [
|
|
"prometheus",
|
|
"monitoring"
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"auto": true,
|
|
"auto_count": 300,
|
|
"auto_min": "1m",
|
|
"current": {
|
|
"selected": false,
|
|
"text": "1d",
|
|
"value": "1d"
|
|
},
|
|
"hide": 0,
|
|
"label": "Aggregation intarval",
|
|
"name": "aggregation_interval",
|
|
"options": [
|
|
{
|
|
"selected": false,
|
|
"text": "auto",
|
|
"value": "$__auto_interval_aggregation_interval"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "1m",
|
|
"value": "1m"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "10m",
|
|
"value": "10m"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "30m",
|
|
"value": "30m"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "1h",
|
|
"value": "1h"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "6h",
|
|
"value": "6h"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "12h",
|
|
"value": "12h"
|
|
},
|
|
{
|
|
"selected": true,
|
|
"text": "1d",
|
|
"value": "1d"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "7d",
|
|
"value": "7d"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "14d",
|
|
"value": "14d"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "30d",
|
|
"value": "30d"
|
|
}
|
|
],
|
|
"query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
|
|
"refresh": 2,
|
|
"skipUrlSync": false,
|
|
"type": "interval"
|
|
},
|
|
{
|
|
"allValue": null,
|
|
"current": {
|
|
"text": "0.99",
|
|
"value": "0.99"
|
|
},
|
|
"datasource": "Prometheus",
|
|
"definition": "query_result({__name__=~\"prometheus_.*_seconds\"})",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"index": -1,
|
|
"label": "Quantile",
|
|
"multi": false,
|
|
"name": "quantile",
|
|
"options": [],
|
|
"query": "query_result({__name__=~\"prometheus_.*_seconds\"})",
|
|
"refresh": 2,
|
|
"regex": "/quantile=\"([^\"]+)\"/",
|
|
"skipUrlSync": false,
|
|
"sort": 1,
|
|
"tagValuesQuery": "",
|
|
"tags": [],
|
|
"tagsQuery": "",
|
|
"type": "query",
|
|
"useTags": false
|
|
},
|
|
{
|
|
"auto": false,
|
|
"auto_count": 30,
|
|
"auto_min": "10s",
|
|
"current": {
|
|
"selected": false,
|
|
"text": "3d",
|
|
"value": "3d"
|
|
},
|
|
"hide": 0,
|
|
"label": null,
|
|
"name": "long_term_interval",
|
|
"options": [
|
|
{
|
|
"selected": false,
|
|
"text": "10m",
|
|
"value": "10m"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "30m",
|
|
"value": "30m"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "1h",
|
|
"value": "1h"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "6h",
|
|
"value": "6h"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "12h",
|
|
"value": "12h"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "1d",
|
|
"value": "1d"
|
|
},
|
|
{
|
|
"selected": true,
|
|
"text": "3d",
|
|
"value": "3d"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "1w",
|
|
"value": "1w"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "4w",
|
|
"value": "4w"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "12w",
|
|
"value": "12w"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "1y",
|
|
"value": "1y"
|
|
}
|
|
],
|
|
"query": "10m,30m,1h,6h,12h,1d,3d,1w,4w,12w,1y",
|
|
"refresh": 2,
|
|
"skipUrlSync": false,
|
|
"type": "interval"
|
|
}
|
|
]
|
|
},
|
|
"time": {
|
|
"from": "now-7d",
|
|
"to": "now"
|
|
},
|
|
"timepicker": {
|
|
"refresh_intervals": [
|
|
"5s",
|
|
"10s",
|
|
"30s",
|
|
"1m",
|
|
"5m",
|
|
"15m",
|
|
"30m",
|
|
"1h",
|
|
"2h",
|
|
"1d"
|
|
],
|
|
"time_options": [
|
|
"5m",
|
|
"15m",
|
|
"1h",
|
|
"6h",
|
|
"12h",
|
|
"24h",
|
|
"2d",
|
|
"7d",
|
|
"30d"
|
|
]
|
|
},
|
|
"timezone": "browser",
|
|
"title": "Prometheus",
|
|
"uid": "iLC3yp9Zz",
|
|
"variables": {
|
|
"list": []
|
|
},
|
|
"version": 3
|
|
}
|