danderson/infra
0
0
Fork 0
forked from the-distro/infra
Code Pull requests Activity

Add nginx log exporter

Browse source
This commit is contained in:
Ilya K 2024-07-05 15:18:23 +03:00 committed by Ilya K
parent 63b31e98cf
commit 42f8ad8fa4
3 changed files with 738 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
services/monitoring/exporters/default.nix
View file

@ -7,7 +7,10 @@ let
inherit (lib) mkOption types;
in
{
imports = [./baseline.nix];
imports = [
./baseline.nix
./nginx.nix
];
options.bagel = {
meta.monitoring = {

37
services/monitoring/exporters/nginx.nix Normal file
View file

@ -0,0 +1,37 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.monitoring.exporters.nginx;
inherit (lib) mkEnableOption mkIf;
logFormat = ''$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'';
in
{
options.bagel.monitoring.exporters.nginx.enable = (mkEnableOption "Nginx access.log exporter") // { default = config.services.nginx.enable; };
config = mkIf cfg.enable {
services.nginx.appendHttpConfig = ''
log_format ours '${logFormat}';
access_log /var/log/nginx/access.log ours;
'';
services.prometheus.exporters.nginxlog = {
enable = true;
port = 9103;
group = "nginx";
settings.namespaces = [
{
name = "nginx";
format = logFormat;
source.files = ["/var/log/nginx/access.log"];
}
];
};
bagel.meta.monitoring.exporters = [
{ port = 9103; }
];
};
}

697
services/monitoring/lgtm/dashboards/nginx.json Normal file
View file

@ -0,0 +1,697 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"description": "NGINX Log metrics with Prometheus based on https://github.com/martin-helmich/prometheus-nginxlog-exporter \r\nBased on namespace prefix 'nginx'. If different, you may need to adjust the metrics.\r\nDashboard based on 6482 dashboard",
"editable": true,
"fiscalYearStartMonth": 0,
"gnetId": 15947,
"graphTooltip": 0,
"id": 4,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-GrYlRd"
},
"mappings": [],
"thresholds": {
"mode": "percentage",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 5
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 11,
"x": 0,
"y": 0
},
"id": 12,
"options": {
"displayMode": "lcd",
"maxVizHeight": 300,
"minVizHeight": 16,
"minVizWidth": 8,
"namePlacement": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": true,
"sizing": "auto",
"text": {},
"valueMode": "color"
},
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"editorMode": "code",
"exemplar": true,
"expr": "sum(rate(nginx_http_response_count_total{status=~\"^2..\",instance=\"$host\"}[1m])) / sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) * 100",
"hide": false,
"interval": "",
"legendFormat": "2** status codes",
"range": true,
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_count_total{status=~\"^4..\",instance=\"$host\"}[1m])) / sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) * 100",
"interval": "",
"legendFormat": "4** status codes",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_count_total{status=~\"^5..\",instance=\"$host\"}[1m])) / sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) * 100",
"hide": false,
"interval": "",
"legendFormat": "5** status codes",
"refId": "B"
}
],
"title": "Percentage Ratio of status codes to all status codes",
"transparent": true,
"type": "bargauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 4,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 11,
"x": 11,
"y": 0
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "8.5.0-54880pre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_time_seconds_count{instance=\"$host\"}[1m])) ",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "$host",
"refId": "A"
}
],
"title": "Requests per Second",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"description": "Response sizes in bytes",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 4,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 11,
"x": 11,
"y": 7
},
"id": 8,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "8.5.0-54880pre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_size_bytes{instance=\"$host\"}[5m])) ",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "$host",
"refId": "A"
}
],
"title": "HTTP Traffic",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 21,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 4,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 11,
"x": 0,
"y": 8
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.5.0-54880pre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_time_seconds_sum{instance=\"$host\"}[5m])) / sum(rate(nginx_http_response_time_seconds_count{instance=\"$host\"}[5m])) ",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "$host",
"refId": "A"
}
],
"title": "Average Response Time [5m]",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 3,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 11,
"x": 11,
"y": 14
},
"id": 10,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "8.5.0-54880pre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) by (status)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "",
"refId": "A"
}
],
"title": "Status codes per second",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 4,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 11,
"x": 0,
"y": 15
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "8.5.0-54880pre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "nginx_http_response_time_seconds{quantile=\"0.9\",method=\"GET\",status=~\"2[0-9]*\",instance=~\"$host\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "$host",
"refId": "A"
}
],
"title": "Response time (90% quantile)",
"type": "timeseries"
}
],
"refresh": "5s",
"schemaVersion": 39,
"tags": [],
"templating": {
"list": [
{
"current": {
"isNone": true,
"selected": false,
"text": "None",
"value": ""
},
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"definition": "label_values(nginx_http_response_count_total,instance)",
"hide": 0,
"includeAll": false,
"label": "Host:",
"multi": false,
"name": "host",
"options": [],
"query": {
"query": "label_values(nginx_http_response_count_total,instance)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-15m",
"to": "now"
},
"timeRangeUpdatedDuringEditOrView": false,
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "NGINX Log Metrics [M]",
"uid": "JfOTY2Pnk",
"version": 4,
"weekStart": ""
}