the-distro/infra
9
0
Fork 5
Code Issues 39 Pull requests 8 Packages Projects 1 Releases Wiki Activity

Grafana #24

Merged
raito merged 6 commits from grafana into main 2024-07-05 16:43:17 +00:00
Conversation 0 Commits 6 Files changed 30 +35725 -13
30 changed files with 35725 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.envrc Normal file
View file

@ -0,0 +1,2 @@
# shellcheck shell=bash
use flake

1
.gitignore vendored
View file

@ -1,3 +1,4 @@
result
.gcroots
config.tf.json
.direnv

27
common/admins.nix
View file

@ -1,13 +1,16 @@
{
users.users.root.openssh.authorizedKeys.keys = [
# delroth
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV"
# raito
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM="
# k900
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun"
];
let
keys = import ./ssh-keys.nix;
in {
users.users.root.openssh.authorizedKeys.keys =
keys.users.delroth ++
keys.users.k900 ++
keys.users.raito ++
keys.users.maxine ++
[
# more raito
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM="
];
}

3
common/ssh-keys.nix
View file

@ -2,10 +2,13 @@
machines = {
bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsO4bNqY04uG13Pg3ubHfRDssTphDLzZ4YUniE5/p+M";
meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
gerrit01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+eSZu+u9sCynrMlsmFzQHLIELQAuVg0Cs1pBvwb4+A";
};
users = {
delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ];
raito = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp" ];
k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
};
}

10
flake.nix
View file

@ -57,7 +57,15 @@
'');
};
};
defaultApp.${system} = self.apps.${system}.apply;
apps.${system}.default = self.apps.${system}.apply;
devShells.${system}.default = pkgs.mkShell {
packages = [
inputs.agenix.packages.${system}.agenix
inputs.colmena.packages.${system}.colmena
];
};
colmena = {
meta.nixpkgs = import nixpkgs {
localSystem = system;

1
hosts/bagel-box/default.nix
View file

@ -40,6 +40,7 @@
hydra.enable = true;
hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
};
bagel.meta.monitoring.address = "bagel-box.delroth.net";
security.acme.acceptTerms = true;
security.acme.defaults.email = "bagel@delroth.net";

1
hosts/cl.forkos.org/default.nix
View file

@ -24,6 +24,7 @@
};
};
};
bagel.meta.monitoring.address = "gerrit01.infra.forkos.org";
fileSystems."/gerrit-data" = {
device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";

4
hosts/meta01.nixpkgs.lahfa.xyz/default.nix
View file

@ -21,6 +21,10 @@
enable = true;
domain = "netbox.forkos.org";
};
bagel.meta.monitoring.address = "meta01.infra.forkos.org";
bagel.services.prometheus.enable = true;
bagel.services.loki.enable = true;
bagel.services.grafana.enable = true;
i18n.defaultLocale = "fr_FR.UTF-8";

7
secrets.nix
View file

@ -7,6 +7,13 @@ let
hydra-s3-credentials = [ machines.bagel-box ];
hydra-ssh-key-priv = [ machines.bagel-box ];
netbox-environment = [ machines.meta01 ];
mimir-environment = [ machines.meta01 ];
grafana-oauth-secret = [ machines.meta01 ];
loki-environment = [ machines.meta01 ];
# These are the same password, but nginx wants it in htpasswd format
loki-htpasswd = [ machines.meta01 ];
promtail-password = builtins.attrValues machines;
};
in
builtins.listToAttrs (

7
secrets/grafana-oauth-secret.age Normal file
View file

@ -0,0 +1,7 @@
age-encryption.org/v1
-> ssh-ed25519 j2r2qQ Xl0fSOuF0xNTJrtVGdRLRIszd15LFrG5KCFNvSBK4Go
qSEMBBw90jz4j8elpoUeyS4CTLBhZtNDhLNigesJq+0
-> ssh-ed25519 K3b7BA cKI0twKiuuTKv1Js4jqt5v8cOqpxEMY9dmVghgJtbzw
K5o31XP/nLsswsrMaxnIzCXVUtJqmJWoFglWFsV7+AQ
--- X8pvqCHeCQ0LjzcjIHThkqp6YeOOT8dBMLuktgdgeY4
sZÓ¸ŠíØ[þ²X<C2B2>“¡èÅ®Š5°=÷6)ÇT¿Q†‘N{•x³I1ƒ!Ó–ÜøB ƒzš*×íåL~K

9
secrets/loki-environment.age Normal file
View file

@ -0,0 +1,9 @@
age-encryption.org/v1
-> ssh-ed25519 j2r2qQ w0lLquFUUcmEZ/Fh1YSt85tAJkBwavORQbwMr7gMqF4
J4T+EHm1uHbCZkAUNoNcB9uGSz082mFL8+dkCnvYQnM
-> ssh-ed25519 K3b7BA 28bJZgBPPc2KIE5+b8LJuQ5L4YAiRAJzucEuOqXHdVM
7hKENFr8QX0jpwuuQEjGFrUywJuhL1Tdi2V4/gR8JWE
--- GSPZxz39TMMWv0qhotNgnXa5679Q7VK8JGjQjI7A8oM
J²\@F“N• ³å…2‹®ô¨w×!¯1Vf»§˜Ž·’ÞO²CÓw®®V°£šÌº.^݆ 7<C2A0>w‡n4äàdW-Ö¾"@0¨ú¹EÏ¿·°ck,]M}x<>øÌ<C3B8>¡Ûy°[×ÁJ:!è‘ !ø螀c¬
Bë¹R
nøê€þÀáÆ^9í¤–M<ú

7
secrets/loki-htpasswd.age Normal file
View file

@ -0,0 +1,7 @@
age-encryption.org/v1
-> ssh-ed25519 j2r2qQ nLWy3WcVJWCl3rXkhcSbp1joqmkk06QnxhCZ4UtSvmw
iQ+Hx/vhiFgkWfbxHwGjxMBEqzyGww4/9do3W7V/y1Y
-> ssh-ed25519 K3b7BA RkF2ADcjOGtivl9MrhO/HFwxlTAkbFHWL3iinUldMiM
7q/zdVTMLevukZjkHtcN88iYzfTLvq2s3QdkgsFSO9M
--- 1b2HiK06vJPqBgHVDD0QELOtfkl7/rlgGS9uI1mSbus
„uܧoL;õå¬" 4¦Û»Z¼˜@§öãƒÐ’3+93Q4óÄ o•ŒØwé“„6<>M-²DkJn´;ñ*g <0A>OÛYœ75ËSò)Ù°©

BIN
secrets/mimir-environment.age Normal file
View file

Binary file not shown.

12
secrets/promtail-password.age Normal file
View file

@ -0,0 +1,12 @@
age-encryption.org/v1
-> ssh-ed25519 +HUDfA ZUM0ACC/NIekvX1PkCiXTHaTeE3ybudmY3piHw2iekQ
cHj94FIR6gNJ3Hw9FI7K15OYgxbjkajGtCftD+2Mr8c
-> ssh-ed25519 2D+APA tzlyOnAXnLxXO/47n45sFPiJF3FXd98UU5ajPhD2wSs
P8ZdUiBeME17SU2BpMgOq4plyAqgzLOQWHa1+Q7cjYo
-> ssh-ed25519 j2r2qQ 3OikD9JOmug7kdPAPz+JT/ryB6xBQhu2+cwS9h5sKGI
XiIuxOyey2I6hmqabUCPzLc85q/1r9OwVGjHWYNQsp0
-> ssh-ed25519 K3b7BA Bdqcqt4GgLzuSiEnIyImDiOQGwyIhhozRXMmNrp7glI
65joZcnl0Hqe90Th2EdVgbcxUJFpy3fOgk6oPiSHh2A
--- 6x6BFNypc+u3DpsHX3SajwEy1TqsAtbFei0ddRpEoBg
äªUG¾xj4»®Îþ‡b=óžóñ¼Rd<52>3sHYÝ Ô<>*Qµ9Ã6‚n‹34&äw›»~h!§ ^„[êš

1
services/default.nix
View file

@ -4,5 +4,6 @@
./postgres
./netbox
./gerrit
./monitoring
];
}

7
services/monitoring/default.nix Normal file
View file

@ -0,0 +1,7 @@
{
imports = [
./exporters
./lgtm
./promtail.nix
];
}

34
services/monitoring/exporters/baseline.nix Normal file
View file

@ -0,0 +1,34 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.monitoring.exporters.baseline;
inherit (lib) mkEnableOption mkIf;
in
{
options.bagel.monitoring.exporters.baseline.enable = (mkEnableOption "Standard set of exporters") // { default = true; };
config = mkIf cfg.enable {
services.prometheus.exporters.node = {
enable = true;
enabledCollectors = [
"processes"
"systemd"
];
port = 9101;
};
services.cadvisor = {
enable = true;
port = 9102;
listenAddress = "0.0.0.0";
};
bagel.meta.monitoring.exporters = [
{ port = 9101; }
{ port = 9102; }
];
};
}

36
services/monitoring/exporters/default.nix Normal file
View file

@ -0,0 +1,36 @@
{
config,
lib,
...
}:
let
inherit (lib) mkOption types;
in
{
imports = [
./baseline.nix
./nginx.nix
./postgres.nix
];
options.bagel = {
meta.monitoring = {
address = mkOption {
description = "Node's public address";
type = types.str;
};
exporters = mkOption {
description = "List of all exporters to scrape";
type = types.listOf (types.submodule {
options.port = mkOption {
description = "Exporter port";
type = types.int;
};
});
default = [];
};
};
};
config.networking.firewall.allowedTCPPorts = map (e: e.port) config.bagel.meta.monitoring.exporters;
}

37
services/monitoring/exporters/nginx.nix Normal file
View file

@ -0,0 +1,37 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.monitoring.exporters.nginx;
inherit (lib) mkEnableOption mkIf;
logFormat = ''$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'';
in
{
options.bagel.monitoring.exporters.nginx.enable = (mkEnableOption "Nginx access.log exporter") // { default = config.services.nginx.enable; };
config = mkIf cfg.enable {
services.nginx.appendHttpConfig = ''
log_format ours '${logFormat}';
access_log /var/log/nginx/access.log ours;
'';
services.prometheus.exporters.nginxlog = {
enable = true;
port = 9103;
group = "nginx";
settings.namespaces = [
{
name = "nginx";
format = logFormat;
source.files = ["/var/log/nginx/access.log"];
}
];
};
bagel.meta.monitoring.exporters = [
{ port = 9103; }
];
};
}

31
services/monitoring/exporters/postgres.nix Normal file
View file

@ -0,0 +1,31 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.monitoring.exporters.postgres;
inherit (lib) mkEnableOption mkIf;
in
{
options.bagel.monitoring.exporters.postgres.enable = (mkEnableOption "Postgres exporter") // { default = config.services.postgresql.enable; };
config = mkIf cfg.enable {
services.prometheus.exporters.postgres = {
enable = true;
port = 9104;
runAsLocalSuperUser = true;
extraFlags = [
"--collector.long_running_transactions"
"--collector.stat_activity_autovacuum"
"--collector.stat_statements"
];
};
services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
bagel.meta.monitoring.exporters = [
{ port = 9104; }
];
};
}

0
services/monitoring/lgtm/alerts/.gitkeep Normal file
View file

697
services/monitoring/lgtm/dashboards/nginx.json Normal file
View file

@ -0,0 +1,697 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"description": "NGINX Log metrics with Prometheus based on https://github.com/martin-helmich/prometheus-nginxlog-exporter \r\nBased on namespace prefix 'nginx'. If different, you may need to adjust the metrics.\r\nDashboard based on 6482 dashboard",
"editable": true,
"fiscalYearStartMonth": 0,
"gnetId": 15947,
"graphTooltip": 0,
"id": 4,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-GrYlRd"
},
"mappings": [],
"thresholds": {
"mode": "percentage",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 5
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 11,
"x": 0,
"y": 0
},
"id": 12,
"options": {
"displayMode": "lcd",
"maxVizHeight": 300,
"minVizHeight": 16,
"minVizWidth": 8,
"namePlacement": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": true,
"sizing": "auto",
"text": {},
"valueMode": "color"
},
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"editorMode": "code",
"exemplar": true,
"expr": "sum(rate(nginx_http_response_count_total{status=~\"^2..\",instance=\"$host\"}[1m])) / sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) * 100",
"hide": false,
"interval": "",
"legendFormat": "2** status codes",
"range": true,
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_count_total{status=~\"^4..\",instance=\"$host\"}[1m])) / sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) * 100",
"interval": "",
"legendFormat": "4** status codes",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_count_total{status=~\"^5..\",instance=\"$host\"}[1m])) / sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) * 100",
"hide": false,
"interval": "",
"legendFormat": "5** status codes",
"refId": "B"
}
],
"title": "Percentage Ratio of status codes to all status codes",
"transparent": true,
"type": "bargauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 4,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 11,
"x": 11,
"y": 0
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "8.5.0-54880pre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_time_seconds_count{instance=\"$host\"}[1m])) ",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "$host",
"refId": "A"
}
],
"title": "Requests per Second",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"description": "Response sizes in bytes",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 4,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 11,
"x": 11,
"y": 7
},
"id": 8,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "8.5.0-54880pre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_size_bytes{instance=\"$host\"}[5m])) ",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "$host",
"refId": "A"
}
],
"title": "HTTP Traffic",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 21,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 4,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 11,
"x": 0,
"y": 8
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.5.0-54880pre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_time_seconds_sum{instance=\"$host\"}[5m])) / sum(rate(nginx_http_response_time_seconds_count{instance=\"$host\"}[5m])) ",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "$host",
"refId": "A"
}
],
"title": "Average Response Time [5m]",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 3,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 11,
"x": 11,
"y": 14
},
"id": 10,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "8.5.0-54880pre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) by (status)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "",
"refId": "A"
}
],
"title": "Status codes per second",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 4,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 11,
"x": 0,
"y": 15
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "8.5.0-54880pre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "nginx_http_response_time_seconds{quantile=\"0.9\",method=\"GET\",status=~\"2[0-9]*\",instance=~\"$host\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "$host",
"refId": "A"
}
],
"title": "Response time (90% quantile)",
"type": "timeseries"
}
],
"refresh": "5s",
"schemaVersion": 39,
"tags": [],
"templating": {
"list": [
{
"current": {
"isNone": true,
"selected": false,
"text": "None",
"value": ""
},
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"definition": "label_values(nginx_http_response_count_total,instance)",
"hide": 0,
"includeAll": false,
"label": "Host:",
"multi": false,
"name": "host",
"options": [],
"query": {
"query": "label_values(nginx_http_response_count_total,instance)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-15m",
"to": "now"
},
"timeRangeUpdatedDuringEditOrView": false,
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "NGINX Log Metrics [M]",
"uid": "JfOTY2Pnk",
"version": 4,
"weekStart": ""
}

24148
services/monitoring/lgtm/dashboards/node_exporter.json Normal file
View file

File diff suppressed because it is too large Load diff

7000
services/monitoring/lgtm/dashboards/postgres.json Normal file
View file

File diff suppressed because it is too large Load diff

3280
services/monitoring/lgtm/dashboards/postgres2.json Normal file
View file

File diff suppressed because it is too large Load diff

7
services/monitoring/lgtm/default.nix Normal file
View file

@ -0,0 +1,7 @@
{
imports = [
./grafana.nix
./loki.nix
./prometheus.nix
];
}

133
services/monitoring/lgtm/grafana.nix Normal file
View file

@ -0,0 +1,133 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.services.grafana;
inherit (lib) mkEnableOption mkIf;
in
{
options.bagel.services.grafana.enable = mkEnableOption "Grafana frontend";
config = mkIf cfg.enable {
age.secrets.grafana-oauth-secret = {
file = ../../../secrets/grafana-oauth-secret.age;
owner = "grafana";
};
bagel.services.postgres.enable = true;
services = {
grafana = {
enable = true;
settings = {
server = {
domain = "grafana.forkos.org";
http_addr = "127.0.0.1";
http_port = 2342;
root_url = "https://grafana.forkos.org/";
};
database = {
type = "postgres";
user = "grafana";
host = "/run/postgresql";
};
"auth.generic_oauth" = {
enabled = true;
name = "Lix SSO";
client_id = "forkos-grafana";
client_secret = "$__file{${config.age.secrets.grafana-oauth-secret.path}}";
auth_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
token_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
api_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/userinfo";
login_attribute_path = "username";
email_attribute_path = "email";
name_attribute_path = "full_name";
scopes = [
"openid"
"profile"
"email"
"offline_access"
"roles"
];
allow_sign_up = true;
auto_login = true;
allow_assign_grafana_admin = true;
role_attribute_path = "contains(grafana_roles[*], 'Admin') && 'GrafanaAdmin' || contains(grafana_roles[*], 'Editor') && 'Editor' || 'Viewer'";
};
dashboards.default_home_dashboard_path = "${./dashboards/node_exporter.json}";
feature_toggles.enable = "autoMigrateOldPanels newVizTooltips";
security.angular_support_enabled = false;
};
provision = {
dashboards.settings = {
apiVersion = 1;
providers = [
{
name = "default";
options.path = ./dashboards;
}
];
};
datasources.settings = {
apiVersion = 1;
datasources = [
{
name = "Mimir";
type = "prometheus";
uid = "mimir";
access = "proxy";
url = "http://127.0.0.1:9009/prometheus";
}
{
name = "Loki";
type = "loki";
uid = "loki";
access = "proxy";
url = "http://127.0.0.1:9090/";
}
];
};
};
};
postgresql = {
ensureDatabases = [ "grafana" ];
ensureUsers = [
{
name = "grafana";
ensureDBOwnership = true;
}
];
};
nginx = let
scfg = config.services.grafana.settings.server;
in {
enable = true;
virtualHosts."${scfg.domain}" = {
enableACME = true;
forceSSL = true;
locations."/" = {
proxyPass = "http://${scfg.http_addr}:${toString scfg.http_port}";
proxyWebsockets = true;
};
};
};
};
};
}

100
services/monitoring/lgtm/loki.nix Normal file
View file

@ -0,0 +1,100 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.services.loki;
inherit (lib) mkEnableOption mkIf;
in
{
options.bagel.services.loki.enable = mkEnableOption "Loki storage";
config = mkIf cfg.enable {
age.secrets = {
loki-htpasswd = {
file = ../../../secrets/loki-htpasswd.age;
owner = "nginx";
};
loki-environment.file = ../../../secrets/loki-environment.age;
};
services.loki = {
enable = true;
extraFlags = ["--config.expand-env"];
configuration = {
server = {
http_listen_port = 9090;
grpc_listen_port = 9096;
# 16M
grpc_server_max_recv_msg_size = 16777216;
grpc_server_max_send_msg_size = 16777216;
};
auth_enabled = false;
common = {
storage.s3 = {
endpoint = "s3.delroth.net";
region = "garage";
bucketnames = "bagel-loki";
secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
access_key_id = "\${S3_KEY_ID}";
s3forcepathstyle = true;
};
ring.kvstore.store = "memberlist";
replication_factor = 1;
};
memberlist = {
bind_port = 7947;
advertise_port = 7947;
};
storage_config.tsdb_shipper = {
active_index_directory = "/var/lib/loki/index";
cache_location = "/var/lib/loki/cache";
};
compactor = {
working_directory = "/var/lib/loki/compactor";
compaction_interval = "10m";
retention_enabled = true;
retention_delete_delay = "1s";
retention_delete_worker_count = 150;
delete_request_store = "filesystem";
};
limits_config.retention_period = "1w";
schema_config = {
configs = [
{
from = "2024-07-01";
store = "tsdb";
object_store = "s3";
schema = "v13";
index = {
prefix = "index_";
period = "24h";
};
}
];
};
};
};
systemd.services.loki.serviceConfig.EnvironmentFile = [ config.age.secrets.loki-environment.path ];
services.nginx.virtualHosts."loki.forkos.org" = {
enableACME = true;
forceSSL = true;
locations."/loki/api/v1/push" = {
proxyPass = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
basicAuthFile = config.age.secrets.loki-htpasswd.path;
};
};
};
}

83
services/monitoring/lgtm/prometheus.nix Normal file
View file

@ -0,0 +1,83 @@
{
config,
lib,
nodes,
...
}:
let
cfg = config.bagel.services.prometheus;
inherit (lib) mkEnableOption mkIf;
forEachMachine = fn: map fn (builtins.attrValues nodes);
allMetas = forEachMachine (machine: {
name = machine.config.networking.hostName;
address = machine.config.bagel.meta.monitoring.address or null;
exporters = machine.config.bagel.meta.monitoring.exporters or [];
});
scrapableMetas = builtins.filter (m: m.address != null && m.exporters != []) allMetas;
toJobConfig = m: {
job_name = m.name;
static_configs = [
{ targets = map (e: m.address + ":" + (toString e.port)) m.exporters; }
];
};
jobConfigs = map toJobConfig scrapableMetas;
in
{
options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
config = mkIf cfg.enable {
age.secrets.mimir-environment.file = ../../../secrets/mimir-environment.age;
services.prometheus = {
enable = true;
enableAgentMode = true;
listenAddress = "127.0.0.1";
port = 9001;
globalConfig.scrape_interval = "15s";
scrapeConfigs = jobConfigs;
remoteWrite = [
{ url = "http://localhost:9009/api/v1/push"; }
];
};
services.mimir = {
enable = true;
extraFlags = ["--config.expand-env=true"];
configuration = {
multitenancy_enabled = false;
common.storage = {
backend = "s3";
s3 = {
endpoint = "s3.delroth.net";
bucket_name = "bagel-mimir";
secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
access_key_id = "\${S3_KEY_ID}";
};
};
server = {
http_listen_port = 9009;
grpc_server_max_recv_msg_size = 104857600;
grpc_server_max_send_msg_size = 104857600;
grpc_server_max_concurrent_streams = 1000;
};
ingester.ring.replication_factor = 1;
blocks_storage.backend = "s3";
ruler_storage = {
backend = "local";
local.directory = ./alerts;
};
};
};
systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
};
}

53
services/monitoring/promtail.nix Normal file
View file

@ -0,0 +1,53 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.monitoring.promtail;
inherit (lib) mkEnableOption mkIf;
in
{
options.bagel.monitoring.promtail.enable = (mkEnableOption "Promtail log export") // { default = true; };
config = mkIf cfg.enable {
age.secrets.promtail-password = {
file = ../../secrets/promtail-password.age;
owner = "promtail";
};
services.promtail = {
enable = true;
configuration = {
server.disable = true;
clients = [
{
url = "https://loki.forkos.org/loki/api/v1/push";
basic_auth = {
username = "promtail";
password_file = config.age.secrets.promtail-password.path;
};
}
];
scrape_configs = [
{
job_name = "system";
journal = {
max_age = "12h";
labels = {
job = "systemd-journal";
host = config.networking.hostName;
};
};
relabel_configs = [
{
source_labels = [ "__journal__systemd_unit" ];
target_label = "unit";
}
];
}
];
};
};
};
}