Compare commits

...

6 commits

Author SHA1 Message Date
Ilya K 435f831b89 Add Loki + Promtail setup 2024-07-05 19:07:52 +03:00
Ilya K 0cc353e583 Add postgres exporter 2024-07-05 19:07:52 +03:00
Ilya K b1ad697d8d Add nginx log exporter 2024-07-05 19:07:52 +03:00
Ilya K 97ea0913d7 Add Grafana/Prometheus/Mimir minimal setup
More later, Loki also later.
2024-07-05 19:07:52 +03:00
Ilya K be2da46c46 Add devShell with agenix and colmena 2024-07-05 19:07:28 +03:00
Ilya K 841461d6c9 Clean up SSH key dupes, add Maxine 2024-07-05 19:07:28 +03:00
30 changed files with 35725 additions and 13 deletions

2
.envrc Normal file
View file

@ -0,0 +1,2 @@
# shellcheck shell=bash
use flake

1
.gitignore vendored
View file

@ -1,3 +1,4 @@
result result
.gcroots .gcroots
config.tf.json config.tf.json
.direnv

View file

@ -1,13 +1,16 @@
{ let
users.users.root.openssh.authorizedKeys.keys = [ keys = import ./ssh-keys.nix;
# delroth in {
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" users.users.root.openssh.authorizedKeys.keys =
# raito keys.users.delroth ++
keys.users.k900 ++
keys.users.raito ++
keys.users.maxine ++
[
# more raito
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD" "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM=" "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM="
# k900
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun"
]; ];
} }

View file

@ -2,10 +2,13 @@
machines = { machines = {
bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsO4bNqY04uG13Pg3ubHfRDssTphDLzZ4YUniE5/p+M"; bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsO4bNqY04uG13Pg3ubHfRDssTphDLzZ4YUniE5/p+M";
meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT"; meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
gerrit01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+eSZu+u9sCynrMlsmFzQHLIELQAuVg0Cs1pBvwb4+A";
}; };
users = { users = {
delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ]; delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ];
raito = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp" ]; raito = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp" ];
k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
}; };
} }

View file

@ -57,7 +57,15 @@
''); '');
}; };
}; };
defaultApp.${system} = self.apps.${system}.apply; apps.${system}.default = self.apps.${system}.apply;
devShells.${system}.default = pkgs.mkShell {
packages = [
inputs.agenix.packages.${system}.agenix
inputs.colmena.packages.${system}.colmena
];
};
colmena = { colmena = {
meta.nixpkgs = import nixpkgs { meta.nixpkgs = import nixpkgs {
localSystem = system; localSystem = system;

View file

@ -40,6 +40,7 @@
hydra.enable = true; hydra.enable = true;
hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra"; hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
}; };
bagel.meta.monitoring.address = "bagel-box.delroth.net";
security.acme.acceptTerms = true; security.acme.acceptTerms = true;
security.acme.defaults.email = "bagel@delroth.net"; security.acme.defaults.email = "bagel@delroth.net";

View file

@ -24,6 +24,7 @@
}; };
}; };
}; };
bagel.meta.monitoring.address = "gerrit01.infra.forkos.org";
fileSystems."/gerrit-data" = { fileSystems."/gerrit-data" = {
device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4"; device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";

View file

@ -21,6 +21,10 @@
enable = true; enable = true;
domain = "netbox.forkos.org"; domain = "netbox.forkos.org";
}; };
bagel.meta.monitoring.address = "meta01.infra.forkos.org";
bagel.services.prometheus.enable = true;
bagel.services.loki.enable = true;
bagel.services.grafana.enable = true;
i18n.defaultLocale = "fr_FR.UTF-8"; i18n.defaultLocale = "fr_FR.UTF-8";

View file

@ -7,6 +7,13 @@ let
hydra-s3-credentials = [ machines.bagel-box ]; hydra-s3-credentials = [ machines.bagel-box ];
hydra-ssh-key-priv = [ machines.bagel-box ]; hydra-ssh-key-priv = [ machines.bagel-box ];
netbox-environment = [ machines.meta01 ]; netbox-environment = [ machines.meta01 ];
mimir-environment = [ machines.meta01 ];
grafana-oauth-secret = [ machines.meta01 ];
loki-environment = [ machines.meta01 ];
# These are the same password, but nginx wants it in htpasswd format
loki-htpasswd = [ machines.meta01 ];
promtail-password = builtins.attrValues machines;
}; };
in in
builtins.listToAttrs ( builtins.listToAttrs (

View file

@ -0,0 +1,7 @@
age-encryption.org/v1
-> ssh-ed25519 j2r2qQ Xl0fSOuF0xNTJrtVGdRLRIszd15LFrG5KCFNvSBK4Go
qSEMBBw90jz4j8elpoUeyS4CTLBhZtNDhLNigesJq+0
-> ssh-ed25519 K3b7BA cKI0twKiuuTKv1Js4jqt5v8cOqpxEMY9dmVghgJtbzw
K5o31XP/nLsswsrMaxnIzCXVUtJqmJWoFglWFsV7+AQ
--- X8pvqCHeCQ0LjzcjIHThkqp6YeOOT8dBMLuktgdgeY4
sZÓ¸ŠíØ[þ²X<C2B2>“¡èÅ®Š5°=÷6)ÇT¿Q†N{•x³I1ƒ!ÓÜøB ƒzš*×íåL~K

View file

@ -0,0 +1,9 @@
age-encryption.org/v1
-> ssh-ed25519 j2r2qQ w0lLquFUUcmEZ/Fh1YSt85tAJkBwavORQbwMr7gMqF4
J4T+EHm1uHbCZkAUNoNcB9uGSz082mFL8+dkCnvYQnM
-> ssh-ed25519 K3b7BA 28bJZgBPPc2KIE5+b8LJuQ5L4YAiRAJzucEuOqXHdVM
7hKENFr8QX0jpwuuQEjGFrUywJuhL1Tdi2V4/gR8JWE
--- GSPZxz39TMMWv0qhotNgnXa5679Q7VK8JGjQjI7A8oM
J²\@F“N• ³å2®ô¨w×!¯1Vf»§˜Ž·ÞO²CÓw®®V°£šÌº.^݆ 7<C2A0>w‡n4äàdW-Ö¾"@0¨ú¹EÏ¿·°ck,]M}x<>øÌ<C3B8>¡Ûy°[×ÁJ:!è‘ !ø螀c¬
Bë¹R
nøê€þÀáÆ^9í¤M<ú

View file

@ -0,0 +1,7 @@
age-encryption.org/v1
-> ssh-ed25519 j2r2qQ nLWy3WcVJWCl3rXkhcSbp1joqmkk06QnxhCZ4UtSvmw
iQ+Hx/vhiFgkWfbxHwGjxMBEqzyGww4/9do3W7V/y1Y
-> ssh-ed25519 K3b7BA RkF2ADcjOGtivl9MrhO/HFwxlTAkbFHWL3iinUldMiM
7q/zdVTMLevukZjkHtcN88iYzfTLvq2s3QdkgsFSO9M
--- 1b2HiK06vJPqBgHVDD0QELOtfkl7/rlgGS9uI1mSbus
„uܧoL;õå¬" 4¦Û»Z¼˜@§öãƒÐ3+93Q4óÄ o•ŒØwé“„6<>M-²DkJn´;ñ*g <0A>Yœ75ËSò)Ù°©

Binary file not shown.

View file

@ -0,0 +1,12 @@
age-encryption.org/v1
-> ssh-ed25519 +HUDfA ZUM0ACC/NIekvX1PkCiXTHaTeE3ybudmY3piHw2iekQ
cHj94FIR6gNJ3Hw9FI7K15OYgxbjkajGtCftD+2Mr8c
-> ssh-ed25519 2D+APA tzlyOnAXnLxXO/47n45sFPiJF3FXd98UU5ajPhD2wSs
P8ZdUiBeME17SU2BpMgOq4plyAqgzLOQWHa1+Q7cjYo
-> ssh-ed25519 j2r2qQ 3OikD9JOmug7kdPAPz+JT/ryB6xBQhu2+cwS9h5sKGI
XiIuxOyey2I6hmqabUCPzLc85q/1r9OwVGjHWYNQsp0
-> ssh-ed25519 K3b7BA Bdqcqt4GgLzuSiEnIyImDiOQGwyIhhozRXMmNrp7glI
65joZcnl0Hqe90Th2EdVgbcxUJFpy3fOgk6oPiSHh2A
--- 6x6BFNypc+u3DpsHX3SajwEy1TqsAtbFei0ddRpEoBg
äªUG¾xj4»®Îþ‡b=óžóñ¼Rd<52>3sHYÝ Ô<>*Qµ9Ã6n34&äw»~h!§ ^„[êš

View file

@ -4,5 +4,6 @@
./postgres ./postgres
./netbox ./netbox
./gerrit ./gerrit
./monitoring
]; ];
} }

View file

@ -0,0 +1,7 @@
{
imports = [
./exporters
./lgtm
./promtail.nix
];
}

View file

@ -0,0 +1,34 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.monitoring.exporters.baseline;
inherit (lib) mkEnableOption mkIf;
in
{
options.bagel.monitoring.exporters.baseline.enable = (mkEnableOption "Standard set of exporters") // { default = true; };
config = mkIf cfg.enable {
services.prometheus.exporters.node = {
enable = true;
enabledCollectors = [
"processes"
"systemd"
];
port = 9101;
};
services.cadvisor = {
enable = true;
port = 9102;
listenAddress = "0.0.0.0";
};
bagel.meta.monitoring.exporters = [
{ port = 9101; }
{ port = 9102; }
];
};
}

View file

@ -0,0 +1,36 @@
{
config,
lib,
...
}:
let
inherit (lib) mkOption types;
in
{
imports = [
./baseline.nix
./nginx.nix
./postgres.nix
];
options.bagel = {
meta.monitoring = {
address = mkOption {
description = "Node's public address";
type = types.str;
};
exporters = mkOption {
description = "List of all exporters to scrape";
type = types.listOf (types.submodule {
options.port = mkOption {
description = "Exporter port";
type = types.int;
};
});
default = [];
};
};
};
config.networking.firewall.allowedTCPPorts = map (e: e.port) config.bagel.meta.monitoring.exporters;
}

View file

@ -0,0 +1,37 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.monitoring.exporters.nginx;
inherit (lib) mkEnableOption mkIf;
logFormat = ''$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'';
in
{
options.bagel.monitoring.exporters.nginx.enable = (mkEnableOption "Nginx access.log exporter") // { default = config.services.nginx.enable; };
config = mkIf cfg.enable {
services.nginx.appendHttpConfig = ''
log_format ours '${logFormat}';
access_log /var/log/nginx/access.log ours;
'';
services.prometheus.exporters.nginxlog = {
enable = true;
port = 9103;
group = "nginx";
settings.namespaces = [
{
name = "nginx";
format = logFormat;
source.files = ["/var/log/nginx/access.log"];
}
];
};
bagel.meta.monitoring.exporters = [
{ port = 9103; }
];
};
}

View file

@ -0,0 +1,31 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.monitoring.exporters.postgres;
inherit (lib) mkEnableOption mkIf;
in
{
options.bagel.monitoring.exporters.postgres.enable = (mkEnableOption "Postgres exporter") // { default = config.services.postgresql.enable; };
config = mkIf cfg.enable {
services.prometheus.exporters.postgres = {
enable = true;
port = 9104;
runAsLocalSuperUser = true;
extraFlags = [
"--collector.long_running_transactions"
"--collector.stat_activity_autovacuum"
"--collector.stat_statements"
];
};
services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
bagel.meta.monitoring.exporters = [
{ port = 9104; }
];
};
}

View file

View file

@ -0,0 +1,697 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"description": "NGINX Log metrics with Prometheus based on https://github.com/martin-helmich/prometheus-nginxlog-exporter \r\nBased on namespace prefix 'nginx'. If different, you may need to adjust the metrics.\r\nDashboard based on 6482 dashboard",
"editable": true,
"fiscalYearStartMonth": 0,
"gnetId": 15947,
"graphTooltip": 0,
"id": 4,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-GrYlRd"
},
"mappings": [],
"thresholds": {
"mode": "percentage",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 5
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 11,
"x": 0,
"y": 0
},
"id": 12,
"options": {
"displayMode": "lcd",
"maxVizHeight": 300,
"minVizHeight": 16,
"minVizWidth": 8,
"namePlacement": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": true,
"sizing": "auto",
"text": {},
"valueMode": "color"
},
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"editorMode": "code",
"exemplar": true,
"expr": "sum(rate(nginx_http_response_count_total{status=~\"^2..\",instance=\"$host\"}[1m])) / sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) * 100",
"hide": false,
"interval": "",
"legendFormat": "2** status codes",
"range": true,
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_count_total{status=~\"^4..\",instance=\"$host\"}[1m])) / sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) * 100",
"interval": "",
"legendFormat": "4** status codes",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_count_total{status=~\"^5..\",instance=\"$host\"}[1m])) / sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) * 100",
"hide": false,
"interval": "",
"legendFormat": "5** status codes",
"refId": "B"
}
],
"title": "Percentage Ratio of status codes to all status codes",
"transparent": true,
"type": "bargauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 4,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 11,
"x": 11,
"y": 0
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "8.5.0-54880pre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_time_seconds_count{instance=\"$host\"}[1m])) ",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "$host",
"refId": "A"
}
],
"title": "Requests per Second",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"description": "Response sizes in bytes",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 4,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 11,
"x": 11,
"y": 7
},
"id": 8,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "8.5.0-54880pre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_size_bytes{instance=\"$host\"}[5m])) ",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "$host",
"refId": "A"
}
],
"title": "HTTP Traffic",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 21,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 4,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 11,
"x": 0,
"y": 8
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.5.0-54880pre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_time_seconds_sum{instance=\"$host\"}[5m])) / sum(rate(nginx_http_response_time_seconds_count{instance=\"$host\"}[5m])) ",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "$host",
"refId": "A"
}
],
"title": "Average Response Time [5m]",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 3,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 11,
"x": 11,
"y": 14
},
"id": 10,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "8.5.0-54880pre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) by (status)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "",
"refId": "A"
}
],
"title": "Status codes per second",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 4,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 11,
"x": 0,
"y": 15
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "8.5.0-54880pre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"exemplar": true,
"expr": "nginx_http_response_time_seconds{quantile=\"0.9\",method=\"GET\",status=~\"2[0-9]*\",instance=~\"$host\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "$host",
"refId": "A"
}
],
"title": "Response time (90% quantile)",
"type": "timeseries"
}
],
"refresh": "5s",
"schemaVersion": 39,
"tags": [],
"templating": {
"list": [
{
"current": {
"isNone": true,
"selected": false,
"text": "None",
"value": ""
},
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"definition": "label_values(nginx_http_response_count_total,instance)",
"hide": 0,
"includeAll": false,
"label": "Host:",
"multi": false,
"name": "host",
"options": [],
"query": {
"query": "label_values(nginx_http_response_count_total,instance)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-15m",
"to": "now"
},
"timeRangeUpdatedDuringEditOrView": false,
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "NGINX Log Metrics [M]",
"uid": "JfOTY2Pnk",
"version": 4,
"weekStart": ""
}

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,7 @@
{
imports = [
./grafana.nix
./loki.nix
./prometheus.nix
];
}

View file

@ -0,0 +1,133 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.services.grafana;
inherit (lib) mkEnableOption mkIf;
in
{
options.bagel.services.grafana.enable = mkEnableOption "Grafana frontend";
config = mkIf cfg.enable {
age.secrets.grafana-oauth-secret = {
file = ../../../secrets/grafana-oauth-secret.age;
owner = "grafana";
};
bagel.services.postgres.enable = true;
services = {
grafana = {
enable = true;
settings = {
server = {
domain = "grafana.forkos.org";
http_addr = "127.0.0.1";
http_port = 2342;
root_url = "https://grafana.forkos.org/";
};
database = {
type = "postgres";
user = "grafana";
host = "/run/postgresql";
};
"auth.generic_oauth" = {
enabled = true;
name = "Lix SSO";
client_id = "forkos-grafana";
client_secret = "$__file{${config.age.secrets.grafana-oauth-secret.path}}";
auth_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
token_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
api_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/userinfo";
login_attribute_path = "username";
email_attribute_path = "email";
name_attribute_path = "full_name";
scopes = [
"openid"
"profile"
"email"
"offline_access"
"roles"
];
allow_sign_up = true;
auto_login = true;
allow_assign_grafana_admin = true;
role_attribute_path = "contains(grafana_roles[*], 'Admin') && 'GrafanaAdmin' || contains(grafana_roles[*], 'Editor') && 'Editor' || 'Viewer'";
};
dashboards.default_home_dashboard_path = "${./dashboards/node_exporter.json}";
feature_toggles.enable = "autoMigrateOldPanels newVizTooltips";
security.angular_support_enabled = false;
};
provision = {
dashboards.settings = {
apiVersion = 1;
providers = [
{
name = "default";
options.path = ./dashboards;
}
];
};
datasources.settings = {
apiVersion = 1;
datasources = [
{
name = "Mimir";
type = "prometheus";
uid = "mimir";
access = "proxy";
url = "http://127.0.0.1:9009/prometheus";
}
{
name = "Loki";
type = "loki";
uid = "loki";
access = "proxy";
url = "http://127.0.0.1:9090/";
}
];
};
};
};
postgresql = {
ensureDatabases = [ "grafana" ];
ensureUsers = [
{
name = "grafana";
ensureDBOwnership = true;
}
];
};
nginx = let
scfg = config.services.grafana.settings.server;
in {
enable = true;
virtualHosts."${scfg.domain}" = {
enableACME = true;
forceSSL = true;
locations."/" = {
proxyPass = "http://${scfg.http_addr}:${toString scfg.http_port}";
proxyWebsockets = true;
};
};
};
};
};
}

View file

@ -0,0 +1,100 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.services.loki;
inherit (lib) mkEnableOption mkIf;
in
{
options.bagel.services.loki.enable = mkEnableOption "Loki storage";
config = mkIf cfg.enable {
age.secrets = {
loki-htpasswd = {
file = ../../../secrets/loki-htpasswd.age;
owner = "nginx";
};
loki-environment.file = ../../../secrets/loki-environment.age;
};
services.loki = {
enable = true;
extraFlags = ["--config.expand-env"];
configuration = {
server = {
http_listen_port = 9090;
grpc_listen_port = 9096;
# 16M
grpc_server_max_recv_msg_size = 16777216;
grpc_server_max_send_msg_size = 16777216;
};
auth_enabled = false;
common = {
storage.s3 = {
endpoint = "s3.delroth.net";
region = "garage";
bucketnames = "bagel-loki";
secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
access_key_id = "\${S3_KEY_ID}";
s3forcepathstyle = true;
};
ring.kvstore.store = "memberlist";
replication_factor = 1;
};
memberlist = {
bind_port = 7947;
advertise_port = 7947;
};
storage_config.tsdb_shipper = {
active_index_directory = "/var/lib/loki/index";
cache_location = "/var/lib/loki/cache";
};
compactor = {
working_directory = "/var/lib/loki/compactor";
compaction_interval = "10m";
retention_enabled = true;
retention_delete_delay = "1s";
retention_delete_worker_count = 150;
delete_request_store = "filesystem";
};
limits_config.retention_period = "1w";
schema_config = {
configs = [
{
from = "2024-07-01";
store = "tsdb";
object_store = "s3";
schema = "v13";
index = {
prefix = "index_";
period = "24h";
};
}
];
};
};
};
systemd.services.loki.serviceConfig.EnvironmentFile = [ config.age.secrets.loki-environment.path ];
services.nginx.virtualHosts."loki.forkos.org" = {
enableACME = true;
forceSSL = true;
locations."/loki/api/v1/push" = {
proxyPass = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
basicAuthFile = config.age.secrets.loki-htpasswd.path;
};
};
};
}

View file

@ -0,0 +1,83 @@
{
config,
lib,
nodes,
...
}:
let
cfg = config.bagel.services.prometheus;
inherit (lib) mkEnableOption mkIf;
forEachMachine = fn: map fn (builtins.attrValues nodes);
allMetas = forEachMachine (machine: {
name = machine.config.networking.hostName;
address = machine.config.bagel.meta.monitoring.address or null;
exporters = machine.config.bagel.meta.monitoring.exporters or [];
});
scrapableMetas = builtins.filter (m: m.address != null && m.exporters != []) allMetas;
toJobConfig = m: {
job_name = m.name;
static_configs = [
{ targets = map (e: m.address + ":" + (toString e.port)) m.exporters; }
];
};
jobConfigs = map toJobConfig scrapableMetas;
in
{
options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
config = mkIf cfg.enable {
age.secrets.mimir-environment.file = ../../../secrets/mimir-environment.age;
services.prometheus = {
enable = true;
enableAgentMode = true;
listenAddress = "127.0.0.1";
port = 9001;
globalConfig.scrape_interval = "15s";
scrapeConfigs = jobConfigs;
remoteWrite = [
{ url = "http://localhost:9009/api/v1/push"; }
];
};
services.mimir = {
enable = true;
extraFlags = ["--config.expand-env=true"];
configuration = {
multitenancy_enabled = false;
common.storage = {
backend = "s3";
s3 = {
endpoint = "s3.delroth.net";
bucket_name = "bagel-mimir";
secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
access_key_id = "\${S3_KEY_ID}";
};
};
server = {
http_listen_port = 9009;
grpc_server_max_recv_msg_size = 104857600;
grpc_server_max_send_msg_size = 104857600;
grpc_server_max_concurrent_streams = 1000;
};
ingester.ring.replication_factor = 1;
blocks_storage.backend = "s3";
ruler_storage = {
backend = "local";
local.directory = ./alerts;
};
};
};
systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
};
}

View file

@ -0,0 +1,53 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.monitoring.promtail;
inherit (lib) mkEnableOption mkIf;
in
{
options.bagel.monitoring.promtail.enable = (mkEnableOption "Promtail log export") // { default = true; };
config = mkIf cfg.enable {
age.secrets.promtail-password = {
file = ../../secrets/promtail-password.age;
owner = "promtail";
};
services.promtail = {
enable = true;
configuration = {
server.disable = true;
clients = [
{
url = "https://loki.forkos.org/loki/api/v1/push";
basic_auth = {
username = "promtail";
password_file = config.age.secrets.promtail-password.path;
};
}
];
scrape_configs = [
{
job_name = "system";
journal = {
max_age = "12h";
labels = {
job = "systemd-journal";
host = config.networking.hostName;
};
};
relabel_configs = [
{
source_labels = [ "__journal__systemd_unit" ];
target_label = "unit";
}
];
}
];
};
};
};
}