Add Grafana/Prometheus/Mimir minimal setup
More later, Loki also later.
This commit is contained in:
parent
99f715caca
commit
63b31e98cf
|
@ -40,6 +40,7 @@
|
|||
hydra.enable = true;
|
||||
hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
|
||||
};
|
||||
bagel.meta.monitoring.address = "bagel-box.delroth.net";
|
||||
|
||||
security.acme.acceptTerms = true;
|
||||
security.acme.defaults.email = "bagel@delroth.net";
|
||||
|
|
|
@ -24,6 +24,7 @@
|
|||
};
|
||||
};
|
||||
};
|
||||
bagel.meta.monitoring.address = "gerrit01.infra.forkos.org";
|
||||
|
||||
fileSystems."/gerrit-data" = {
|
||||
device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
|
||||
|
|
|
@ -21,6 +21,9 @@
|
|||
enable = true;
|
||||
domain = "netbox.forkos.org";
|
||||
};
|
||||
bagel.meta.monitoring.address = "meta01.infra.forkos.org";
|
||||
bagel.services.prometheus.enable = true;
|
||||
bagel.services.grafana.enable = true;
|
||||
|
||||
i18n.defaultLocale = "fr_FR.UTF-8";
|
||||
|
||||
|
|
|
@ -7,6 +7,8 @@ let
|
|||
hydra-s3-credentials = [ machines.bagel-box ];
|
||||
hydra-ssh-key-priv = [ machines.bagel-box ];
|
||||
netbox-environment = [ machines.meta01 ];
|
||||
mimir-environment = [ machines.meta01 ];
|
||||
grafana-oauth-secret = [ machines.meta01 ];
|
||||
};
|
||||
in
|
||||
builtins.listToAttrs (
|
||||
|
|
7
secrets/grafana-oauth-secret.age
Normal file
7
secrets/grafana-oauth-secret.age
Normal file
|
@ -0,0 +1,7 @@
|
|||
age-encryption.org/v1
|
||||
-> ssh-ed25519 j2r2qQ Xl0fSOuF0xNTJrtVGdRLRIszd15LFrG5KCFNvSBK4Go
|
||||
qSEMBBw90jz4j8elpoUeyS4CTLBhZtNDhLNigesJq+0
|
||||
-> ssh-ed25519 K3b7BA cKI0twKiuuTKv1Js4jqt5v8cOqpxEMY9dmVghgJtbzw
|
||||
K5o31XP/nLsswsrMaxnIzCXVUtJqmJWoFglWFsV7+AQ
|
||||
--- X8pvqCHeCQ0LjzcjIHThkqp6YeOOT8dBMLuktgdgeY4
|
||||
sZÓ¸ŠíØ[þ²X<C2B2>“¡èÅ®Š5°=÷6)ÇT¿Q†‘N{•x³I1ƒ!Ó–ÜøB
ƒzš*×íåL~K
|
BIN
secrets/mimir-environment.age
Normal file
BIN
secrets/mimir-environment.age
Normal file
Binary file not shown.
|
@ -4,5 +4,6 @@
|
|||
./postgres
|
||||
./netbox
|
||||
./gerrit
|
||||
./monitoring
|
||||
];
|
||||
}
|
||||
|
|
6
services/monitoring/default.nix
Normal file
6
services/monitoring/default.nix
Normal file
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
imports = [
|
||||
./exporters
|
||||
./lgtm
|
||||
];
|
||||
}
|
34
services/monitoring/exporters/baseline.nix
Normal file
34
services/monitoring/exporters/baseline.nix
Normal file
|
@ -0,0 +1,34 @@
|
|||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.bagel.monitoring.exporters.baseline;
|
||||
inherit (lib) mkEnableOption mkIf;
|
||||
in
|
||||
{
|
||||
options.bagel.monitoring.exporters.baseline.enable = (mkEnableOption "Standard set of exporters") // { default = true; };
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
services.prometheus.exporters.node = {
|
||||
enable = true;
|
||||
enabledCollectors = [
|
||||
"processes"
|
||||
"systemd"
|
||||
];
|
||||
port = 9101;
|
||||
};
|
||||
|
||||
services.cadvisor = {
|
||||
enable = true;
|
||||
port = 9102;
|
||||
listenAddress = "0.0.0.0";
|
||||
};
|
||||
|
||||
bagel.meta.monitoring.exporters = [
|
||||
{ port = 9101; }
|
||||
{ port = 9102; }
|
||||
];
|
||||
};
|
||||
}
|
32
services/monitoring/exporters/default.nix
Normal file
32
services/monitoring/exporters/default.nix
Normal file
|
@ -0,0 +1,32 @@
|
|||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
inherit (lib) mkOption types;
|
||||
in
|
||||
{
|
||||
imports = [./baseline.nix];
|
||||
|
||||
options.bagel = {
|
||||
meta.monitoring = {
|
||||
address = mkOption {
|
||||
description = "Node's public address";
|
||||
type = types.str;
|
||||
};
|
||||
exporters = mkOption {
|
||||
description = "List of all exporters to scrape";
|
||||
type = types.listOf (types.submodule {
|
||||
options.port = mkOption {
|
||||
description = "Exporter port";
|
||||
type = types.int;
|
||||
};
|
||||
});
|
||||
default = [];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config.networking.firewall.allowedTCPPorts = map (e: e.port) config.bagel.meta.monitoring.exporters;
|
||||
}
|
0
services/monitoring/lgtm/alerts/.gitkeep
Normal file
0
services/monitoring/lgtm/alerts/.gitkeep
Normal file
24148
services/monitoring/lgtm/dashboards/node_exporter.json
Normal file
24148
services/monitoring/lgtm/dashboards/node_exporter.json
Normal file
File diff suppressed because it is too large
Load diff
6
services/monitoring/lgtm/default.nix
Normal file
6
services/monitoring/lgtm/default.nix
Normal file
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
imports = [
|
||||
./grafana.nix
|
||||
./prometheus.nix
|
||||
];
|
||||
}
|
126
services/monitoring/lgtm/grafana.nix
Normal file
126
services/monitoring/lgtm/grafana.nix
Normal file
|
@ -0,0 +1,126 @@
|
|||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.bagel.services.grafana;
|
||||
inherit (lib) mkEnableOption mkIf;
|
||||
in
|
||||
{
|
||||
options.bagel.services.grafana.enable = mkEnableOption "Grafana frontend";
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
age.secrets.grafana-oauth-secret = {
|
||||
file = ../../../secrets/grafana-oauth-secret.age;
|
||||
owner = "grafana";
|
||||
};
|
||||
|
||||
bagel.services.postgres.enable = true;
|
||||
|
||||
services = {
|
||||
grafana = {
|
||||
enable = true;
|
||||
|
||||
settings = {
|
||||
server = {
|
||||
domain = "grafana.forkos.org";
|
||||
http_addr = "127.0.0.1";
|
||||
http_port = 2342;
|
||||
root_url = "https://grafana.forkos.org/";
|
||||
};
|
||||
|
||||
database = {
|
||||
type = "postgres";
|
||||
user = "grafana";
|
||||
host = "/run/postgresql";
|
||||
};
|
||||
|
||||
"auth.generic_oauth" = {
|
||||
enabled = true;
|
||||
|
||||
name = "Lix SSO";
|
||||
client_id = "forkos-grafana";
|
||||
client_secret = "$__file{${config.age.secrets.grafana-oauth-secret.path}}";
|
||||
|
||||
auth_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
|
||||
token_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
|
||||
api_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/userinfo";
|
||||
|
||||
login_attribute_path = "username";
|
||||
email_attribute_path = "email";
|
||||
name_attribute_path = "full_name";
|
||||
|
||||
scopes = [
|
||||
"openid"
|
||||
"profile"
|
||||
"email"
|
||||
"offline_access"
|
||||
"roles"
|
||||
];
|
||||
|
||||
allow_sign_up = true;
|
||||
auto_login = true;
|
||||
allow_assign_grafana_admin = true;
|
||||
|
||||
role_attribute_path = "contains(grafana_roles[*], 'Admin') && 'GrafanaAdmin' || contains(grafana_roles[*], 'Editor') && 'Editor' || 'Viewer'";
|
||||
};
|
||||
|
||||
dashboards.default_home_dashboard_path = "${./dashboards/node_exporter.json}";
|
||||
|
||||
feature_toggles.enable = "autoMigrateOldPanels newVizTooltips";
|
||||
security.angular_support_enabled = false;
|
||||
};
|
||||
|
||||
provision = {
|
||||
dashboards.settings = {
|
||||
apiVersion = 1;
|
||||
providers = [
|
||||
{
|
||||
name = "default";
|
||||
options.path = ./dashboards;
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
datasources.settings = {
|
||||
apiVersion = 1;
|
||||
datasources = [
|
||||
{
|
||||
name = "Mimir";
|
||||
type = "prometheus";
|
||||
uid = "mimir";
|
||||
access = "proxy";
|
||||
url = "http://127.0.0.1:9009/prometheus";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
postgresql = {
|
||||
ensureDatabases = [ "grafana" ];
|
||||
ensureUsers = [
|
||||
{
|
||||
name = "grafana";
|
||||
ensureDBOwnership = true;
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
nginx = let
|
||||
scfg = config.services.grafana.settings.server;
|
||||
in {
|
||||
enable = true;
|
||||
virtualHosts."${scfg.domain}" = {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
locations."/" = {
|
||||
proxyPass = "http://${scfg.http_addr}:${toString scfg.http_port}";
|
||||
proxyWebsockets = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
83
services/monitoring/lgtm/prometheus.nix
Normal file
83
services/monitoring/lgtm/prometheus.nix
Normal file
|
@ -0,0 +1,83 @@
|
|||
{
|
||||
config,
|
||||
lib,
|
||||
nodes,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.bagel.services.prometheus;
|
||||
inherit (lib) mkEnableOption mkIf;
|
||||
|
||||
forEachMachine = fn: map fn (builtins.attrValues nodes);
|
||||
|
||||
allMetas = forEachMachine (machine: {
|
||||
name = machine.config.networking.hostName;
|
||||
address = machine.config.bagel.meta.monitoring.address or null;
|
||||
exporters = machine.config.bagel.meta.monitoring.exporters or [];
|
||||
});
|
||||
|
||||
scrapableMetas = builtins.filter (m: m.address != null && m.exporters != []) allMetas;
|
||||
|
||||
toJobConfig = m: {
|
||||
job_name = m.name;
|
||||
static_configs = [
|
||||
{ targets = map (e: m.address + ":" + (toString e.port)) m.exporters; }
|
||||
];
|
||||
};
|
||||
|
||||
jobConfigs = map toJobConfig scrapableMetas;
|
||||
in
|
||||
{
|
||||
options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
age.secrets.mimir-environment.file = ../../../secrets/mimir-environment.age;
|
||||
|
||||
services.prometheus = {
|
||||
enable = true;
|
||||
enableAgentMode = true;
|
||||
listenAddress = "127.0.0.1";
|
||||
port = 9001;
|
||||
globalConfig.scrape_interval = "15s";
|
||||
scrapeConfigs = jobConfigs;
|
||||
remoteWrite = [
|
||||
{ url = "http://localhost:9009/api/v1/push"; }
|
||||
];
|
||||
};
|
||||
|
||||
services.mimir = {
|
||||
enable = true;
|
||||
extraFlags = ["--config.expand-env=true"];
|
||||
configuration = {
|
||||
multitenancy_enabled = false;
|
||||
|
||||
common.storage = {
|
||||
backend = "s3";
|
||||
s3 = {
|
||||
endpoint = "s3.delroth.net";
|
||||
bucket_name = "bagel-mimir";
|
||||
secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
|
||||
access_key_id = "\${S3_KEY_ID}";
|
||||
};
|
||||
};
|
||||
|
||||
server = {
|
||||
http_listen_port = 9009;
|
||||
grpc_server_max_recv_msg_size = 104857600;
|
||||
grpc_server_max_send_msg_size = 104857600;
|
||||
grpc_server_max_concurrent_streams = 1000;
|
||||
};
|
||||
|
||||
ingester.ring.replication_factor = 1;
|
||||
|
||||
blocks_storage.backend = "s3";
|
||||
ruler_storage = {
|
||||
backend = "local";
|
||||
local.directory = ./alerts;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
|
||||
};
|
||||
}
|
Loading…
Reference in a new issue