Compare commits

...

2 commits

Author SHA1 Message Date
Ilya K fd56c0509b Add Grafana/Prometheus/Mimir minimal setup
More later, Loki also later.
2024-07-05 14:25:27 +03:00
Ilya K 8bceca380a Add devShell with agenix and colmena 2024-07-05 13:06:34 +03:00
19 changed files with 24463 additions and 1 deletions

2
.envrc Normal file
View file

@ -0,0 +1,2 @@
# shellcheck shell=bash
use flake

1
.gitignore vendored
View file

@ -1,3 +1,4 @@
result result
.gcroots .gcroots
config.tf.json config.tf.json
.direnv

View file

@ -29,6 +29,7 @@ in
config = mkIf cfg.enable { config = mkIf cfg.enable {
services.qemuGuest.enable = true; services.qemuGuest.enable = true;
networking.useDHCP = false;
systemd.network.enable = true; systemd.network.enable = true;
security.acme.defaults.email = "bagel-acme@lahfa.xyz"; security.acme.defaults.email = "bagel-acme@lahfa.xyz";
security.acme.acceptTerms = true; security.acme.acceptTerms = true;

View file

@ -57,7 +57,15 @@
''); '');
}; };
}; };
defaultApp.${system} = self.apps.${system}.apply; apps.${system}.default = self.apps.${system}.apply;
devShells.${system}.default = pkgs.mkShell {
packages = [
inputs.agenix.packages.${system}.agenix
inputs.colmena.packages.${system}.colmena
];
};
colmena = { colmena = {
meta.nixpkgs = import nixpkgs { meta.nixpkgs = import nixpkgs {
localSystem = system; localSystem = system;

View file

@ -40,6 +40,7 @@
hydra.enable = true; hydra.enable = true;
hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra"; hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
}; };
bagel.meta.monitoring.address = "bagel-box.delroth.net";
security.acme.acceptTerms = true; security.acme.acceptTerms = true;
security.acme.defaults.email = "bagel@delroth.net"; security.acme.defaults.email = "bagel@delroth.net";

View file

@ -24,6 +24,7 @@
}; };
}; };
}; };
bagel.meta.monitoring.address = "gerrit01.infra.forkos.org";
fileSystems."/gerrit-data" = { fileSystems."/gerrit-data" = {
device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4"; device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";

View file

@ -21,6 +21,9 @@
enable = true; enable = true;
domain = "netbox.forkos.org"; domain = "netbox.forkos.org";
}; };
bagel.meta.monitoring.address = "meta01.infra.forkos.org";
bagel.services.prometheus.enable = true;
bagel.services.grafana.enable = true;
i18n.defaultLocale = "fr_FR.UTF-8"; i18n.defaultLocale = "fr_FR.UTF-8";

View file

@ -7,6 +7,8 @@ let
hydra-s3-credentials = [ machines.bagel-box ]; hydra-s3-credentials = [ machines.bagel-box ];
hydra-ssh-key-priv = [ machines.bagel-box ]; hydra-ssh-key-priv = [ machines.bagel-box ];
netbox-environment = [ machines.meta01 ]; netbox-environment = [ machines.meta01 ];
mimir-environment = [ machines.meta01 ];
grafana-oauth-secret = [ machines.meta01 ];
}; };
in in
builtins.listToAttrs ( builtins.listToAttrs (

View file

@ -0,0 +1,7 @@
age-encryption.org/v1
-> ssh-ed25519 j2r2qQ Xl0fSOuF0xNTJrtVGdRLRIszd15LFrG5KCFNvSBK4Go
qSEMBBw90jz4j8elpoUeyS4CTLBhZtNDhLNigesJq+0
-> ssh-ed25519 K3b7BA cKI0twKiuuTKv1Js4jqt5v8cOqpxEMY9dmVghgJtbzw
K5o31XP/nLsswsrMaxnIzCXVUtJqmJWoFglWFsV7+AQ
--- X8pvqCHeCQ0LjzcjIHThkqp6YeOOT8dBMLuktgdgeY4
sZÓ¸ŠíØ[þ²X<C2B2>“¡èÅ®Š5°=÷6)ÇT¿Q†N{•x³I1ƒ!ÓÜøB ƒzš*×íåL~K

Binary file not shown.

View file

@ -4,5 +4,6 @@
./postgres ./postgres
./netbox ./netbox
./gerrit ./gerrit
./monitoring
]; ];
} }

View file

@ -0,0 +1,6 @@
{
imports = [
./exporters
./lgtm
];
}

View file

@ -0,0 +1,34 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.monitoring.exporters.baseline;
inherit (lib) mkEnableOption mkIf;
in
{
options.bagel.monitoring.exporters.baseline.enable = (mkEnableOption "Standard set of exporters") // { default = true; };
config = mkIf cfg.enable {
services.prometheus.exporters.node = {
enable = true;
enabledCollectors = [
"processes"
"systemd"
];
port = 9101;
};
services.cadvisor = {
enable = true;
port = 9102;
listenAddress = "0.0.0.0";
};
bagel.meta.monitoring.exporters = [
{ port = 9101; }
{ port = 9102; }
];
};
}

View file

@ -0,0 +1,32 @@
{
config,
lib,
...
}:
let
inherit (lib) mkOption types;
in
{
imports = [./baseline.nix];
options.bagel = {
meta.monitoring = {
address = mkOption {
description = "Node's public address";
type = types.str;
};
exporters = mkOption {
description = "List of all exporters to scrape";
type = types.listOf (types.submodule {
options.port = mkOption {
description = "Exporter port";
type = types.int;
};
});
default = [];
};
};
};
config.networking.firewall.allowedTCPPorts = map (e: e.port) config.bagel.meta.monitoring.exporters;
}

View file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,6 @@
{
imports = [
./grafana.nix
./prometheus.nix
];
}

View file

@ -0,0 +1,126 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.services.grafana;
inherit (lib) mkEnableOption mkIf;
in
{
options.bagel.services.grafana.enable = mkEnableOption "Grafana frontend";
config = mkIf cfg.enable {
age.secrets.grafana-oauth-secret = {
file = ../../../secrets/grafana-oauth-secret.age;
owner = "grafana";
};
bagel.services.postgres.enable = true;
services = {
grafana = {
enable = true;
settings = {
server = {
domain = "grafana.forkos.org";
http_addr = "127.0.0.1";
http_port = 2342;
root_url = "https://grafana.forkos.org/";
};
database = {
type = "postgres";
user = "grafana";
host = "/run/postgresql";
};
"auth.generic_oauth" = {
enabled = true;
name = "Lix SSO";
client_id = "forkos-grafana";
client_secret = "$__file{${config.age.secrets.grafana-oauth-secret.path}}";
auth_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
token_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
api_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/userinfo";
login_attribute_path = "username";
email_attribute_path = "email";
name_attribute_path = "full_name";
scopes = [
"openid"
"profile"
"email"
"offline_access"
"roles"
];
allow_sign_up = true;
auto_login = true;
allow_assign_grafana_admin = true;
role_attribute_path = "contains(roles[*], 'grafana-admin') && 'GrafanaAdmin' || contains(roles[*], 'grafana-editor') && 'Editor' || 'Viewer'";
};
dashboards.default_home_dashboard_path = "${./dashboards/node_exporter.json}";
feature_toggles.enable = "autoMigrateOldPanels newVizTooltips";
security.angular_support_enabled = false;
};
provision = {
dashboards.settings = {
apiVersion = 1;
providers = [
{
name = "default";
options.path = ./dashboards;
}
];
};
datasources.settings = {
apiVersion = 1;
datasources = [
{
name = "Mimir";
type = "prometheus";
uid = "mimir";
access = "proxy";
url = "http://127.0.0.1:9009/prometheus";
}
];
};
};
};
postgresql = {
ensureDatabases = [ "grafana" ];
ensureUsers = [
{
name = "grafana";
ensureDBOwnership = true;
}
];
};
nginx = let
scfg = config.services.grafana.settings.server;
in {
enable = true;
virtualHosts."${scfg.domain}" = {
enableACME = true;
forceSSL = true;
locations."/" = {
proxyPass = "http://${scfg.http_addr}:${toString scfg.http_port}";
proxyWebsockets = true;
};
};
};
};
};
}

View file

@ -0,0 +1,83 @@
{
config,
lib,
nodes,
...
}:
let
cfg = config.bagel.services.prometheus;
inherit (lib) mkEnableOption mkIf;
forEachMachine = fn: map fn (builtins.attrValues nodes);
allMetas = forEachMachine (machine: {
name = machine.config.networking.hostName;
address = machine.config.bagel.meta.monitoring.address or null;
exporters = machine.config.bagel.meta.monitoring.exporters or [];
});
scrapableMetas = builtins.filter (m: m.address != null && m.exporters != []) allMetas;
toJobConfig = m: {
job_name = m.name;
static_configs = [
{ targets = map (e: m.address + ":" + (toString e.port)) m.exporters; }
];
};
jobConfigs = map toJobConfig scrapableMetas;
in
{
options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
config = mkIf cfg.enable {
age.secrets.mimir-environment.file = ../../../secrets/mimir-environment.age;
services.prometheus = {
enable = true;
enableAgentMode = true;
listenAddress = "127.0.0.1";
port = 9001;
globalConfig.scrape_interval = "15s";
scrapeConfigs = jobConfigs;
remoteWrite = [
{ url = "http://localhost:9009/api/v1/push"; }
];
};
services.mimir = {
enable = true;
extraFlags = ["--config.expand-env=true"];
configuration = {
multitenancy_enabled = false;
common.storage = {
backend = "s3";
s3 = {
endpoint = "s3.delroth.net";
bucket_name = "bagel-mimir";
secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
access_key_id = "\${S3_KEY_ID}";
};
};
server = {
http_listen_port = 9009;
grpc_server_max_recv_msg_size = 104857600;
grpc_server_max_send_msg_size = 104857600;
grpc_server_max_concurrent_streams = 1000;
};
ingester.ring.replication_factor = 1;
blocks_storage.backend = "s3";
ruler_storage = {
backend = "local";
local.directory = ./alerts;
};
};
};
systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
};
}