Add Grafana/Prometheus/Mimir minimal setup

More later, Loki also later.

.envrc

@@ -0,0 +1,2 @@
+# shellcheck shell=bash
+use flake

.gitignore

@@ -1,3 +1,4 @@
 result
 .gcroots
 config.tf.json
+.direnv

common/base-server.nix

@@ -1,7 +1,7 @@
 { lib, pkgs, ... }: {
   nixpkgs.overlays = import ../overlays;
 
-  nix.package = pkgs.lix;
+  nix.package = lib.mkDefault pkgs.lix;
   services.openssh.enable = lib.mkForce true;
 
   networking.firewall.enable = true;

flake.nix

@@ -57,7 +57,15 @@
           '');
         };
     };
-    defaultApp.${system} = self.apps.${system}.apply;
+    apps.${system}.default = self.apps.${system}.apply;
+
+    devShells.${system}.default = pkgs.mkShell {
+      packages = [ 
+        inputs.agenix.packages.${system}.agenix
+        inputs.colmena.packages.${system}.colmena
+      ];
+    };
+
     colmena = {
       meta.nixpkgs = import nixpkgs {
         localSystem = system;
@@ -74,7 +82,7 @@
           inputs.hydra.nixosModules.hydra
 
           ./services
-
+          ./common
           ./hosts/bagel-box
         ];
       };

hosts/bagel-box/default.nix

@@ -40,6 +40,7 @@
     hydra.enable = true;
     hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
   };
+  bagel.meta.monitoring.address = "bagel-box.delroth.net";
 
   security.acme.acceptTerms = true;
   security.acme.defaults.email = "bagel@delroth.net";

hosts/cl.forkos.org/default.nix

@@ -24,6 +24,7 @@
       };
     };
   };
+  bagel.meta.monitoring.address = "gerrit01.infra.forkos.org";
 
   fileSystems."/gerrit-data" = {
     device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";

hosts/meta01.nixpkgs.lahfa.xyz/default.nix

@@ -21,6 +21,9 @@
     enable = true;
     domain = "netbox.forkos.org";
   };
+  bagel.meta.monitoring.address = "meta01.infra.forkos.org";
+  bagel.services.prometheus.enable = true;
+  bagel.services.grafana.enable = true;
 
   i18n.defaultLocale = "fr_FR.UTF-8";
 

secrets.nix

@@ -7,6 +7,8 @@ let
     hydra-s3-credentials = [ machines.bagel-box ];
     hydra-ssh-key-priv = [ machines.bagel-box ];
     netbox-environment = [ machines.meta01 ];
+    mimir-environment = [ machines.meta01 ];
+    grafana-oauth-secret = [ machines.meta01 ];
   };
 in
   builtins.listToAttrs (

secrets/grafana-oauth-secret.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ Xl0fSOuF0xNTJrtVGdRLRIszd15LFrG5KCFNvSBK4Go
+qSEMBBw90jz4j8elpoUeyS4CTLBhZtNDhLNigesJq+0
+-> ssh-ed25519 K3b7BA cKI0twKiuuTKv1Js4jqt5v8cOqpxEMY9dmVghgJtbzw
+K5o31XP/nLsswsrMaxnIzCXVUtJqmJWoFglWFsV7+AQ
+--- X8pvqCHeCQ0LjzcjIHThkqp6YeOOT8dBMLuktgdgeY4
+sZÓ¸ŠíØ[þ²X<C2B2>“¡èÅ®Š5°=÷6)ÇT¿Q†‘N{•x³I1ƒ!Ó–ÜøB
ƒzš
*×íåL~K

services/default.nix

@@ -4,5 +4,6 @@
     ./postgres
     ./netbox
     ./gerrit
+    ./monitoring
   ];
 }

services/monitoring/default.nix

@@ -0,0 +1,6 @@
+{
+  imports = [
+    ./exporters
+    ./lgtm
+  ];
+}

services/monitoring/exporters/baseline.nix

@@ -0,0 +1,34 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.monitoring.exporters.baseline;
+  inherit (lib) mkEnableOption mkIf;
+in
+{
+  options.bagel.monitoring.exporters.baseline.enable = (mkEnableOption "Standard set of exporters") // { default = true; };
+
+  config = mkIf cfg.enable {
+    services.prometheus.exporters.node = {
+      enable = true;
+      enabledCollectors = [
+        "processes"
+        "systemd"
+      ];
+      port = 9101;
+    };
+
+    services.cadvisor = {
+      enable = true;
+      port = 9102;
+      listenAddress = "0.0.0.0";
+    };
+
+    bagel.meta.monitoring.exporters = [
+      { port = 9101; }
+      { port = 9102; }
+    ];
+  };
+}

services/monitoring/exporters/default.nix

@@ -0,0 +1,32 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  inherit (lib) mkOption types;
+in
+{
+  imports = [./baseline.nix];
+
+  options.bagel = {
+    meta.monitoring = {
+      address = mkOption {
+        description = "Node's public address";
+        type = types.str;
+      };
+      exporters = mkOption {
+        description = "List of all exporters to scrape";
+        type = types.listOf (types.submodule {
+          options.port = mkOption {
+            description = "Exporter port";
+            type = types.int;
+          };
+        });
+        default = [];
+      };
+    };
+  };
+
+  config.networking.firewall.allowedTCPPorts = map (e: e.port) config.bagel.meta.monitoring.exporters;
+}

services/monitoring/lgtm/default.nix

@@ -0,0 +1,6 @@
+{
+  imports = [
+    ./grafana.nix
+    ./prometheus.nix
+  ];
+}

services/monitoring/lgtm/grafana.nix

@@ -0,0 +1,126 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.services.grafana;
+  inherit (lib) mkEnableOption mkIf;
+in
+{
+  options.bagel.services.grafana.enable = mkEnableOption "Grafana frontend";
+
+  config = mkIf cfg.enable {
+    age.secrets.grafana-oauth-secret = {
+      file = ../../../secrets/grafana-oauth-secret.age;
+      owner = "grafana";
+    };
+
+    bagel.services.postgres.enable = true;
+
+    services = {
+      grafana = {
+        enable = true;
+
+        settings = {
+          server = {
+            domain = "grafana.forkos.org";
+            http_addr = "127.0.0.1";
+            http_port = 2342;
+            root_url = "https://grafana.forkos.org/";
+          };
+
+          database = {
+            type = "postgres";
+            user = "grafana";
+            host = "/run/postgresql";
+          };
+
+          "auth.generic_oauth" = {
+            enabled = true;
+
+            name = "Lix SSO";
+            client_id = "forkos-grafana";
+            client_secret = "$__file{${config.age.secrets.grafana-oauth-secret.path}}";
+
+            auth_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
+            token_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
+            api_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/userinfo";
+
+            login_attribute_path = "username";
+            email_attribute_path = "email";
+            name_attribute_path = "full_name";
+
+            scopes = [
+              "openid"
+              "profile"
+              "email"
+              "offline_access"
+              "roles"
+            ];
+
+            allow_sign_up = true;
+            auto_login = true;
+            allow_assign_grafana_admin = true;
+
+            role_attribute_path = "contains(roles[*], 'grafana-admin') && 'GrafanaAdmin' || contains(roles[*], 'grafana-editor') && 'Editor' || 'Viewer'";
+          };
+
+          dashboards.default_home_dashboard_path = "${./dashboards/node_exporter.json}";
+
+          feature_toggles.enable = "autoMigrateOldPanels newVizTooltips";
+          security.angular_support_enabled = false;
+        };
+
+        provision = {
+          dashboards.settings = {
+            apiVersion = 1;
+            providers = [
+              {
+                name = "default";
+                options.path = ./dashboards;
+              }
+            ];
+          };
+
+          datasources.settings = {
+            apiVersion = 1;
+            datasources = [
+              {
+                name = "Mimir";
+                type = "prometheus";
+                uid = "mimir";
+                access = "proxy";
+                url = "http://127.0.0.1:9009/prometheus";
+              }
+            ];
+          };
+        };
+      };
+
+      postgresql = {
+        ensureDatabases = [ "grafana" ];
+        ensureUsers = [
+          {
+            name = "grafana";
+            ensureDBOwnership = true;
+          }
+        ];
+      };
+
+      nginx = let
+        scfg = config.services.grafana.settings.server;
+      in {
+        enable = true;
+        virtualHosts."${scfg.domain}" = {
+          enableACME = true;
+          forceSSL = true;
+          locations."/" = {
+            proxyPass = "http://${scfg.http_addr}:${toString scfg.http_port}";
+            proxyWebsockets = true;
+          };
+        };
+      };
+    };
+};
+}

services/monitoring/lgtm/prometheus.nix

@@ -0,0 +1,83 @@
+{
+  config,
+  lib,
+  nodes,
+  ...
+}:
+let
+  cfg = config.bagel.services.prometheus;
+  inherit (lib) mkEnableOption mkIf;
+
+  forEachMachine = fn: map fn (builtins.attrValues nodes);
+
+  allMetas = forEachMachine (machine: {
+    name = machine.config.networking.hostName;
+    address = machine.config.bagel.meta.monitoring.address or null;
+    exporters = machine.config.bagel.meta.monitoring.exporters or [];
+  });
+
+  scrapableMetas = builtins.filter (m: m.address != null && m.exporters != []) allMetas;
+
+  toJobConfig = m: {
+    job_name = m.name;
+    static_configs = [
+      { targets = map (e: m.address + ":" + (toString e.port)) m.exporters; }
+    ];
+  };
+
+  jobConfigs = map toJobConfig scrapableMetas;
+in
+{
+  options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
+
+  config = mkIf cfg.enable {
+    age.secrets.mimir-environment.file = ../../../secrets/mimir-environment.age;
+
+    services.prometheus = {
+      enable = true;
+      enableAgentMode = true;
+      listenAddress = "127.0.0.1";
+      port = 9001;
+      globalConfig.scrape_interval = "15s";
+      scrapeConfigs = jobConfigs;
+      remoteWrite = [
+        { url = "http://localhost:9009/api/v1/push"; }
+      ];
+    };
+
+    services.mimir = {
+      enable = true;
+      extraFlags = ["--config.expand-env=true"];
+      configuration = {
+        multitenancy_enabled = false;
+
+        common.storage = {
+          backend = "s3";
+          s3 = {
+            endpoint = "s3.delroth.net";
+            bucket_name = "bagel-mimir";
+            secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
+            access_key_id = "\${S3_KEY_ID}";
+          };
+        };
+
+        server = {
+          http_listen_port = 9009;
+          grpc_server_max_recv_msg_size = 104857600;
+          grpc_server_max_send_msg_size = 104857600;
+          grpc_server_max_concurrent_streams = 1000;
+        };
+
+        ingester.ring.replication_factor = 1;
+
+        blocks_storage.backend = "s3";
+        ruler_storage = {
+          backend = "local";
+          local.directory = ./alerts;
+        };
+      };
+    };
+
+    systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
+  };
+}