grafana-agent: make bagel.monitoring.grafana-agent.exporters an attrset

This allows us to use multiple jobs, one for each additional exporter,
and set their `job_name` accordingly.

`job_name` is exported as `job` label on the resulting metrics.
This allows us to quickly get an understanding what metrics of an
exporter are actually available by simply filtering all metrics by
`{job="$jobname"}`

.gitignore

@@ -3,3 +3,4 @@ result
 config.tf.json
 .direnv
 .terraform
+.terraform.lock.hcl

flake.nix

@@ -31,9 +31,10 @@
     terraform = pkgs.opentofu;
     terraformCfg = terranix.lib.terranixConfiguration {
       inherit system;
-      modules = [ 
+      modules = [
         ./terraform
         {
+          bagel.gandi.enable = true;
           bagel.hydra.enable = true;
         }
       ];
@@ -41,38 +42,16 @@
   in
   {
     apps.${system} = {
-      apply = {
+      tf = {
         type = "app";
-        program = toString (pkgs.writers.writeBash "apply" ''
+        program = toString (pkgs.writers.writeBash "tf" ''
             set -eo pipefail
-            rm -f config.tf.json
-            cp ${terraformCfg} config.tf.json
-            ${lib.getExe terraform} init
-            ${lib.getExe terraform} apply
+            ln -snf ${terraformCfg} config.tf.json
+            exec ${lib.getExe terraform} "$@"
           '');
-        };
-      plan = {
-        type = "app";
-        program = toString (pkgs.writers.writeBash "plan" ''
-            set -eo pipefail
-            rm -f config.tf.json
-            cp ${terraformCfg} config.tf.json
-            ${lib.getExe terraform} init
-            ${lib.getExe terraform} plan
-          '');
-        };
-      # nix run ".#destroy"
-      destroy = {
-        type = "app";
-        program = toString (pkgs.writers.writeBash "destroy" ''
-          set -eo pipefail
-          ln -snf ${terraformCfg} config.tf.json
-          ${lib.getExe terraform} init
-          ${lib.getExe terraform} destroy
-        '');
-        };
+      };
 
-      default = self.apps.${system}.apply;
+      default = self.apps.${system}.tf;
     };
 
     devShells.${system}.default = pkgs.mkShell {

hosts/bagel-box/default.nix

@@ -40,12 +40,11 @@
     hydra.enable = true;
     hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
   };
-  bagel.meta.monitoring.address = "bagel-box.delroth.net";
 
   security.acme.acceptTerms = true;
-  security.acme.defaults.email = "bagel@delroth.net";
+  security.acme.defaults.email = "infra@forkos.org";
 
   services.openssh.enable = true;
 
-  deployment.targetHost = "bagel-box.delroth.net";
+  deployment.targetHost = "bagel-box.infra.forkos.org";
 }

hosts/cl.forkos.org/default.nix

@@ -24,7 +24,6 @@
       };
     };
   };
-  bagel.meta.monitoring.address = "gerrit01.infra.forkos.org";
 
   fileSystems."/gerrit-data" = {
     device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";

hosts/fodwatch.forkos.org/default.nix

@@ -24,8 +24,6 @@
     };
   };
 
-  bagel.meta.monitoring.address = "fodwatch.infra.forkos.org";
-
   i18n.defaultLocale = "en_US.UTF-8";
 
   system.stateVersion = "24.05";

hosts/meta01.nixpkgs.lahfa.xyz/default.nix

@@ -21,7 +21,6 @@
     enable = true;
     domain = "netbox.forkos.org";
   };
-  bagel.meta.monitoring.address = "meta01.infra.forkos.org";
   bagel.services.prometheus.enable = true;
   bagel.services.loki.enable = true;
   bagel.services.grafana.enable = true;

services/hydra/default.nix

@@ -42,10 +42,10 @@ in {
       port = port;
       dbi = cfg.dbi;
 
-      hydraURL = "https://hydra.bagel.delroth.net";
+      hydraURL = "https://hydra.forkos.org";
       useSubstitutes = false;
 
-      notificationSender = "bagel@delroth.net";
+      notificationSender = "hydra@forkos.org";
 
       # XXX: hydra overlay sets pkgs.hydra, but hydra's nixos module uses
       # pkgs.hydra_unstable...
@@ -108,7 +108,7 @@ in {
         worker_processes auto;
       '';
 
-      virtualHosts."hydra.bagel.delroth.net" = {
+      virtualHosts."hydra.forkos.org" = {
         forceSSL = true;
         enableACME = true;
 

services/monitoring/agent.nix

@@ -0,0 +1,105 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.monitoring.grafana-agent;
+  inherit (lib) mkEnableOption mkOption mkIf types;
+  passwordAsCredential = "\${CREDENTIALS_DIRECTORY}/password";
+in
+{
+  options.bagel.monitoring.grafana-agent = {
+    enable = (mkEnableOption "Grafana Agent") // { default = true; };
+
+    exporters = mkOption {
+      description = ''
+        Set of additional exporters to scrape.
+
+        The attribute name will be used as `job_name`
+        internally, which ends up exported as `job` label
+        on all metrics of that exporter.
+      '';
+      type = types.attrsOf (types.submodule {
+        options.port = mkOption {
+          description = "Exporter port";
+          type = types.int;
+        };
+      });
+      default = {};
+    };
+  };
+
+  config = mkIf cfg.enable {
+    age.secrets.grafana-agent-password.file = ../../secrets/metrics-push-password.age;
+
+    services.grafana-agent = {
+      enable = true;
+      credentials.password = config.age.secrets.grafana-agent-password.path;
+      settings = {
+        metrics = {
+          global.remote_write = [
+            {
+              url = "https://mimir.forkos.org/api/v1/push";
+              basic_auth = {
+                username = "promtail";
+                password_file = passwordAsCredential;
+              };
+            }
+          ];
+          global.external_labels.hostname = config.networking.hostName;
+          configs = [
+            {
+              name = config.networking.hostName;
+              scrape_configs = lib.mapAttrsToList (name: value: {
+                job_name = name;
+                static_configs = [
+                  { targets = "localhost:" + (toString value.port); }
+                ];
+              }) config.bagel.monitoring.grafana-agent.exporters;
+            }
+          ];
+        };
+        logs = {
+          global.clients = [
+            {
+              url = "https://loki.forkos.org/loki/api/v1/push";
+              basic_auth = {
+                username = "promtail";
+                password_file = passwordAsCredential;
+              };
+            }
+          ];
+          configs = [
+            {
+              name = "journald";
+              scrape_configs = [
+                {
+                  job_name = "system";
+                  journal = {
+                    max_age = "12h";
+                    labels = {
+                      job = "systemd-journal";
+                      host = config.networking.hostName;
+                    };
+                  };
+                  relabel_configs = [
+                    {
+                      source_labels = [ "__journal__systemd_unit" ];
+                      target_label = "unit";
+                    }
+                  ];
+                }
+              ];
+            }
+          ];
+          positions_directory = "\${STATE_DIRECTORY}/positions";
+        };
+        integrations.node_exporter.enable_collectors = [
+          "processes"
+          "systemd"
+        ];
+      };
+    };
+  };
+}

services/monitoring/default.nix

@@ -2,6 +2,6 @@
   imports = [
     ./exporters
     ./lgtm
-    ./promtail.nix
+    ./agent.nix
   ];
 }

services/monitoring/exporters/cadvisor.nix

@@ -17,6 +17,6 @@ in
       listenAddress = "0.0.0.0";
     };
 
-    bagel.meta.monitoring.exporters = [ { port = 9102; } ];
+    bagel.monitoring.grafana-agent.exporters.cadvisor.port = 9102;
   };
 }

services/monitoring/exporters/default.nix

@@ -1,37 +1,7 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  inherit (lib) mkOption types;
-in
 {
   imports = [
     ./cadvisor.nix
-    ./node.nix
     ./nginx.nix
     ./postgres.nix
   ];
-
-  options.bagel = {
-    meta.monitoring = {
-      address = mkOption {
-        description = "Node's public address";
-        type = types.str;
-      };
-      exporters = mkOption {
-        description = "List of all exporters to scrape";
-        type = types.listOf (types.submodule {
-          options.port = mkOption {
-            description = "Exporter port";
-            type = types.int;
-          };
-        });
-        default = [];
-      };
-    };
-  };
-
-  config.networking.firewall.allowedTCPPorts = map (e: e.port) config.bagel.meta.monitoring.exporters;
 }

services/monitoring/exporters/nginx.nix

@@ -30,8 +30,6 @@ in
       ];
     };
 
-    bagel.meta.monitoring.exporters = [
-      { port = 9103; }
-    ];
+    bagel.monitoring.grafana-agent.exporters.nginxlog.port = 9103;
   };
 }

services/monitoring/exporters/node.nix

@@ -1,25 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.bagel.monitoring.exporters.node;
-  inherit (lib) mkEnableOption mkIf;
-in
-{
-  options.bagel.monitoring.exporters.node.enable = (mkEnableOption "Standard node_exporter") // { default = true; };
-
-  config = mkIf cfg.enable {
-    services.prometheus.exporters.node = {
-      enable = true;
-      enabledCollectors = [
-        "processes"
-        "systemd"
-      ];
-      port = 9101;
-    };
-
-    bagel.meta.monitoring.exporters = [ { port = 9101; } ];
-  };
-}

services/monitoring/exporters/postgres.nix

@@ -24,8 +24,6 @@ in
 
     services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
 
-    bagel.meta.monitoring.exporters = [
-      { port = 9104; }
-    ];
+    bagel.monitoring.grafana-agent.exporters.postgres.port = 9104;
   };
 }

services/monitoring/lgtm/alerts/demo.yaml

@@ -0,0 +1,5 @@
+groups:
+  - name: Demo alerts
+    rules:
+      - alert: Demo alert
+        expr: 1

services/monitoring/lgtm/default.nix

@@ -2,6 +2,6 @@
   imports = [
     ./grafana.nix
     ./loki.nix
-    ./prometheus.nix
+    ./mimir.nix
   ];
 }

services/monitoring/lgtm/grafana.nix

@@ -92,6 +92,7 @@ in
                 uid = "mimir";
                 access = "proxy";
                 url = "http://127.0.0.1:9009/prometheus";
+                isDefault = true;
               }
               {
                 name = "Loki";
@@ -100,6 +101,17 @@ in
                 access = "proxy";
                 url = "http://127.0.0.1:9090/";
               }
+              {
+                name = "Mimir Alertmanager";
+                type = "alertmanager";
+                uid = "mimir-alertmanager";
+                access = "proxy";
+                url = "http://127.0.0.1:9009/";
+                jsonData = {
+                  handleGrafanaManagedAlerts = true;
+                  implementation = "mimir";
+                };
+              }
             ];
           };
         };

services/monitoring/lgtm/loki.nix

@@ -12,8 +12,8 @@ in
 
   config = mkIf cfg.enable {
     age.secrets = {
-      loki-htpasswd = {
-        file = ../../../secrets/loki-htpasswd.age;
+      metrics-push-htpasswd = {
+        file = ../../../secrets/metrics-push-htpasswd.age;
         owner = "nginx";
       };
       loki-environment.file = ../../../secrets/loki-environment.age;
@@ -93,7 +93,7 @@ in
       forceSSL = true;
       locations."/loki/api/v1/push" = {
         proxyPass = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
-        basicAuthFile = config.age.secrets.loki-htpasswd.path;
+        basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
       };
     };
   };

services/monitoring/lgtm/mimir.nix

@@ -0,0 +1,92 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.bagel.services.prometheus;
+  inherit (lib) mkEnableOption mkIf;
+
+  mimirPort = config.services.mimir.configuration.server.http_listen_port;
+in
+{
+  options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
+
+  config = mkIf cfg.enable {
+    age.secrets = {
+      metrics-push-htpasswd = {
+        file = ../../../secrets/metrics-push-htpasswd.age;
+        owner = "nginx";
+      };
+      mimir-environment.file = ../../../secrets/mimir-environment.age;
+    };
+
+    services.mimir = {
+      enable = true;
+      extraFlags = ["--config.expand-env=true"];
+      configuration = {
+        target = "all,alertmanager";
+
+        multitenancy_enabled = false;
+
+        common.storage = {
+          backend = "s3";
+          s3 = {
+            endpoint = "s3.delroth.net";
+            bucket_name = "bagel-mimir";
+            secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
+            access_key_id = "\${S3_KEY_ID}";
+          };
+        };
+
+        server = {
+          http_listen_port = 9009;
+          grpc_server_max_recv_msg_size = 104857600;
+          grpc_server_max_send_msg_size = 104857600;
+          grpc_server_max_concurrent_streams = 1000;
+        };
+
+        ingester.ring.replication_factor = 1;
+
+        blocks_storage.backend = "s3";
+        ruler_storage = {
+          backend = "local";
+          local.directory = pkgs.runCommand "mimir-rules" {} ''
+            mkdir -p $out
+            ln -s ${./alerts} $out/anonymous
+          '';
+        };
+
+        alertmanager = {
+          sharding_ring.replication_factor = 1;
+          fallback_config_file = pkgs.writers.writeYAML "alertmanager.yaml" {
+            route = {
+              group_by = ["alertname"];
+              receiver = "matrix";
+            };
+            receivers = [
+              {
+                name = "matrix";
+              }
+            ];
+          };
+        };
+        alertmanager_storage.backend = "filesystem";
+
+        ruler.alertmanager_url = "http://localhost:${toString mimirPort}/alertmanager";
+      };
+    };
+
+    systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
+
+    services.nginx.virtualHosts."mimir.forkos.org" = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/api/v1/push" = {
+        proxyPass = "http://localhost:${toString mimirPort}";
+        basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+      };
+    };
+  };
+}

services/monitoring/lgtm/prometheus.nix

@@ -1,83 +0,0 @@
-{
-  config,
-  lib,
-  nodes,
-  ...
-}:
-let
-  cfg = config.bagel.services.prometheus;
-  inherit (lib) mkEnableOption mkIf;
-
-  forEachMachine = fn: map fn (builtins.attrValues nodes);
-
-  allMetas = forEachMachine (machine: {
-    name = machine.config.networking.hostName;
-    address = machine.config.bagel.meta.monitoring.address or null;
-    exporters = machine.config.bagel.meta.monitoring.exporters or [];
-  });
-
-  scrapableMetas = builtins.filter (m: m.address != null && m.exporters != []) allMetas;
-
-  toJobConfig = m: {
-    job_name = m.name;
-    static_configs = [
-      { targets = map (e: m.address + ":" + (toString e.port)) m.exporters; }
-    ];
-  };
-
-  jobConfigs = map toJobConfig scrapableMetas;
-in
-{
-  options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
-
-  config = mkIf cfg.enable {
-    age.secrets.mimir-environment.file = ../../../secrets/mimir-environment.age;
-
-    services.prometheus = {
-      enable = true;
-      enableAgentMode = true;
-      listenAddress = "127.0.0.1";
-      port = 9001;
-      globalConfig.scrape_interval = "15s";
-      scrapeConfigs = jobConfigs;
-      remoteWrite = [
-        { url = "http://localhost:9009/api/v1/push"; }
-      ];
-    };
-
-    services.mimir = {
-      enable = true;
-      extraFlags = ["--config.expand-env=true"];
-      configuration = {
-        multitenancy_enabled = false;
-
-        common.storage = {
-          backend = "s3";
-          s3 = {
-            endpoint = "s3.delroth.net";
-            bucket_name = "bagel-mimir";
-            secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
-            access_key_id = "\${S3_KEY_ID}";
-          };
-        };
-
-        server = {
-          http_listen_port = 9009;
-          grpc_server_max_recv_msg_size = 104857600;
-          grpc_server_max_send_msg_size = 104857600;
-          grpc_server_max_concurrent_streams = 1000;
-        };
-
-        ingester.ring.replication_factor = 1;
-
-        blocks_storage.backend = "s3";
-        ruler_storage = {
-          backend = "local";
-          local.directory = ./alerts;
-        };
-      };
-    };
-
-    systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
-  };
-}

services/monitoring/promtail.nix

@@ -1,53 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.bagel.monitoring.promtail;
-  inherit (lib) mkEnableOption mkIf;
-in
-{
-  options.bagel.monitoring.promtail.enable = (mkEnableOption "Promtail log export") // { default = true; };
-
-  config = mkIf cfg.enable {
-    age.secrets.promtail-password = {
-      file = ../../secrets/promtail-password.age;
-      owner = "promtail";
-    };
-
-    services.promtail = {
-      enable = true;
-      configuration = {
-        server.disable = true;
-        clients = [ 
-          {
-            url = "https://loki.forkos.org/loki/api/v1/push";
-            basic_auth = {
-              username = "promtail";
-              password_file = config.age.secrets.promtail-password.path;
-            };
-          }
-        ];
-        scrape_configs = [
-          {
-            job_name = "system";
-            journal = {
-              max_age = "12h";
-              labels = {
-                job = "systemd-journal";
-                host = config.networking.hostName;
-              };
-            };
-            relabel_configs = [
-              {
-                source_labels = [ "__journal__systemd_unit" ];
-                target_label = "unit";
-              }
-            ];
-          }
-        ];
-      };
-    };
-  };
-}

terraform/common.nix

@@ -0,0 +1,7 @@
+{
+  # Until we get some kind of KMS operational, store secrets in the state file.
+  terraform.required_providers.secret = {
+    version = "~> 1.2.1";
+    source = "numtide/secret";
+  };
+}

terraform/default.nix

@@ -1,5 +1,7 @@
 {
   imports = [
+    ./common.nix
+    ./gandi.nix
     ./hydra.nix
     ./state.nix
   ];

terraform/gandi.nix

@@ -0,0 +1,67 @@
+{ lib, config, ... }:
+let
+  inherit (lib) mkEnableOption mkIf tf;
+  cfg = config.bagel.gandi;
+in
+{
+  options.bagel.gandi = {
+    enable = mkEnableOption "the Gandi DNS configuration";
+  };
+
+  config = mkIf cfg.enable {
+    terraform.required_providers.gandi = {
+      version = "~> 2.3.0";
+      source = "go-gandi/gandi";
+    };
+
+    resource.secret_resource.gandi_pat.lifecycle.prevent_destroy = true;
+
+    provider.gandi = {
+      personal_access_token = tf.ref "resource.secret_resource.gandi_pat.value";
+    };
+
+    resource.gandi_livedns_domain.forkos_org = {
+      name = "forkos.org";
+    };
+
+    resource.gandi_livedns_record = let
+      record = name: ttl: type: values: {
+        inherit name ttl type values;
+      };
+
+      # TODO: make less fragile and have actual unique and stable names
+      canonicalName = record: let
+        name = builtins.replaceStrings ["."] ["_"] record.name;
+      in
+        "forkos_org_${record.type}_${name}";
+
+      forkosRecords = records:
+        builtins.listToAttrs (map (record: {
+          name = canonicalName record;
+          value = record // {
+            zone = tf.ref "resource.gandi_livedns_domain.forkos_org.id";
+          };
+        }) records);
+
+    in forkosRecords [
+      (record "cl" 3600 "A" ["163.172.69.160"])
+      (record "cl" 3600 "AAAA" ["2001:bc8:38ee:100:1000::10"])
+
+      (record "fodwatch" 3600 "A" ["163.172.69.160"])
+      (record "fodwatch" 3600 "AAAA" ["2001:bc8:38ee:100:1000::30"])
+
+      (record "netbox" 3600 "A" ["163.172.69.160"])
+      (record "netbox" 3600 "AAAA" ["2001:bc8:38ee:100:1000::20"])
+
+      (record "bagel-box.infra" 3600 "AAAA" ["2001:bc8:38ee:100:100::1"])
+      (record "gerrit01.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::10"])
+      (record "fodwatch.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::30"])
+      (record "meta01.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::20"])
+
+      (record "grafana" 3600 "CNAME" ["netbox"])
+      (record "hydra" 3600 "CNAME" ["bagel-box.infra"])
+      (record "loki" 3600 "CNAME" ["meta01.infra"])
+      (record "mimir" 3600 "CNAME" ["grafana"])
+    ];
+  };
+}

terraform/hydra.nix

@@ -1,6 +1,6 @@
 { lib, config, ... }:
 let
-  inherit (lib) mkEnableOption mkIf types mkOption;
+  inherit (lib) mkEnableOption mkIf types mkOption tf;
   cfg = config.bagel.hydra;
 in
 {
@@ -14,9 +14,12 @@ in
       source = "DeterminateSystems/hydra";
     };
 
+    resource.secret_resource.hydra_password.lifecycle.prevent_destroy = true;
+
     provider.hydra = {
-      host = "https://hydra.bagel.delroth.net";
-      # username/password are provided via HYDRA_USERNAME/HYDRA_PASSWORD
+      host = "https://hydra.forkos.org";
+      username = "terraform";
+      password = tf.ref "resource.secret_resource.hydra_password.value";
     };
 
     resource.hydra_project.forkos = {
@@ -24,7 +27,7 @@ in
       display_name = "ForkOS";
       description = "ForkOS packages collection";
       homepage = "https://cl.forkos.org";
-      owner = "raito";
+      owner = "terraform";
       enabled = true;
       visible = true;
     };