Switch to push metrics with Grafana Agent

Still TODO: actually connect it to Matrix

.gitignore

@@ -2,3 +2,4 @@ result
 .gcroots
 config.tf.json
 .direnv
+.terraform

flake.nix

@@ -31,7 +31,12 @@
     terraform = pkgs.opentofu;
     terraformCfg = terranix.lib.terranixConfiguration {
       inherit system;
-      modules = [ ];
+      modules = [ 
+        ./terraform
+        {
+          bagel.hydra.enable = true;
+        }
+      ];
     };
   in
   {
@@ -46,23 +51,36 @@
             ${lib.getExe terraform} apply
           '');
         };
-        # nix run ".#destroy"
+      plan = {
-        destroy = {
+        type = "app";
-          type = "app";
+        program = toString (pkgs.writers.writeBash "plan" ''
-          program = toString (pkgs.writers.writeBash "destroy" ''
             set -eo pipefail
-            ln -snf ${terraformCfg} config.tf.json
+            rm -f config.tf.json
+            cp ${terraformCfg} config.tf.json
             ${lib.getExe terraform} init
-            ${lib.getExe terraform} destroy
+            ${lib.getExe terraform} plan
           '');
         };
+      # nix run ".#destroy"
+      destroy = {
+        type = "app";
+        program = toString (pkgs.writers.writeBash "destroy" ''
+          set -eo pipefail
+          ln -snf ${terraformCfg} config.tf.json
+          ${lib.getExe terraform} init
+          ${lib.getExe terraform} destroy
+        '');
+        };
+
+      default = self.apps.${system}.apply;
     };
-    apps.${system}.default = self.apps.${system}.apply;
 
     devShells.${system}.default = pkgs.mkShell {
       packages = [
         inputs.agenix.packages.${system}.agenix
         inputs.colmena.packages.${system}.colmena
+
+        pkgs.opentofu
       ];
     };
 

hosts/bagel-box/default.nix

@@ -40,7 +40,6 @@
     hydra.enable = true;
     hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
   };
-  bagel.meta.monitoring.address = "bagel-box.delroth.net";
 
   security.acme.acceptTerms = true;
   security.acme.defaults.email = "bagel@delroth.net";

hosts/cl.forkos.org/default.nix

@@ -24,7 +24,6 @@
       };
     };
   };
-  bagel.meta.monitoring.address = "gerrit01.infra.forkos.org";
 
   fileSystems."/gerrit-data" = {
     device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";

hosts/fodwatch.forkos.org/default.nix

@@ -24,8 +24,6 @@
     };
   };
 
-  bagel.meta.monitoring.address = "fodwatch.infra.forkos.org";
-
   i18n.defaultLocale = "en_US.UTF-8";
 
   system.stateVersion = "24.05";

hosts/meta01.nixpkgs.lahfa.xyz/default.nix

@@ -21,7 +21,6 @@
     enable = true;
     domain = "netbox.forkos.org";
   };
-  bagel.meta.monitoring.address = "meta01.infra.forkos.org";
   bagel.services.prometheus.enable = true;
   bagel.services.loki.enable = true;
   bagel.services.grafana.enable = true;

services/monitoring/agent.nix

@@ -0,0 +1,100 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.monitoring.grafana-agent;
+  inherit (lib) mkEnableOption mkOption mkIf types;
+  passwordAsCredential = "\${CREDENTIALS_DIRECTORY}/password";
+in
+{
+  options.bagel.monitoring.grafana-agent = {
+    enable = (mkEnableOption "Grafana Agent") // { default = true; };
+
+    exporters = mkOption {
+      description = "List of all exporters to scrape";
+      type = types.listOf (types.submodule {
+        options.port = mkOption {
+          description = "Exporter port";
+          type = types.int;
+        };
+      });
+      default = [];
+    };
+  };
+
+  config = mkIf cfg.enable {
+    age.secrets.grafana-agent-password.file = ../../secrets/metrics-push-password.age;
+
+    services.grafana-agent = {
+      enable = true;
+      credentials.password = config.age.secrets.grafana-agent-password.path;
+      settings = {
+        metrics = {
+          global.remote_write = [
+            {
+              url = "https://mimir.forkos.org/api/v1/push";
+              basic_auth = {
+                username = "promtail";
+                password_file = passwordAsCredential;
+              };
+            }
+          ];
+          configs = [
+            {
+              name = config.networking.hostName;
+              scrape_configs = [
+                {
+                  job_name = config.networking.hostName;
+                  static_configs = [
+                    { targets = map (e: "localhost:" + (toString e.port)) config.bagel.monitoring.grafana-agent.exporters; }
+                  ];
+                }
+              ];
+            }
+          ];
+        };
+        logs = {
+          global.clients = [
+            {
+              url = "https://loki.forkos.org/loki/api/v1/push";
+              basic_auth = {
+                username = "promtail";
+                password_file = passwordAsCredential;
+              };
+            }
+          ];
+          configs = [
+            {
+              name = "journald";
+              scrape_configs = [
+                {
+                  job_name = "system";
+                  journal = {
+                    max_age = "12h";
+                    labels = {
+                      job = "systemd-journal";
+                      host = config.networking.hostName;
+                    };
+                  };
+                  relabel_configs = [
+                    {
+                      source_labels = [ "__journal__systemd_unit" ];
+                      target_label = "unit";
+                    }
+                  ];
+                }
+              ];
+            }
+          ];
+          positions_directory = "\${STATE_DIRECTORY}/positions";
+        };
+        integrations.node_exporter.enable_collectors = [
+          "processes"
+          "systemd"
+        ];
+      };
+    };
+  };
+}

services/monitoring/default.nix

@@ -2,6 +2,6 @@
   imports = [
     ./exporters
     ./lgtm
-    ./promtail.nix
+    ./agent.nix
   ];
 }

services/monitoring/exporters/cadvisor.nix

@@ -17,6 +17,6 @@ in
       listenAddress = "0.0.0.0";
     };
 
-    bagel.meta.monitoring.exporters = [ { port = 9102; } ];
+    bagel.monitoring.grafana-agent.exporters = [ { port = 9102; } ];
   };
 }

services/monitoring/exporters/default.nix

@@ -1,37 +1,7 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  inherit (lib) mkOption types;
-in
 {
   imports = [
     ./cadvisor.nix
-    ./node.nix
     ./nginx.nix
     ./postgres.nix
   ];
-
-  options.bagel = {
-    meta.monitoring = {
-      address = mkOption {
-        description = "Node's public address";
-        type = types.str;
-      };
-      exporters = mkOption {
-        description = "List of all exporters to scrape";
-        type = types.listOf (types.submodule {
-          options.port = mkOption {
-            description = "Exporter port";
-            type = types.int;
-          };
-        });
-        default = [];
-      };
-    };
-  };
-
-  config.networking.firewall.allowedTCPPorts = map (e: e.port) config.bagel.meta.monitoring.exporters;
 }

services/monitoring/exporters/nginx.nix

@@ -30,7 +30,7 @@ in
       ];
     };
 
-    bagel.meta.monitoring.exporters = [
+    bagel.monitoring.grafana-agent.exporters = [
       { port = 9103; }
     ];
   };

services/monitoring/exporters/node.nix

@@ -1,25 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.bagel.monitoring.exporters.node;
-  inherit (lib) mkEnableOption mkIf;
-in
-{
-  options.bagel.monitoring.exporters.node.enable = (mkEnableOption "Standard node_exporter") // { default = true; };
-
-  config = mkIf cfg.enable {
-    services.prometheus.exporters.node = {
-      enable = true;
-      enabledCollectors = [
-        "processes"
-        "systemd"
-      ];
-      port = 9101;
-    };
-
-    bagel.meta.monitoring.exporters = [ { port = 9101; } ];
-  };
-}

services/monitoring/exporters/postgres.nix

@@ -24,7 +24,7 @@ in
 
     services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
 
-    bagel.meta.monitoring.exporters = [
+    bagel.monitoring.grafana-agent.exporters = [
       { port = 9104; }
     ];
   };

services/monitoring/lgtm/alerts/demo.yaml

@@ -0,0 +1,5 @@
+groups:
+  - name: Demo alerts
+    rules:
+      - alert: Demo alert
+        expr: 1

services/monitoring/lgtm/default.nix

@@ -2,6 +2,6 @@
   imports = [
     ./grafana.nix
     ./loki.nix
-    ./prometheus.nix
+    ./mimir.nix
   ];
 }

services/monitoring/lgtm/grafana.nix

@@ -92,6 +92,7 @@ in
                 uid = "mimir";
                 access = "proxy";
                 url = "http://127.0.0.1:9009/prometheus";
+                isDefault = true;
               }
               {
                 name = "Loki";
@@ -100,6 +101,17 @@ in
                 access = "proxy";
                 url = "http://127.0.0.1:9090/";
               }
+              {
+                name = "Mimir Alertmanager";
+                type = "alertmanager";
+                uid = "mimir-alertmanager";
+                access = "proxy";
+                url = "http://127.0.0.1:9009/";
+                jsonData = {
+                  handleGrafanaManagedAlerts = true;
+                  implementation = "mimir";
+                };
+              }
             ];
           };
         };

services/monitoring/lgtm/loki.nix

@@ -12,8 +12,8 @@ in
 
   config = mkIf cfg.enable {
     age.secrets = {
-      loki-htpasswd = {
+      metrics-push-htpasswd = {
-        file = ../../../secrets/loki-htpasswd.age;
+        file = ../../../secrets/metrics-push-htpasswd.age;
         owner = "nginx";
       };
       loki-environment.file = ../../../secrets/loki-environment.age;
@@ -93,7 +93,7 @@ in
       forceSSL = true;
       locations."/loki/api/v1/push" = {
         proxyPass = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
-        basicAuthFile = config.age.secrets.loki-htpasswd.path;
+        basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
       };
     };
   };

services/monitoring/lgtm/mimir.nix

@@ -0,0 +1,92 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.bagel.services.prometheus;
+  inherit (lib) mkEnableOption mkIf;
+
+  mimirPort = config.services.mimir.configuration.server.http_listen_port;
+in
+{
+  options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
+
+  config = mkIf cfg.enable {
+    age.secrets = {
+      metrics-push-htpasswd = {
+        file = ../../../secrets/metrics-push-htpasswd.age;
+        owner = "nginx";
+      };
+      mimir-environment.file = ../../../secrets/mimir-environment.age;
+    };
+
+    services.mimir = {
+      enable = true;
+      extraFlags = ["--config.expand-env=true"];
+      configuration = {
+        target = "all,alertmanager";
+
+        multitenancy_enabled = false;
+
+        common.storage = {
+          backend = "s3";
+          s3 = {
+            endpoint = "s3.delroth.net";
+            bucket_name = "bagel-mimir";
+            secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
+            access_key_id = "\${S3_KEY_ID}";
+          };
+        };
+
+        server = {
+          http_listen_port = 9009;
+          grpc_server_max_recv_msg_size = 104857600;
+          grpc_server_max_send_msg_size = 104857600;
+          grpc_server_max_concurrent_streams = 1000;
+        };
+
+        ingester.ring.replication_factor = 1;
+
+        blocks_storage.backend = "s3";
+        ruler_storage = {
+          backend = "local";
+          local.directory = pkgs.runCommand "mimir-rules" {} ''
+            mkdir -p $out
+            ln -s ${./alerts} $out/anonymous
+          '';
+        };
+
+        alertmanager = {
+          sharding_ring.replication_factor = 1;
+          fallback_config_file = pkgs.writers.writeYAML "alertmanager.yaml" {
+            route = {
+              group_by = ["alertname"];
+              receiver = "matrix";
+            };
+            receivers = [
+              {
+                name = "matrix";
+              }
+            ];
+          };
+        };
+        alertmanager_storage.backend = "filesystem";
+
+        ruler.alertmanager_url = "http://localhost:${toString mimirPort}/alertmanager";
+      };
+    };
+
+    systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
+
+    services.nginx.virtualHosts."mimir.forkos.org" = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/api/v1/push" = {
+        proxyPass = "http://localhost:${toString mimirPort}";
+        basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+      };
+    };
+  };
+}

services/monitoring/lgtm/prometheus.nix

@@ -1,83 +0,0 @@
-{
-  config,
-  lib,
-  nodes,
-  ...
-}:
-let
-  cfg = config.bagel.services.prometheus;
-  inherit (lib) mkEnableOption mkIf;
-
-  forEachMachine = fn: map fn (builtins.attrValues nodes);
-
-  allMetas = forEachMachine (machine: {
-    name = machine.config.networking.hostName;
-    address = machine.config.bagel.meta.monitoring.address or null;
-    exporters = machine.config.bagel.meta.monitoring.exporters or [];
-  });
-
-  scrapableMetas = builtins.filter (m: m.address != null && m.exporters != []) allMetas;
-
-  toJobConfig = m: {
-    job_name = m.name;
-    static_configs = [
-      { targets = map (e: m.address + ":" + (toString e.port)) m.exporters; }
-    ];
-  };
-
-  jobConfigs = map toJobConfig scrapableMetas;
-in
-{
-  options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
-
-  config = mkIf cfg.enable {
-    age.secrets.mimir-environment.file = ../../../secrets/mimir-environment.age;
-
-    services.prometheus = {
-      enable = true;
-      enableAgentMode = true;
-      listenAddress = "127.0.0.1";
-      port = 9001;
-      globalConfig.scrape_interval = "15s";
-      scrapeConfigs = jobConfigs;
-      remoteWrite = [
-        { url = "http://localhost:9009/api/v1/push"; }
-      ];
-    };
-
-    services.mimir = {
-      enable = true;
-      extraFlags = ["--config.expand-env=true"];
-      configuration = {
-        multitenancy_enabled = false;
-
-        common.storage = {
-          backend = "s3";
-          s3 = {
-            endpoint = "s3.delroth.net";
-            bucket_name = "bagel-mimir";
-            secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
-            access_key_id = "\${S3_KEY_ID}";
-          };
-        };
-
-        server = {
-          http_listen_port = 9009;
-          grpc_server_max_recv_msg_size = 104857600;
-          grpc_server_max_send_msg_size = 104857600;
-          grpc_server_max_concurrent_streams = 1000;
-        };
-
-        ingester.ring.replication_factor = 1;
-
-        blocks_storage.backend = "s3";
-        ruler_storage = {
-          backend = "local";
-          local.directory = ./alerts;
-        };
-      };
-    };
-
-    systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
-  };
-}

services/monitoring/promtail.nix

@@ -1,53 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.bagel.monitoring.promtail;
-  inherit (lib) mkEnableOption mkIf;
-in
-{
-  options.bagel.monitoring.promtail.enable = (mkEnableOption "Promtail log export") // { default = true; };
-
-  config = mkIf cfg.enable {
-    age.secrets.promtail-password = {
-      file = ../../secrets/promtail-password.age;
-      owner = "promtail";
-    };
-
-    services.promtail = {
-      enable = true;
-      configuration = {
-        server.disable = true;
-        clients = [ 
-          {
-            url = "https://loki.forkos.org/loki/api/v1/push";
-            basic_auth = {
-              username = "promtail";
-              password_file = config.age.secrets.promtail-password.path;
-            };
-          }
-        ];
-        scrape_configs = [
-          {
-            job_name = "system";
-            journal = {
-              max_age = "12h";
-              labels = {
-                job = "systemd-journal";
-                host = config.networking.hostName;
-              };
-            };
-            relabel_configs = [
-              {
-                source_labels = [ "__journal__systemd_unit" ];
-                target_label = "unit";
-              }
-            ];
-          }
-        ];
-      };
-    };
-  };
-}

terraform/default.nix

@@ -0,0 +1,6 @@
+{
+  imports = [
+    ./hydra.nix
+    ./state.nix
+  ];
+}

terraform/hydra.nix

@@ -0,0 +1,72 @@
+{ lib, config, ... }:
+let
+  inherit (lib) mkEnableOption mkIf types mkOption;
+  cfg = config.bagel.hydra;
+in
+{
+  options.bagel.hydra = {
+    enable = mkEnableOption "the Hydra jobsets";
+  };
+
+  config = mkIf cfg.enable {
+    terraform.required_providers.hydra = {
+      version = "~> 0.1";
+      source = "DeterminateSystems/hydra";
+    };
+
+    provider.hydra = {
+      host = "https://hydra.bagel.delroth.net";
+      # username/password are provided via HYDRA_USERNAME/HYDRA_PASSWORD
+    };
+
+    resource.hydra_project.forkos = {
+      name = "forkos";
+      display_name = "ForkOS";
+      description = "ForkOS packages collection";
+      homepage = "https://cl.forkos.org";
+      owner = "raito";
+      enabled = true;
+      visible = true;
+    };
+
+    resource.hydra_jobset.raito-nixos-rolling-small = {
+      project = config.resource.hydra_project.forkos.name;
+      state = "enabled";
+      visible = true;
+      name = "raito-nixos-rolling-small";
+      type = "legacy";
+      description = "master branch for raito-nixos";
+
+      nix_expression = {
+        file = "pkgs/top-level/release.nix";
+        input = "nixpkgs";
+      };
+
+      check_interval = 0;
+      scheduling_shares = 3000;
+      keep_evaluations = 3;
+
+      email_notifications = false;
+
+      input = [
+        {
+          name = "nixpkgs";
+          type = "git";
+          value = "https://cl.forkos.org/nixpkgs";
+          notify_committers = false;
+        }
+        {
+          name = "officialRelease";
+          type = "boolean";
+          value = "false";
+          notify_committers = false;
+        }
+        {
+          name = "supportedSystems";
+          type = "nix";
+          value = ''[ "x86_64-linux" ]'';
+        }
+      ];
+    };
+  };
+}

terraform/state.nix

@@ -0,0 +1,21 @@
+{
+  # We use terraform.backend.s3 directly instead of the type-checked Terranix
+  # backend.s3 options. The latter does not support setting arbitrary s3
+  # endpoints.
+  #
+  # Note: currently requires the user to provide AWS_ACCESS_KEY_ID as well as
+  # AWS_SECRET_ACCESS_KEY in their environment variables.
+
+  terraform.backend.s3 = {
+    endpoints.s3 = "s3.delroth.net";
+    region = "garage";
+    bucket = "bagel-terraform-state";
+    key = "state";
+
+    # It's just a dump Garage server, don't try to be smart.
+    skip_credentials_validation = true;
+    skip_region_validation = true;
+    skip_requesting_account_id = true;
+    skip_metadata_api_check = true;
+  };
+}