feat(monitoring): add uptime-kuma for status page, fixes #97

Adds a service for a status page using
[`uptime-kuma`](https://uptime.kuma.pet/).

Open questions here included:
- who could set up the application password (currently a manual step in
services.uptime-kuma), after which the stateless client can be re-built;

common/admins.nix

@@ -1,7 +1,7 @@
 let
   keys = import ./ssh-keys.nix;
 in {
-  users.users.root.openssh.authorizedKeys.keys = 
+  users.users.root.openssh.authorizedKeys.keys =
     keys.users.delroth ++
     keys.users.emilylange ++
     keys.users.hexchen ++
@@ -12,5 +12,6 @@ in {
     keys.users.maxine ++
     keys.users.raito ++
     keys.users.thubrecht ++
-    keys.users.yuka;
+    keys.users.yuka ++
+    keys.users.winter;
 }

common/ssh-keys.nix

@@ -51,5 +51,6 @@
     ];
     thubrecht = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPM1jpXR7BWQa7Sed7ii3SbvIPRRlKb3G91qC0vOwfJn" ];
     yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKath4/fDnlv/4fzxkPrQN1ttmoPRNu/m9bEtdPJBDfY cardno:16_933_242" ];
+    winter = [ "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIH/LDRUG+U+++UmlxvA2kspioTjktQZ8taDcHq8gVlkfAAAABHNzaDo=" ];
   };
 }

flake.lock

@@ -715,6 +715,7 @@
         ],
         "nix-gerrit": "nix-gerrit",
         "nixpkgs": "nixpkgs_2",
+        "stateless-uptime-kuma": "stateless-uptime-kuma",
         "terranix": "terranix"
       }
     },
@@ -763,6 +764,22 @@
         "type": "github"
       }
     },
+    "stateless-uptime-kuma": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1713725430,
+        "narHash": "sha256-e3a4/7bc3GO8/kfFndtDa4/6ob3+XjkOgrN8SfDec8c=",
+        "ref": "refs/heads/master",
+        "rev": "c6baf60295e4bee4e4c13cf5c628ccd3ab89b141",
+        "revCount": 22,
+        "type": "git",
+        "url": "https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git"
+      }
+    },
     "systems": {
       "locked": {
         "lastModified": 1681028828,

flake.nix

@@ -28,6 +28,9 @@
     channel-scripts.url = "git+https://git.lix.systems/the-distro/channel-scripts.git";
     channel-scripts.inputs.nixpkgs.follows = "nixpkgs";
 
+    stateless-uptime-kuma.url = "git+https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git";
+    stateless-uptime-kuma.flake = false;
+
     lix.follows = "hydra/lix";
 
     grapevine = {
@@ -55,6 +58,7 @@
           inputs.lix.overlays.default
           inputs.nix-gerrit.overlays.default
           inputs.channel-scripts.overlays.default
+          (import "${inputs.stateless-uptime-kuma}/overlay.nix")
         ];
       };
       terraform = pkgs.opentofu;

hosts/public01/default.nix

@@ -9,6 +9,11 @@
   # TODO: make it the default
   networking.domain = "infra.forkos.org";
 
+  bagel.status = {
+    enable = true;
+    domain = "status.forkos.org";
+  };
+
   bagel.sysadmin.enable = true;
   # Newsletter is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;

secrets.nix

@@ -41,6 +41,7 @@ let
 
     newsletter-secrets = [ machines.public01 ];
     s3-revproxy-api-keys = [ machines.public01 ];
+    stateless-uptime-kuma-password = [ machines.public01 ];
   };
 in
   builtins.listToAttrs (

secrets/stateless-uptime-kuma-password.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 CyxfgQ D2o8bUccO13DKF4COLBQ9mJbACsE2XsRa5S+N71WnTk
+ZaldT7HhQxbxf2ptIwdMYkC60eGtzihc7uwcAkq7s00
+-> ssh-ed25519 K3b7BA AiUCG5CnNyv1DPu+iEwEgW9GqZ8zgpgxKJTAp350ADc
+cUVaDv7F1haQIF11/UhhDAR5DrfJlPttGfDjkv+z9vY
+-> ssh-ed25519 +qVung 1JXeXyea+2Pcwoln/NLRiR8IPPIiB3gaFCP4imyv4DA
+JWmAY6ZnyU46KxzhRrQigGmUPba9lJDDyRQ2GjQShqc
+-> ssh-rsa krWCLQ
+ciLu/+cXfQrB1ms8oTv+xi4eADyL4j0qwnY/6TE0wAXkQHuNXDmpF6ccWZoS2DqN
+NcnGXL6+WyWxmwlyBEq/rsBPvi1g0M6Md7Z4gXn2UvjJ+S7WyA8QEwkxoTDkJS7x
+k/NvtunmggVsWVK4Xdi5DKRw+f32qr/8GysDhIPrTt43iReBKNbyuYWmC5Ec85ep
+JU4JzCNZjJ07kixS5Y9BhaJbpEr47lCXE/KtJUvm3VAxS9IwfUn7KHHdFWynbExi
+F898j3zOR/kgYmeA0oTiexRD3Y2LCvjXIHQZ3MobbZ/PBrjWxe78Sw2vy2t5JLtB
+gFG0K8M1z8DT6a8TtvXEgg
+-> ssh-ed25519 /vwQcQ kUM21TO9iSa8oVXMlNxR7Kc+8TV4C/uTzyQ+t3xnARA
+oXt+egWWONsKT48H4vZ2CPdy3Zfb2QeQVe9l7dDyO/w
+-> ssh-ed25519 0R97PA e/piqf2RD5QgPaQs6jsJdzJgfZR9n1JDIWpbvLZErSs
+UTJH8POFdZ4+N9WkLoNESl1pvcVD0MS1qn7AdS/mg34
+--- 9aYEP0eHDKMacIf09h+OJqIYw+N99+FrW/x/do8Lbo4
+$ ÖëWÛ\zú—¾=s/à@.Ç,?ƒW6n^ù#–i!§Ã–ï¶1]±Nvù±Ž'Ï¥¹6?‚'mµpPÒqýŸº

services/default.nix

@@ -5,6 +5,7 @@
     ./hydra
     ./matrix
     ./monitoring
+    ./uptime-kuma
     ./netbox
     ./ofborg
     ./postgres

services/gerrit/default.nix

@@ -41,6 +41,7 @@ in
   imports = [
     ./www.nix
     ./one-way-sync.nix
+    ./git-gc-preserve.nix
   ];
 
   config = mkIf cfg.enable {
@@ -318,6 +319,13 @@ in
       environment.REVWALK_USE_PRIORITY_QUEUE = "true";
     };
 
+    bagel.services.git-gc-preserve = {
+      nixpkgs = {
+        enable = true;
+        repoPath = "/var/lib/gerrit/git/nixpkgs.git";
+      };
+    };
+
     age.secrets.gerrit-prometheus-bearer-token.file = ../../secrets/gerrit-prometheus-bearer-token.age;
     bagel.monitoring.grafana-agent.exporters.gerrit = {
       port = 4778;  # grrt

services/gerrit/git-gc-preserve.nix

@@ -0,0 +1,86 @@
+{ lib, utils, config, pkgs, ... }: let
+  inherit (lib) mkOption mkEnableOption types;
+  cfg = config.bagel.services.git-gc-preserve;
+  enabledServices = lib.filterAttrs (_: gcConfig: gcConfig.enable) cfg;
+in
+{
+  options.bagel.services.git-gc-preserve = mkOption {
+    default = { };
+    description = "Repositories that should be garbage collected";
+    type = types.attrsOf (types.submodule {
+      options = {
+        enable = mkEnableOption "git-gc-preserve";
+        user = mkOption {
+          type = types.str;
+          default = "git";
+          description = "The user which will run the garbage collection script";
+          example = "forgejo";
+        };
+        group = mkOption {
+          type = types.str;
+          default = "git";
+          description = "The group which will run the garbage collection script";
+          example = "forgejo";
+        };
+        repoPath = mkOption {
+          type = types.path;
+          description = "The path to the git repository that should be garbage collected";
+          example = "/var/lib/gerrit/git/nixpkgs";
+        };
+        timeoutSec = mkOption {
+          type = types.str;
+          default = "1h";
+          description = "Garbage collection Systemd unit timeout";
+          example = "infinity";
+        };
+        timerConfig = mkOption {
+          type = types.attrsOf utils.systemdUtils.unitOptions.unitOption;
+          default = {
+            OnCalendar = "daily";
+          };
+          description = ''
+            When to run the git-gc-preserve. See {manpage}`systemd.timer(5)` for details.
+          '';
+          example = {
+            OnCalendar = "00:05";
+            RandomizedDelaySec = "5h";
+            Persistent = true;
+          };
+        };
+      };
+    });
+  };
+  config = {
+    systemd.services =
+      let
+        mkGCService = name: gcConfig: {
+          name = "git-gc-preserve-${name}";
+          value = {
+            description = "Git-GC-Preserve Service - ${name}";
+            serviceConfig = {
+              WorkingDirectory = gcConfig.repoPath;
+              Type = "oneshot";
+              User = gcConfig.user;
+              Group = gcConfig.group;
+              ExecStart = lib.getExe pkgs.git-gc-preserve;
+              TimeoutSec = gcConfig.timeoutSec;
+            };
+          };
+        };
+        mkServices = lib.mapAttrs' mkGCService;
+      in
+        mkServices enabledServices;
+
+    systemd.timers = let
+      mkGCTimer = name: gcConfig: {
+        name = "git-gc-preserve-${name}";
+        value = {
+          wantedBy = [ "timers.target" ];
+          after = [ "multi-user.target" ];
+          timerConfig = gcConfig.timerConfig;
+        };
+      };
+      mkTimer = lib.mapAttrs' mkGCTimer;
+    in mkTimer enabledServices;
+  };
+}

services/uptime-kuma/default.nix

@@ -0,0 +1,93 @@
+{
+  inputs,
+  lib,
+  config,
+  ...
+}:
+let
+  cfg = config.bagel.status;
+  # TODO: pull domains from a central place
+  subdomains = [
+    "cl"
+    "netbox"
+    "cache"
+    "grafana"
+    "hydra"
+    "loki"
+    "mimir"
+    "pyroscope"
+    "matrix"
+    "tempo"
+    "amqp"
+    "fodwatch"
+    "git"
+    "alerts"
+    "buildbot"
+    "b"
+    "postgres"
+    "news"
+  ];
+  port = 3001;
+in
+{
+  imports = [ "${inputs.stateless-uptime-kuma}/nixos/module.nix" ];
+
+  options.bagel.status = {
+    enable = lib.mkEnableOption "the status page service (uptime-kuma)";
+    domain = lib.mkOption {
+      type = lib.types.str;
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.uptime-kuma.enable = true;
+
+    services.nginx = {
+      enable = true;
+      virtualHosts.${cfg.domain} = {
+        enableACME = true;
+        forceSSL = true;
+        locations."/" = {
+          proxyPass = "http://127.0.0.1:${builtins.toString port}";
+          proxyWebsockets = true;
+        };
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = [
+      80
+      443
+    ];
+
+    statelessUptimeKuma = {
+      probesConfig = {
+        monitors = lib.genAttrs subdomains (name: {
+          type = "http";
+          url = "https://${name}.forkos.org/";
+          tags = [];
+        });
+        status_pages = {
+          "forkos" = {
+            title = "ForkOS";
+            description = "health of the ForkOS infra";
+            showTags = true;
+            publicGroupList = [
+              {
+                name = "Services";
+                weight = 1;
+                monitorList = lib.genAttrs subdomains (id: {
+                  inherit id;
+                });
+              }
+            ];
+          };
+        };
+      };
+      extraFlags = [ "-s" ];
+      host = "http://localhost:${builtins.toString port}/";
+      username = "forkos";
+      passwordFile = config.age.secrets."stateless-uptime-kuma-password".path;
+      enableService = true;
+    };
+  };
+}

terraform/dnsimple.nix

@@ -114,6 +114,7 @@ in
           (record "b" 300 "CNAME" "public01.infra.p.forkos.org")
           (record "postgres" 300 "CNAME" "bagel-box.infra.p.forkos.org")
           (record "news" 3600 "CNAME" "public01.infra.p.forkos.org")
+          (record "status" 3600 "CNAME" "public01.infra.p.forkos.org")
 
           # S3 in delroth's basement
           (record "cache" 300 "AAAA" "2a02:168:6426::12") # smol.delroth.net

terraform/gandi.nix

@@ -88,6 +88,7 @@ in
       (record "b" 300 "CNAME" ["public01.infra.p"])
       (record "postgres" 300 "CNAME" ["bagel-box.infra.p"])
       (record "news" 3600 "CNAME" ["public01.infra.p"])
+      (record "status" 3600 "CNAME" ["public01.infra.p"])
 
       # S3 in delroth's basement
       (record "cache" 300 "AAAA" ["2a02:168:6426::12"]) # smol.delroth.net